feat(mobile): environment-backed voice transcription

apps/mobile/README.md

@@ -52,8 +52,8 @@ vp run ios:dev
 ```
 
 If your Xcode account only has a Personal Team, use a bundle identifier you control and opt into the
-reduced-capability local build. Personal Team builds omit the widget and share extensions, push
-entitlement, and native Sign in with Apple entitlement; builds without this opt-in are unchanged.
+reduced-capability local build. Personal Team builds omit widget and share extensions, push
+notifications, associated domains, and native Sign in with Apple.
 
 ```bash
 T3CODE_IOS_PERSONAL_TEAM=1 \

apps/mobile/app.config.ts

@@ -192,14 +192,15 @@ const config: ExpoConfig = {
     // showcase capture build requires full screen (see infoPlist below).
     requireFullScreen: process.env.T3_SHOWCASE_CAPTURE_BUILD === "1",
     bundleIdentifier: iosBundleIdentifier,
-    // Pin code signing to the T3 Tools team so non-interactive `expo run:ios`
-    // does not fall back to a personal team (which cannot sign app groups,
-    // Sign in with Apple, or push notification entitlements).
-    appleTeamId: "ARK85ZXQ4Z",
-    associatedDomains: [
-      `applinks:${variant.relyingParty}`,
-      `webcredentials:${variant.relyingParty}`,
-    ],
+    ...(!isIosPersonalTeamBuild
+      ? {
+          appleTeamId: "ARK85ZXQ4Z",
+          associatedDomains: [
+            `applinks:${variant.relyingParty}`,
+            `webcredentials:${variant.relyingParty}`,
+          ],
+        }
+      : {}),
     infoPlist: {
       NSAppTransportSecurity: {
         NSAllowsArbitraryLoads: true,
@@ -242,6 +243,8 @@ const config: ExpoConfig = {
     favicon: variant.assets.appIcon,
   },
   plugins: [
+    // Same-type mods run last-registered-first; remove restricted entitlements after SDK mods.
+    ...(isIosPersonalTeamBuild ? ["./plugins/withoutIosPersonalTeamCapabilities.cjs"] : []),
     "expo-asset",
     [
       "expo-font",
@@ -280,8 +283,6 @@ const config: ExpoConfig = {
         mode: APP_VARIANT === "development" ? "development" : "production",
       },
     ],
-    // appleSignIn must be gated here: withoutIosPersonalTeamCapabilities.cjs runs before
-    // plugins earlier in this array, so it cannot strip the entitlement Clerk would add.
     ["@clerk/expo", { theme: "./clerk-theme.json", appleSignIn: !isIosPersonalTeamBuild }],
     "expo-web-browser",
     [
@@ -356,7 +357,6 @@ const config: ExpoConfig = {
     "./plugins/withAndroidModernAlertDialog.cjs",
     "./plugins/withAndroidPredictiveBackCompat.cjs",
     "./plugins/withAndroidTabletOrientation.cjs",
-    ...(isIosPersonalTeamBuild ? ["./plugins/withoutIosPersonalTeamCapabilities.cjs"] : []),
   ],
   extra: {
     appVariant: APP_VARIANT,

apps/mobile/src/features/settings/SettingsEnvironmentsRouteScreen.tsx

@@ -1,19 +1,25 @@
 import { NativeHeaderToolbar, NativeStackScreenOptions } from "../../native/StackHeader";
+import { useAtomSet, useAtomValue } from "@effect/atom-react";
+import { AsyncResult } from "effect/unstable/reactivity";
 import { useNavigation } from "@react-navigation/native";
 import { SymbolView } from "../../components/AppSymbol";
 import type { EnvironmentId } from "@t3tools/contracts";
 import { useCallback, useState } from "react";
-import { Platform, ScrollView, View } from "react-native";
+import { Platform, Pressable, ScrollView, View } from "react-native";
 import { useSafeAreaInsets } from "react-native-safe-area-context";
 
 import { AppText as Text } from "../../components/AppText";
+import { ControlPillMenu } from "../../components/ControlPill";
 import { AndroidScreenHeader } from "../../components/AndroidScreenHeader";
 import { CloudEnvironmentRows } from "../connection/CloudEnvironmentRows";
 import { ConnectionEnvironmentRow } from "../connection/ConnectionEnvironmentRow";
 import { splitEnvironmentSections } from "../connection/environmentSections";
 import { cn } from "../../lib/cn";
 import { useUniwindTheme } from "../../lib/useUniwindTheme";
 import { useRemoteConnections } from "../../state/use-remote-environment-registry";
+import { getLocalVoiceTranscriber } from "../../native/voiceTranscription";
+import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences";
+import { serverEnvironment } from "../../state/server";
 import {
   applyShowcaseLocalEnvironmentDisplayUrls,
   resolveShowcaseEnvironmentUpdateDisplayUrl,
@@ -171,7 +177,89 @@ export function SettingsEnvironmentsRouteScreen() {
               }
             : {})}
         />
+        {[...localEnvironments, ...connectedCloudEnvironments].map((environment) => (
+          <EnvironmentTranscriptionRow
+            key={`transcription-${environment.environmentId}`}
+            environmentId={environment.environmentId}
+            environmentLabel={environment.environmentLabel}
+          />
+        ))}
       </ScrollView>
     </View>
   );
 }
+
+function EnvironmentTranscriptionRow(props: {
+  readonly environmentId: EnvironmentId;
+  readonly environmentLabel: string;
+}) {
+  const services = useAtomValue(
+    serverEnvironment.transcriptionServicesValueAtom(props.environmentId),
+  );
+  const preferences = useAtomValue(mobilePreferencesAtom);
+  const savePreferences = useAtomSet(updateMobilePreferencesAtom);
+  if (services.length === 0) return null;
+
+  const localAvailable = getLocalVoiceTranscriber() !== null;
+  const savedSource = AsyncResult.isSuccess(preferences)
+    ? preferences.value.voiceTranscriptionSources?.[props.environmentId]
+    : undefined;
+  const selectedSource = savedSource ?? (localAvailable ? "local" : services[0]?.id);
+  const selectedLabel =
+    selectedSource === "local"
+      ? "On this device"
+      : (services.find((service) => service.id === selectedSource)?.label ?? "Unavailable");
+  const actions = [
+    ...(localAvailable
+      ? [
+          {
+            id: "local",
+            title: "On this device",
+            state: selectedSource === "local" ? ("on" as const) : ("off" as const),
+          },
+        ]
+      : []),
+    ...services.map((service) => ({
+      id: service.id,
+      title: service.label,
+      state: selectedSource === service.id ? ("on" as const) : ("off" as const),
+    })),
+  ];
+
+  return (
+    <ControlPillMenu
+      actions={actions}
+      onPressAction={({ nativeEvent }) => {
+        const current = AsyncResult.isSuccess(preferences)
+          ? (preferences.value.voiceTranscriptionSources ?? {})
+          : {};
+        savePreferences({
+          voiceTranscriptionSources: {
+            ...current,
+            [props.environmentId]: nativeEvent.event,
+          },
+        });
+      }}
+    >
+      <Pressable
+        accessibilityLabel={`${props.environmentLabel} voice transcription: ${selectedLabel}`}
+        accessibilityRole="button"
+        className="mt-3 flex-row items-center justify-between rounded-[24px] bg-card px-4 py-3"
+      >
+        <View className="flex-row items-center gap-3">
+          <SymbolView
+            name="waveform"
+            size={18}
+            tintColorClassName="accent-icon-muted"
+            type="monochrome"
+          />
+          <View>
+            <Text className="text-sm text-foreground">Voice transcription</Text>
+            <Text className="text-xs text-foreground-muted">{props.environmentLabel}</Text>
+          </View>
+        </View>
+        <Text className="text-sm text-foreground-muted">{selectedLabel}</Text>
+      </Pressable>
+    </ControlPillMenu>
+  );
+}

apps/mobile/src/features/threads/NewTaskDraftScreen.tsx

@@ -315,6 +315,7 @@ export function NewTaskDraftScreen(props: {
     onUpdateInteractionMode: flow.planModeEnabled ? flow.setInteractionMode : undefined,
   });
   const voiceInput = useVoiceInputController({
+    environmentId: selectedProject?.environmentId ?? null,
     ownerKey: flow.draftKey,
     draftMessage: flow.prompt,
     selection: composerMenu.selection,

apps/mobile/src/features/threads/ThreadComposer.tsx

@@ -355,6 +355,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer
     onUpdateInteractionMode: props.onUpdateInteractionMode,
   });
   const voiceInput = useVoiceInputController({
+    environmentId: props.environmentId,
     ownerKey: composerOwnerKey,
     draftMessage: props.draftMessage,
     selection: composerMenu.selection,

apps/mobile/src/features/voice-input/environmentVoiceTransport.ts

@@ -0,0 +1,15 @@
+import { File, UploadType } from "expo-file-system";
+import type { EnvironmentVoiceTranscriptionTransport } from "@t3tools/client-runtime/voice-input";
+
+export const environmentVoiceTransport: EnvironmentVoiceTranscriptionTransport = {
+  sizeBytes: async (uri) => new File(uri).size,
+  upload: async ({ uri, url, mimeType, signal }) => {
+    const result = await new File(uri).upload(url, {
+      httpMethod: "POST",
+      uploadType: UploadType.BINARY_CONTENT,
+      headers: { "Content-Type": mimeType },
+      signal,
+    });
+    return { status: result.status, bodyText: result.body };
+  },
+};

apps/mobile/src/features/voice-input/useVoiceInputController.ts

@@ -8,21 +8,33 @@ import {
 } from "expo-audio";
 import { File } from "expo-file-system";
 import { useFocusEffect } from "@react-navigation/native";
+import { useAtomValue } from "@effect/atom-react";
+import type { EnvironmentId } from "@t3tools/contracts";
+import * as Option from "effect/Option";
+import { AsyncResult } from "effect/unstable/reactivity";
 import { useCallback, useEffect, useRef, useState } from "react";
 import { AppState } from "react-native";
 import { useSharedValue } from "react-native-reanimated";
 
 import type { ComposerEditorSelection } from "../../components/ComposerEditor";
 import { getLocalVoiceTranscriber } from "../../native/voiceTranscription";
 import {
+  createEnvironmentVoiceTranscriber,
   VoiceInputController,
   VOICE_RECORDING_LIMIT_SECONDS,
   voiceInputBlocksSubmission,
   voiceInputFreezesEditor,
   type VoiceDraftSnapshot,
   type VoiceInputState,
 } from "@t3tools/client-runtime/voice-input";
+import { resolveAssetUrl } from "@t3tools/client-runtime/state/assets";
+import { environmentVoiceTransport } from "./environmentVoiceTransport";
 import { normalizeVoiceInputDecibels, VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering";
+import { appAtomRegistry } from "../../state/atom-registry";
+import { mobilePreferencesAtom } from "../../state/preferences";
+import { serverEnvironment } from "../../state/server";
+import { environmentSession } from "../../state/session";
+import { transcriptionEnvironment } from "../../state/transcription";
 
 const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
 const VOICE_METERING_INTERVAL_MS = 80;
@@ -61,6 +73,7 @@ async function configureVoiceRecordingAudio(): Promise<void> {
 }
 
 export function useVoiceInputController(input: {
+  readonly environmentId: EnvironmentId | null;
   readonly ownerKey: string | null;
   readonly draftMessage: string;
   readonly selection: ComposerEditorSelection;
@@ -70,6 +83,27 @@ export function useVoiceInputController(input: {
 }) {
   const [state, setState] = useState<VoiceInputState>(INITIAL_STATE);
   const [elapsedSeconds, setElapsedSeconds] = useState(0);
+  const preferences = useAtomValue(mobilePreferencesAtom);
+  const services = useAtomValue(
+    serverEnvironment.transcriptionServicesValueAtom(input.environmentId),
+  );
+  const localTranscriber = getLocalVoiceTranscriber();
+  const selectedSource =
+    input.environmentId !== null && AsyncResult.isSuccess(preferences)
+      ? preferences.value.voiceTranscriptionSources?.[input.environmentId]
+      : undefined;
+  const voiceSelectionRef = useRef({
+    environmentId: input.environmentId,
+    services,
+    localTranscriber,
+    selectedSource,
+  });
+  voiceSelectionRef.current = {
+    environmentId: input.environmentId,
+    services,
+    localTranscriber,
+    selectedSource,
+  };
   const elapsedSecondsRef = useRef(0);
   const audioLevelsRef = useRef(Array<number>(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0));
   const audioLevels = useSharedValue(audioLevelsRef.current);
@@ -99,7 +133,38 @@ export function useVoiceInputController(input: {
   if (!controllerRef.current) {
     controllerRef.current = new VoiceInputController({
       recorder,
-      getTranscriber: getLocalVoiceTranscriber,
+      getTranscriber: () => {
+        const selection = voiceSelectionRef.current;
+        const source =
+          selection.selectedSource ??
+          (selection.localTranscriber !== null ? "local" : selection.services[0]?.id);
+        if (source === "local" || source === undefined) return selection.localTranscriber;
+        if (selection.environmentId === null) return null;
+        const environmentId = selection.environmentId;
+        return createEnvironmentVoiceTranscriber({
+          environmentId,
+          serviceId: source,
+          locale: Intl.DateTimeFormat().resolvedOptions().locale,
+          mimeType: "audio/mp4",
+          transport: environmentVoiceTransport,
+          registry: appAtomRegistry,
+          environment: transcriptionEnvironment,
+          getServices: () =>
+            appAtomRegistry.get(serverEnvironment.transcriptionServicesValueAtom(environmentId)),
+          isConnected: () =>
+            Option.isSome(
+              appAtomRegistry.get(environmentSession.preparedConnectionValueAtom(environmentId)),
+            ),
+          resolveUrl: (relativeUrl) => {
+            const connection = appAtomRegistry.get(
+              environmentSession.preparedConnectionValueAtom(environmentId),
+            );
+            return Option.isSome(connection)
+              ? resolveAssetUrl(connection.value.httpBaseUrl, relativeUrl)
+              : null;
+          },
+        });
+      },
       requestPermission: async () => {
         const permission = await requestRecordingPermissionsAsync();
         return { granted: permission.granted, canAskAgain: permission.canAskAgain };
@@ -203,7 +268,7 @@ export function useVoiceInputController(input: {
   const cancel = useCallback(() => controller.cancel(), [controller]);
 
   return {
-    isAvailable: getLocalVoiceTranscriber() !== null,
+    isAvailable: localTranscriber !== null || services.length > 0,
     state,
     audioLevels,
     elapsedSeconds,

apps/mobile/src/persistence/mobile-preferences.ts

@@ -45,6 +45,8 @@ export interface Preferences {
   readonly threadListV2SettledShelfExpanded?: boolean;
   /** Undefined preserves the default collapsed Snoozed shelf. */
   readonly threadListV2SnoozedShelfExpanded?: boolean;
+  /** Selected voice transcription source by stable environment id. */
+  readonly voiceTranscriptionSources?: Readonly<Record<string, string>>;
 }
 
 export class MobilePreferencesLoadError extends Schema.TaggedErrorClass<MobilePreferencesLoadError>()(
@@ -104,6 +106,7 @@ function sanitizePreferences(parsed: Preferences): Preferences {
     planModeEnabled?: boolean;
     threadListV2SettledShelfExpanded?: boolean;
     threadListV2SnoozedShelfExpanded?: boolean;
+    voiceTranscriptionSources?: Readonly<Record<string, string>>;
   } = {};
 
   if (typeof parsed.liveActivitiesEnabled === "boolean") {
@@ -177,6 +180,18 @@ function sanitizePreferences(parsed: Preferences): Preferences {
   if (typeof parsed.threadListV2SnoozedShelfExpanded === "boolean") {
     preferences.threadListV2SnoozedShelfExpanded = parsed.threadListV2SnoozedShelfExpanded;
   }
+  if (
+    typeof parsed.voiceTranscriptionSources === "object" &&
+    parsed.voiceTranscriptionSources !== null &&
+    !Array.isArray(parsed.voiceTranscriptionSources)
+  ) {
+    preferences.voiceTranscriptionSources = Object.fromEntries(
+      Object.entries(parsed.voiceTranscriptionSources).filter(
+        (entry): entry is [string, string] =>
+          entry[0].length > 0 && typeof entry[1] === "string" && entry[1].length > 0,
+      ),
+    );
+  }
   return preferences;
 }
 

apps/mobile/src/state/transcription.ts

@@ -0,0 +1,5 @@
+import { createTranscriptionEnvironmentAtoms } from "@t3tools/client-runtime/state/transcription";
+
+import { connectionAtomRuntime } from "../connection/runtime";
+
+export const transcriptionEnvironment = createTranscriptionEnvironmentAtoms(connectionAtomRuntime);

apps/server/src/auth/RpcAuthorization.ts

@@ -86,6 +86,7 @@ export const RPC_REQUIRED_SCOPES = {
   [WS_METHODS.assetsCreateUrl]: AuthOrchestrationReadScope,
   [WS_METHODS.attachmentsCreateUploadUrl]: AuthOrchestrationOperateScope,
   [WS_METHODS.attachmentsDelete]: AuthOrchestrationOperateScope,
+  [WS_METHODS.transcriptionCreateUrl]: AuthOrchestrationOperateScope,
   [WS_METHODS.providerUploadFeedback]: AuthOrchestrationOperateScope,
   [WS_METHODS.subscribeVcsStatus]: AuthOrchestrationReadScope,
   [WS_METHODS.subscribeResourceTelemetry]: AuthOrchestrationReadScope,

apps/server/src/environment/ServerEnvironment.ts

@@ -206,6 +206,7 @@ export const make = Effect.gen(function* () {
       repositoryIdentity: true,
       connectionProbe: true,
       attachmentUploads: true,
+      transcription: true,
       fileAttachments: { maxUploadBytes: PROVIDER_SEND_TURN_MAX_FILE_BYTES },
       pullRequests: true,
       threadSettlement: true,