Run Qwen3-ASR natively in Swift

Package.swift

@@ -13,14 +13,17 @@ let package = Package(
     ],
     dependencies: [
         .package(url: "https://github.com/argmaxinc/argmax-oss-swift.git", from: "1.0.0"),
+        .package(url: "https://github.com/Blaizzy/mlx-audio-swift.git", exact: "0.1.3"),
         .package(url: "https://github.com/huggingface/swift-transformers", from: "1.3.3"),
-        .package(url: "https://github.com/ml-explore/mlx-swift-lm", branch: "main"),
+        .package(url: "https://github.com/ml-explore/mlx-swift-lm", exact: "3.31.4"),
     ],
     targets: [
         .executableTarget(
             name: "OpenType",
             dependencies: [
                 .product(name: "WhisperKit", package: "argmax-oss-swift"),
+                .product(name: "MLXAudioCore", package: "mlx-audio-swift"),
+                .product(name: "MLXAudioSTT", package: "mlx-audio-swift"),
                 .product(name: "Hub", package: "swift-transformers"),
                 .product(name: "Tokenizers", package: "swift-transformers"),
                 .product(name: "MLXLLM", package: "mlx-swift-lm"),
@@ -39,7 +42,6 @@ let package = Package(
                 .copy("Resources/SettingsStyleIllustration.png"),
                 .copy("Resources/SettingsIntegrationsIllustration.png"),
                 .copy("Resources/SettingsAboutIllustration.png"),
-                .copy("Resources/Scripts"),
                 .copy("Resources/Sounds"),
                 .copy("Resources/AppIcon.icns"),
                 .copy("Resources/AppIconLight.icns"),

README.md

@@ -38,7 +38,7 @@ Three output modes are available:
 
 | Feature | Description |
 |---|---|
-| **Multiple Speech Engines** | Apple Speech, WhisperKit, Doubao ASR, Qwen3-ASR, or MiMo-V2.5-ASR |
+| **Multiple Speech Engines** | Apple Speech, WhisperKit, Doubao ASR, or Qwen3-ASR |
 | **Smart Text Processing** | Local MLX Qwen2.5/Qwen3 or remote LLM infers spoken intent — contextual cleanup, "scratch that" restarts, self-correction handling, spoken punctuation, technical terms, numbers/ranges/units, and structured formatting |
 | **LLM-Owned Spoken Formatting** | Spoken casing, no-space dictation, identifiers, file paths, shortcuts, emoji, Markdown tasks, dates/times, quantities, units, formulas, fractions, and digit sequences are handled by the Smart Format / Voice Command prompts instead of local hardcoded rewrite rules |
 | **Voice Edit Commands** | In Voice Command mode, an LLM classifies safe structured actions for replacing, undoing, proofreading, titling, summarizing, drafting replies, making meeting notes, extracting key points/decisions/questions/risks/deadlines/owners/action items, rewriting tone, expanding, making tables/lists, or deleting the previous Utter insertion or selected text |
@@ -131,10 +131,9 @@ Utter supports both **OpenAI-compatible** and **Anthropic** API formats:
 
 | Provider | Local runtime | Default model |
 |---|---|---|
-| Qwen3-ASR | `qwen3-asr-mlx` + MLX on Apple Silicon | `mlx-community/Qwen3-ASR-1.7B-bf16` |
-| MiMo-V2.5-ASR | Xiaomi's local Python runtime files + local model folders | `XiaomiMiMo/MiMo-V2.5-ASR` + `XiaomiMiMo/MiMo-Audio-Tokenizer` |
+| Qwen3-ASR | Native Swift + MLX on Apple Silicon | `mlx-community/Qwen3-ASR-1.7B-bf16` |
 
-These engines do not call hosted ASR APIs. The app downloads the selected model into the same model storage used by WhisperKit/MLX, prepares the Qwen Python runtime in an app-managed virtual environment, downloads MiMo runtime files when needed, finds an available Python 3 executable, then invokes the bundled local runner script.
+Qwen3-ASR does not call a hosted ASR API. The app downloads the selected model into the same model storage used by WhisperKit and runs inference locally through native Swift and MLX.
 
 ## Project Structure
 

Sources/App/VoicePipeline+Models.swift

@@ -25,7 +25,6 @@ extension VoicePipeline {
 
     func unloadLocalASR() {
         qwenSpeechEngine = nil
-        mimoSpeechEngine = nil
     }
 
     func loadLLM() {
@@ -99,32 +98,14 @@ extension VoicePipeline {
                 markSpeechModelDownloadRequired(showInStatus: requestPermission)
                 return
             }
-            let engine = LocalASREngine(configuration: LocalASRConfiguration(
-                provider: .qwen3,
-                pythonPath: settings.localASRPythonPath,
-                modelPath: catalog.asrModelPath(for: settings.qwenASRModel),
-                tokenizerPath: "",
-                repoPath: ""
-            ))
+            let modelPath = catalog.asrModelPath(for: settings.qwenASRModel)
+            if qwenSpeechEngine?.usesModel(at: modelPath) == true { return }
+            let engine = QwenNativeASREngine(modelPath: modelPath)
             qwenSpeechEngine = engine
             Task { await engine.prepare() }
         case .mimo:
-            let settings = appState.settings
-            let catalog = ModelCatalog.shared
-            guard localASRIsAvailable(settings.mimoASRModel) else {
-                mimoSpeechEngine = nil
-                markSpeechModelDownloadRequired(showInStatus: requestPermission)
-                return
-            }
-            let engine = LocalASREngine(configuration: LocalASRConfiguration(
-                provider: .mimo,
-                pythonPath: settings.localASRPythonPath,
-                modelPath: catalog.asrModelPath(for: settings.mimoASRModel),
-                tokenizerPath: catalog.mimoTokenizerPath(),
-                repoPath: catalog.mimoRepositoryPath()
-            ))
-            mimoSpeechEngine = engine
-            Task { await engine.prepare() }
+            appState.settings.speechEngine = .apple
+            await ensureEngineLoaded(requestPermission: requestPermission)
         }
     }
 

Sources/App/VoicePipeline+Status.swift

@@ -87,9 +87,7 @@ extension VoicePipeline {
             return error.localizedDescription
         case let error as AppleSpeechError:
             return error.localizedDescription
-        case let error as LocalASRError:
-            return error.localizedDescription
-        case let error as LocalASRRuntimeError:
+        case let error as QwenNativeASRError:
             return error.localizedDescription
         case is URLError:
             return L("error.network_request_failed")

Sources/App/VoicePipeline.swift

@@ -13,8 +13,7 @@ final class VoicePipeline {
     var whisperEngine: WhisperEngine?
     var appleSpeechEngine: AppleSpeechEngine?
     var volcSpeechEngine: VolcSpeechEngine?
-    var qwenSpeechEngine: LocalASREngine?
-    var mimoSpeechEngine: LocalASREngine?
+    var qwenSpeechEngine: QwenNativeASREngine?
     var screenOCRTask: Task<ScreenContextSnapshot, Never>?
     var screenOCRStartedAt: CFAbsoluteTime?
     var processingTask: Task<Void, Never>?
@@ -28,7 +27,7 @@ final class VoicePipeline {
         case .apple: return appleSpeechEngine
         case .volc: return volcSpeechEngine
         case .qwen3: return qwenSpeechEngine
-        case .mimo: return mimoSpeechEngine
+        case .mimo: return nil
         }
     }
 

Sources/Config/AppSettings.swift

@@ -35,6 +35,10 @@ enum SpeechEngineType: String, Codable, CaseIterable {
     case qwen3 = "qwen3"
     case mimo = "mimo"
 
+    static var selectableCases: [SpeechEngineType] {
+        allCases.filter { $0 != .mimo }
+    }
+
     var label: String {
         switch self {
         case .whisper: return "WhisperKit"
@@ -288,10 +292,7 @@ final class AppSettings: ObservableObject {
     @Published var volcAppKey: String
     @Published var volcAccessKey: String
     @Published var volcResourceId: String
-    @Published var localASRPythonPath: String
     @Published var qwenASRModel: String
-    @Published var mimoASRRepoPath: String
-    @Published var mimoASRModel: String
     @Published var preloadSpeechModelOnLaunch: Bool
     @Published var preloadFormattingModelOnLaunch: Bool
     @Published var modelStoragePath: String
@@ -315,9 +316,7 @@ final class AppSettings: ObservableObject {
         case useRemoteLLM, remoteProvider, remoteAPIKey, remoteBaseURL, remoteModel
         case menuBarIcon, appIconAppearance
         case volcAppKey, volcAccessKey, volcResourceId
-        case localASRPythonPath
         case qwenASRModel, qwenASRModelPath
-        case mimoASRRepoPath, mimoASRModel, mimoASRModelPath, mimoASRTokenizerPath
         case preloadSpeechModelOnLaunch, preloadFormattingModelOnLaunch
         case modelStoragePath, localWhisperModelPaths, localLLMModelPaths
         case developerInterfaceEnabled, developerHTTPPort, developerHTTPToken
@@ -342,9 +341,20 @@ final class AppSettings: ObservableObject {
             ?? .longPress
         tapInterval = ud.double(forKey: Key.tapInterval.rawValue).nonZero ?? 0.4
         let savedEngine = ud.string(forKey: Key.speechEngine.rawValue) ?? ""
-        speechEngine = SpeechEngineType(rawValue: savedEngine)
+        let loadedSpeechEngine = SpeechEngineType(rawValue: savedEngine)
             ?? (savedEngine.contains("Whisper") || savedEngine.contains("whisper") ? .whisper : nil)
             ?? .apple
+        speechEngine = loadedSpeechEngine == .mimo ? .apple : loadedSpeechEngine
+        if loadedSpeechEngine == .mimo {
+            ud.set(SpeechEngineType.apple.rawValue, forKey: Key.speechEngine.rawValue)
+        }
+        [
+            "localASRPythonPath",
+            "mimoASRRepoPath",
+            "mimoASRModel",
+            "mimoASRModelPath",
+            "mimoASRTokenizerPath",
+        ].forEach(ud.removeObject(forKey:))
         whisperModel = ud.string(forKey: Key.whisperModel.rawValue) ?? "large-v3"
         llmModel = ud.string(forKey: Key.llmModel.rawValue) ?? Self.defaultLLMModelID
         microphoneID = ud.string(forKey: Key.microphoneID.rawValue)
@@ -387,14 +397,9 @@ final class AppSettings: ObservableObject {
         volcAppKey = ud.string(forKey: Key.volcAppKey.rawValue) ?? ""
         volcAccessKey = ud.string(forKey: Key.volcAccessKey.rawValue) ?? ""
         volcResourceId = ud.string(forKey: Key.volcResourceId.rawValue) ?? VolcASRModel.recommended.rawValue
-        localASRPythonPath = ud.string(forKey: Key.localASRPythonPath.rawValue) ?? LocalASRConfiguration.defaultPythonPath
         qwenASRModel = ud.string(forKey: Key.qwenASRModel.rawValue)
             ?? ud.string(forKey: Key.qwenASRModelPath.rawValue)
-            ?? LocalASRConfiguration.qwen3DefaultModel
-        mimoASRRepoPath = ud.string(forKey: Key.mimoASRRepoPath.rawValue) ?? ""
-        mimoASRModel = ud.string(forKey: Key.mimoASRModel.rawValue)
-            ?? ud.string(forKey: Key.mimoASRModelPath.rawValue)
-            ?? LocalASRConfiguration.mimoDefaultModel
+            ?? QwenASRModel.defaultID
         preloadSpeechModelOnLaunch = ud.object(forKey: Key.preloadSpeechModelOnLaunch.rawValue) as? Bool ?? true
         preloadFormattingModelOnLaunch = ud.object(forKey: Key.preloadFormattingModelOnLaunch.rawValue) as? Bool ?? true
         modelStoragePath = ud.string(forKey: Key.modelStoragePath.rawValue) ?? ModelStorage.defaultRoot.path
@@ -456,10 +461,7 @@ final class AppSettings: ObservableObject {
         $volcAppKey.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.volcAppKey.rawValue) }.store(in: &cancellables)
         $volcAccessKey.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.volcAccessKey.rawValue) }.store(in: &cancellables)
         $volcResourceId.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.volcResourceId.rawValue) }.store(in: &cancellables)
-        $localASRPythonPath.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.localASRPythonPath.rawValue) }.store(in: &cancellables)
         $qwenASRModel.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.qwenASRModel.rawValue) }.store(in: &cancellables)
-        $mimoASRRepoPath.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.mimoASRRepoPath.rawValue) }.store(in: &cancellables)
-        $mimoASRModel.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.mimoASRModel.rawValue) }.store(in: &cancellables)
         $preloadSpeechModelOnLaunch.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.preloadSpeechModelOnLaunch.rawValue) }.store(in: &cancellables)
         $preloadFormattingModelOnLaunch.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.preloadFormattingModelOnLaunch.rawValue) }.store(in: &cancellables)
         $modelStoragePath.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.modelStoragePath.rawValue) }.store(in: &cancellables)

Sources/Config/ModelCatalog.swift

@@ -129,10 +129,7 @@ final class ModelCatalog: ObservableObject {
             ModelEntry(id: $0.id, displayName: $0.displayName, hint: $0.hint, family: nil)
         }
         if !asrModels.contains(where: { $0.id == settings.qwenASRModel }) {
-            settings.qwenASRModel = LocalASRConfiguration.qwen3DefaultModel
-        }
-        if !asrModels.contains(where: { $0.id == settings.mimoASRModel }) {
-            settings.mimoASRModel = LocalASRConfiguration.mimoDefaultModel
+            settings.qwenASRModel = QwenASRModel.defaultID
         }
         refreshStatus()
     }