Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 58 additions & 3 deletions electron/stt/extractAudio.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,11 @@ const resolveFfmpegMock = vi.fn<() => string | null>();
vi.mock("node:child_process", () => ({ spawn: (...args: unknown[]) => spawnMock(...args) }));
vi.mock("../media/audioPeaks", () => ({ resolveFfmpeg: () => resolveFfmpegMock() }));

const { extractMono16kPcm, FfmpegUnavailableError, NoAudioTrackError } = await import(
"./extractAudio"
const { extractMono16kPcm, FfmpegUnavailableError, MediaUnreadableError, NoAudioTrackError } =
await import("./extractAudio");
const { STT_MEDIA_UNREADABLE, STT_NATIVE_EXTRACTION_UNAVAILABLE } = await import(
"./transcriptionContract"
);
const { STT_NATIVE_EXTRACTION_UNAVAILABLE } = await import("./transcriptionContract");

/** A stand-in for the ffmpeg child: two pipes and a close event, nothing more. */
function fakeChild() {
Expand Down Expand Up @@ -95,6 +96,60 @@ describe("extractMono16kPcm", () => {
await expect(promise).rejects.toBeInstanceOf(NoAudioTrackError);
});

// Issue #968: the renderer explains these as "check folder access" and retries
// them, so both must carry the marker, which is all that survives the IPC.
it("reports a read the OS refused as unreadable media, not as no audio", async () => {
const child = fakeChild();
spawnMock.mockReturnValue(child);
const promise = extractMono16kPcm("/Users/me/Downloads/a.mp4");
child.stderr.end("/Users/me/Downloads/a.mp4: Operation not permitted");
child.stdout.end();
child.emit("close", 1);

await expect(promise).rejects.toBeInstanceOf(MediaUnreadableError);
await expect(promise).rejects.toThrow(STT_MEDIA_UNREADABLE);
});

it("reports an ffmpeg that hangs opening the file as unreadable media", async () => {
vi.useFakeTimers();
try {
const child = fakeChild();
spawnMock.mockReturnValue(child);
const promise = extractMono16kPcm("/Users/me/Downloads/a.mp4");
const settled = expect(promise).rejects.toThrow(STT_MEDIA_UNREADABLE);
await vi.advanceTimersByTimeAsync(60_000);
await settled;
expect(child.kill).toHaveBeenCalled();
} finally {
vi.useRealTimers();
}
});

it("reports a timeout AFTER audio came out as a plain stall, not unreadable media", async () => {
// The file was readable; pointing the user at folder access would be wrong.
vi.useFakeTimers();
try {
const child = fakeChild();
spawnMock.mockReturnValue(child);
const promise = extractMono16kPcm("/Users/me/Downloads/a.mp4");
const settled = promise.then(
() => {
throw new Error("expected a rejection");
},
(error: Error) => error,
);
child.stdout.write(f32le([0.5]));
await vi.advanceTimersByTimeAsync(60_000);
const error = await settled;
expect(error).not.toBeInstanceOf(MediaUnreadableError);
expect(error.message).not.toContain(STT_MEDIA_UNREADABLE);
expect(error.message).toBe("ffmpeg timed out after 60000ms on /Users/me/Downloads/a.mp4");
expect(child.kill).toHaveBeenCalledWith("SIGKILL");
} finally {
vi.useRealTimers();
}
});

it("keeps the samples when ffmpeg exits non-zero AFTER writing audio", async () => {
// A truncated file still yields usable audio; throwing it away would lose a
// transcript over a trailing byte.
Expand Down
36 changes: 33 additions & 3 deletions electron/stt/extractAudio.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

import { spawn } from "node:child_process";
import { resolveFfmpeg } from "../media/audioPeaks";
import { STT_NATIVE_EXTRACTION_UNAVAILABLE } from "./transcriptionContract";
import { STT_MEDIA_UNREADABLE, STT_NATIVE_EXTRACTION_UNAVAILABLE } from "./transcriptionContract";

/** What whisper.cpp wants, and what `decodePeaks` already asks ffmpeg for. */
const SAMPLE_RATE = 16_000;
Expand Down Expand Up @@ -50,6 +50,22 @@ export class NoAudioTrackError extends Error {
}
}

/**
* The media file could not be read at all: the OS refused it, or ffmpeg hung
* opening it. Unlike `NoAudioTrackError` this is NOT permanent: it clears once
* the user grants access, so it carries `STT_MEDIA_UNREADABLE` for the renderer
* to explain rather than remember.
*/
export class MediaUnreadableError extends Error {
constructor(filePath: string, detail: string) {
super(`${STT_MEDIA_UNREADABLE}: cannot read ${filePath}: ${detail}`);
this.name = "MediaUnreadableError";
}
}

/** What ffmpeg's stderr says when the OS refused to open the input. */
const ACCESS_DENIED = /operation not permitted|permission denied/i;

/**
* Decode `filePath` to mono 16 kHz float samples.
*
Expand Down Expand Up @@ -105,8 +121,16 @@ export async function extractMono16kPcm(

const timer = setTimeout(() => {
child.kill("SIGKILL");
// Only a hang before the first byte says the file could not be opened (a
// pending macOS folder-access prompt produces exactly that, issue #968). Once
// audio has come out the media was readable, and the stall is ffmpeg's.
const nothingRead = total === 0 && carry === null;
finish(() =>
reject(new Error(`ffmpeg timed out after ${EXTRACT_TIMEOUT_MS}ms on ${filePath}`)),
reject(
nothingRead
? new MediaUnreadableError(filePath, `ffmpeg timed out after ${EXTRACT_TIMEOUT_MS}ms`)
: new Error(`ffmpeg timed out after ${EXTRACT_TIMEOUT_MS}ms on ${filePath}`),
),
);
}, EXTRACT_TIMEOUT_MS);

Comment thread
EtienneLescot marked this conversation as resolved.
Expand Down Expand Up @@ -137,7 +161,13 @@ export async function extractMono16kPcm(
// caller treats both the same way — this asset will not transcribe — so they
// share an error type; `stderr` carries which it was.
if (code !== 0 && total === 0) {
finish(() => reject(new NoAudioTrackError(filePath, stderr.trim())));
// Except a read the OS refused: that also exits non-zero with nothing
// decoded, but remembering it as "no audio" would never retry the file
// once access is granted.
const error = ACCESS_DENIED.test(stderr)
? new MediaUnreadableError(filePath, stderr.trim())
: new NoAudioTrackError(filePath, stderr.trim());
finish(() => reject(error));
return;
}
const out = new Float32Array(total);
Expand Down
11 changes: 11 additions & 0 deletions electron/stt/transcriptionContract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,17 @@ export interface SttTranscribeRequest {
*/
export const STT_NATIVE_EXTRACTION_UNAVAILABLE = "stt:native-extraction-unavailable";

/**
* Marker carried in the error message when audio extraction could not read the
* `sourcePath` itself: the OS refused the read, or ffmpeg hung opening the file
* (macOS holds the read behind a pending folder-access prompt, issue #968).
*
* Only `extractAudio.ts` sets it, so the renderer can tell "this file's folder is
* not readable" from an engine failure that happens to mention a permission (model
* loading, the whisper helper). Same IPC reason as above for being a string.
*/
export const STT_MEDIA_UNREADABLE = "stt:media-unreadable";

/** IPC response: main → renderer. */
export interface SttTranscribeResponse {
segments: SttPhraseSegment[];
Expand Down
8 changes: 5 additions & 3 deletions src/components/ai-edition/v4/MediaStage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import {
type AssetTranscriptionView,
isSilentFailure,
transcriptHasSpeech,
transcriptionFailureHintKey,
} from "@/lib/ai-edition/transcription/status";
import { formatBytes } from "@/utils/formatBytes";
import {
Expand Down Expand Up @@ -105,6 +106,9 @@ export function MediaStage({
: { assetId: "", status: "idle" };
const selectedBusy =
selectedTranscription.status === "running" || selectedTranscription.status === "queued";
const failureHintKey = selectedTranscription.failure
? transcriptionFailureHintKey(selectedTranscription.failure)
: null;

const handleImport = async () => {
if (!projectId) {
Expand Down Expand Up @@ -380,9 +384,7 @@ export function MediaStage({
color: "var(--muted)",
}}
>
{selectedTranscription.failure.kind === "error"
? selectedTranscription.failure.message
: t("mediaStage.noAudioTrackHint")}
{failureHintKey ? t(failureHintKey) : selectedTranscription.failure.message}
</p>
) : null}

Expand Down
1 change: 1 addition & 0 deletions src/i18n/locales/ar/editor.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"detectedLanguage": "اللغة المكتشفة: {{language}}",
"noAudioTrack": "لا يوجد مسار صوتي",
"noAudioTrackHint": "لا يحتوي هذا الملف على مسار صوتي — لا يوجد ما يمكن نسخه.",
"mediaUnreadableHint": "تعذّرت قراءة صوت هذا الملف. تأكد من أن OpenScreen يمكنه الوصول إلى المجلد الذي يوجد فيه، ثم أعد الإنشاء.",
"noSpeechDetected": "لم يتم اكتشاف كلام"
},
"exportDialog": {
Expand Down
1 change: 1 addition & 0 deletions src/i18n/locales/cs/editor.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"detectedLanguage": "Zjištěný jazyk: {{language}}",
"noAudioTrack": "Žádná zvuková stopa",
"noAudioTrackHint": "Toto médium nemá zvukovou stopu, takže není co přepisovat.",
"mediaUnreadableHint": "Zvuk tohoto média se nepodařilo přečíst. Ověřte, že OpenScreen má přístup ke složce, ve které je uloženo, a pak jej vygenerujte znovu.",
"noSpeechDetected": "Žádná řeč nezjištěna"
},
"exportDialog": {
Expand Down
1 change: 1 addition & 0 deletions src/i18n/locales/de/editor.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"detectedLanguage": "Erkannte Sprache: {{language}}",
"noAudioTrack": "Keine Audiospur",
"noAudioTrackHint": "Dieses Medium hat keine Audiospur — es gibt nichts zu transkribieren.",
"mediaUnreadableHint": "Der Ton dieses Mediums konnte nicht gelesen werden. Stelle sicher, dass OpenScreen auf den Ordner zugreifen darf, in dem es liegt, und generiere dann neu.",
"noSpeechDetected": "Keine Sprache erkannt"
},
"exportDialog": {
Expand Down
1 change: 1 addition & 0 deletions src/i18n/locales/en/editor.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"detectedLanguage": "Detected language: {{language}}",
"noAudioTrack": "No audio track",
"noAudioTrackHint": "This media has no audio track — there is nothing to transcribe.",
"mediaUnreadableHint": "Couldn't read this media's audio. Make sure OpenScreen can access the folder it's in, then regenerate.",
"noSpeechDetected": "No speech detected"
},
"exportDialog": {
Expand Down
1 change: 1 addition & 0 deletions src/i18n/locales/es/editor.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"detectedLanguage": "Idioma detectado: {{language}}",
"noAudioTrack": "Sin pista de audio",
"noAudioTrackHint": "Este medio no tiene pista de audio: no hay nada que transcribir.",
"mediaUnreadableHint": "No se pudo leer el audio de este medio. Asegúrate de que OpenScreen puede acceder a la carpeta que lo contiene y vuelve a generar.",
"noSpeechDetected": "No se detectó voz"
},
"exportDialog": {
Expand Down
1 change: 1 addition & 0 deletions src/i18n/locales/fr/editor.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"detectedLanguage": "Langue détectée : {{language}}",
"noAudioTrack": "Aucune piste audio",
"noAudioTrackHint": "Ce média n'a pas de piste audio — il n'y a rien à transcrire.",
"mediaUnreadableHint": "Impossible de lire l'audio de ce média. Vérifiez qu'OpenScreen a accès au dossier qui le contient, puis régénérez.",
"noSpeechDetected": "Aucune parole détectée"
},
"exportDialog": {
Expand Down
1 change: 1 addition & 0 deletions src/i18n/locales/it/editor.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"detectedLanguage": "Lingua rilevata: {{language}}",
"noAudioTrack": "Nessuna traccia audio",
"noAudioTrackHint": "Questo contenuto non ha una traccia audio: non c'è nulla da trascrivere.",
"mediaUnreadableHint": "Impossibile leggere l'audio di questo contenuto. Verifica che OpenScreen possa accedere alla cartella che lo contiene, poi rigenera.",
"noSpeechDetected": "Nessun parlato rilevato"
},
"exportDialog": {
Expand Down
1 change: 1 addition & 0 deletions src/i18n/locales/ja-JP/editor.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"detectedLanguage": "検出された言語: {{language}}",
"noAudioTrack": "音声トラックがありません",
"noAudioTrackHint": "このメディアには音声トラックがないため、文字起こしできません。",
"mediaUnreadableHint": "このメディアの音声を読み取れませんでした。OpenScreen が保存先のフォルダーにアクセスできることを確認してから、再生成してください。",
"noSpeechDetected": "音声が検出されませんでした"
},
"exportDialog": {
Expand Down
1 change: 1 addition & 0 deletions src/i18n/locales/ko-KR/editor.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"detectedLanguage": "감지된 언어: {{language}}",
"noAudioTrack": "오디오 트랙 없음",
"noAudioTrackHint": "이 미디어에는 오디오 트랙이 없어 받아쓸 내용이 없습니다.",
"mediaUnreadableHint": "이 미디어의 오디오를 읽을 수 없습니다. OpenScreen이 파일이 있는 폴더에 접근할 수 있는지 확인한 후 다시 생성하세요.",
"noSpeechDetected": "음성이 감지되지 않음"
},
"exportDialog": {
Expand Down
1 change: 1 addition & 0 deletions src/i18n/locales/pt-BR/editor.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"detectedLanguage": "Idioma detectado: {{language}}",
"noAudioTrack": "Sem faixa de áudio",
"noAudioTrackHint": "Esta mídia não tem faixa de áudio — não há nada para transcrever.",
"mediaUnreadableHint": "Não foi possível ler o áudio desta mídia. Verifique se o OpenScreen tem acesso à pasta onde ela está e gere novamente.",
"noSpeechDetected": "Nenhuma fala detectada"
},
"exportDialog": {
Expand Down
1 change: 1 addition & 0 deletions src/i18n/locales/ru/editor.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"detectedLanguage": "Обнаруженный язык: {{language}}",
"noAudioTrack": "Нет аудиодорожки",
"noAudioTrackHint": "В этом медиафайле нет аудиодорожки — расшифровывать нечего.",
"mediaUnreadableHint": "Не удалось прочитать звук этого медиафайла. Убедитесь, что у OpenScreen есть доступ к папке, в которой он находится, и создайте расшифровку заново.",
"noSpeechDetected": "Речь не обнаружена"
},
"exportDialog": {
Expand Down
1 change: 1 addition & 0 deletions src/i18n/locales/tr/editor.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"detectedLanguage": "Algılanan dil: {{language}}",
"noAudioTrack": "Ses parçası yok",
"noAudioTrackHint": "Bu medyada ses parçası yok — metne dökülecek bir şey bulunmuyor.",
"mediaUnreadableHint": "Bu medyanın sesi okunamadı. OpenScreen'in bulunduğu klasöre erişebildiğinden emin olun, ardından yeniden oluşturun.",
"noSpeechDetected": "Konuşma algılanmadı"
},
"exportDialog": {
Expand Down
1 change: 1 addition & 0 deletions src/i18n/locales/vi/editor.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"detectedLanguage": "Ngôn ngữ phát hiện: {{language}}",
"noAudioTrack": "Không có bản âm thanh",
"noAudioTrackHint": "Media này không có bản âm thanh — không có gì để phiên âm.",
"mediaUnreadableHint": "Không thể đọc âm thanh của media này. Hãy đảm bảo OpenScreen có quyền truy cập thư mục chứa nó, rồi tạo lại.",
"noSpeechDetected": "Không phát hiện giọng nói"
},
"exportDialog": {
Expand Down
1 change: 1 addition & 0 deletions src/i18n/locales/zh-CN/editor.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"detectedLanguage": "检测到的语言:{{language}}",
"noAudioTrack": "无音频轨道",
"noAudioTrackHint": "此媒体没有音频轨道,没有可转录的内容。",
"mediaUnreadableHint": "无法读取此媒体的音频。请确认 OpenScreen 可以访问它所在的文件夹,然后重新生成。",
"noSpeechDetected": "未检测到语音"
},
"exportDialog": {
Expand Down
1 change: 1 addition & 0 deletions src/i18n/locales/zh-TW/editor.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"detectedLanguage": "偵測到的語言:{{language}}",
"noAudioTrack": "無音訊軌道",
"noAudioTrackHint": "此媒體沒有音訊軌道,沒有可轉錄的內容。",
"mediaUnreadableHint": "無法讀取此媒體的音訊。請確認 OpenScreen 可以存取它所在的資料夾,然後重新產生。",
"noSpeechDetected": "未偵測到語音"
},
"exportDialog": {
Expand Down
18 changes: 15 additions & 3 deletions src/lib/ai-edition/store/transcriptionStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import {
type TranscriptionFailure,
type TranscriptionPhase,
type TranscriptionProgress,
transcriptionFailureHintKey,
transcriptRelevantAssetIds,
} from "../transcription/status";
import { useProjectStore } from "./projectStore";
Expand Down Expand Up @@ -451,6 +452,9 @@ async function runJob(assetId: string, job: TranscriptionJob): Promise<void> {
}
if (!isCurrentRun(assetId, runId)) return; // superseded by a newer request
const failure = classifyTranscriptionError(error);
// The UI shows a translated hint or a trimmed message; the full detail
// (channel, path, ffmpeg stderr) belongs here.
console.warn(`[transcription] asset ${assetId} failed:`, error);
patchJob(assetId, runId, { status: "failed", phase: undefined, progress: undefined, failure });
flushSettleWaiters(assetId);
await persistPermanentFailure(projectId, assetId, failure);
Expand All @@ -459,8 +463,13 @@ async function runJob(assetId: string, job: TranscriptionJob): Promise<void> {
// queue into the same wall would spend a full retry budget per asset and
// stack one identical toast per asset. Fail them with the same verdict
// instead — the gate then reads "failed" (not "queued forever"), and one
// manual retry re-runs them all once the engine is back.
if (failure.kind === "error") failRemainingQueue(projectId, failure);
// manual retry re-runs them all once the engine is back. A file the OS would
// not let ffmpeg read is the exception: that verdict is about this file's
// folder, and stamping it on media elsewhere would send the user looking at
// the wrong permission.
if (failure.kind === "error" && !failure.unreadableMedia) {
failRemainingQueue(projectId, failure);
}
// A silent recording is an expected outcome, not an incident: the media card
// and every gated button already say so, so the background pass stays quiet.
// A run the user asked for by hand still gets an answer — but an
Expand All @@ -469,7 +478,10 @@ async function runJob(assetId: string, job: TranscriptionJob): Promise<void> {
if (isPermanentFailure(failure.kind)) {
if (job.manual) toast.info(toastText("mediaStage.noAudioTrackHint"));
} else {
toast.error(toastText("mediaStage.transcriptionFailed"), { description: failure.message });
const hintKey = transcriptionFailureHintKey(failure);
toast.error(toastText("mediaStage.transcriptionFailed"), {
description: hintKey ? toastText(hintKey) : failure.message,
});
}
} finally {
if (activeRun?.controller === controller) activeRun = null;
Expand Down
Loading
Loading