diff --git a/electron/stt/extractAudio.integration.test.ts b/electron/stt/extractAudio.integration.test.ts new file mode 100644 index 000000000..2301f6fd3 --- /dev/null +++ b/electron/stt/extractAudio.integration.test.ts @@ -0,0 +1,88 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +// Only the binary lookup is replaced: extraction still spawns and reads real FFmpeg. +vi.mock("../media/audioPeaks", () => ({ resolveFfmpeg: () => "ffmpeg" })); + +import { extractMono16kPcm } from "./extractAudio"; + +const ffmpegAvailable = spawnSync("ffmpeg", ["-version"], { timeout: 5_000 }).status === 0; + +if (!ffmpegAvailable) { + console.warn("Skipping audio-origin integration tests: FFmpeg is unavailable on PATH."); +} + +describe.skipIf(!ffmpegAvailable)("extractMono16kPcm with real FFmpeg", () => { + let fixtureDir: string; + let delayedVideo: string; + let standaloneAudio: string; + + function makeFixture(filename: string, args: string[]): string { + const filePath = path.join(fixtureDir, filename); + const result = spawnSync( + "ffmpeg", + ["-hide_banner", "-loglevel", "error", "-y", ...args, filePath], + { encoding: "utf8", timeout: 10_000 }, + ); + if (result.status !== 0) { + throw new Error(result.error?.message ?? result.stderr); + } + return filePath; + } + + beforeAll(() => { + fixtureDir = mkdtempSync(path.join(tmpdir(), "openscreen-stt-audio-origin-")); + const tone = [ + "-itsoffset", + "2", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=16000:duration=2", + ]; + delayedVideo = makeFixture("delayed.mkv", [ + "-f", + "lavfi", + "-i", + "color=c=black:s=16x16:r=1:d=4", + ...tone, + "-map", + "0:v", + "-map", + "1:a", + "-c:v", + "ffv1", + "-c:a", + "pcm_s16le", + ]); + standaloneAudio = makeFixture("standalone.mka", [...tone, "-c:a", "pcm_s16le"]); + }); + + afterAll(() => { + if (fixtureDir) rmSync(fixtureDir, { recursive: true, force: true }); + }); + + it("keeps delayed audio on the video's clock", async () => { + // The audio stream starts two seconds after video, without encoded silence. + // Dropping that timestamp gap makes every word and text cut two seconds early. + const samples = await extractMono16kPcm(delayedVideo); + expect(samples).toHaveLength(4 * 16_000); + expect(samples.subarray(0, 2 * 16_000).every((sample) => sample === 0)).toBe(true); + const firstAudibleSample = samples.findIndex((sample) => Math.abs(sample) > 0.01); + expect(firstAudibleSample).toBeGreaterThanOrEqual(2 * 16_000); + expect(firstAudibleSample).toBeLessThan(2 * 16_000 + 16); + }); + + it("keeps standalone audio relative to its own media origin", async () => { + // This container starts at two seconds too, but has no earlier video stream. + // Its own origin is still zero: padding it would invent two seconds of silence. + const samples = await extractMono16kPcm(standaloneAudio); + expect(samples).toHaveLength(2 * 16_000); + const firstAudibleSample = samples.findIndex((sample) => Math.abs(sample) > 0.01); + expect(firstAudibleSample).toBeGreaterThanOrEqual(0); + expect(firstAudibleSample).toBeLessThan(16); + }); +}); diff --git a/electron/stt/extractAudio.ts b/electron/stt/extractAudio.ts index 28dd9f177..901078669 100644 --- a/electron/stt/extractAudio.ts +++ b/electron/stt/extractAudio.ts @@ -96,6 +96,9 @@ export async function extractMono16kPcm( "1", "-ar", String(SAMPLE_RATE), + // Pad delayed audio to keep sample zero on the media clock. + "-af", + "aresample=first_pts=0", "-f", "f32le", "-",