Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 76 additions & 0 deletions apps/web/__tests__/unit/transcribe-language.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {
AI_GENERATION_LANGUAGES,
isAiGenerationLanguage,
parseAiGenerationLanguage,
sanitizeTranscriptionVocabulary,
} from "@cap/web-domain";
import { describe, expect, it } from "vitest";
import { getAssemblyAITranscriptionOptions } from "@/lib/assemblyai";
Expand Down Expand Up @@ -47,4 +48,79 @@ describe("AI generation language support", () => {
language_code: "zh",
});
});

it("omits keyterms_prompt without vocabulary", () => {
expect(getAssemblyAITranscriptionOptions("auto")).not.toHaveProperty(
"keyterms_prompt",
);
expect(getAssemblyAITranscriptionOptions("en", [])).not.toHaveProperty(
"keyterms_prompt",
);
expect(
getAssemblyAITranscriptionOptions("en", [" ", ""]),
).not.toHaveProperty("keyterms_prompt");
});

it("passes sanitized vocabulary as keyterms_prompt", () => {
const options = getAssemblyAITranscriptionOptions("en", [
" Rundoo ",
"rundoo",
"Dooey",
"",
]);
expect(options).toMatchObject({
keyterms_prompt: ["Rundoo", "Dooey"],
speech_models: ["universal-3-5-pro", "universal-2"],
language_code: "en",
});
expect(getAssemblyAITranscriptionOptions("auto", ["Dooey"])).toMatchObject({
keyterms_prompt: ["Dooey"],
language_detection: true,
});
});
});

describe("sanitizeTranscriptionVocabulary", () => {
it("splits strings on commas and newlines", () => {
expect(
sanitizeTranscriptionVocabulary("Rundoo, Dooey\nPOS Terminal"),
).toEqual(["Rundoo", "Dooey", "POS Terminal"]);
});

it("trims, drops empties, and dedupes case-insensitively", () => {
expect(
sanitizeTranscriptionVocabulary([
" Rundoo",
"RUNDOO ",
"",
" ",
"Dooey",
]),
).toEqual(["Rundoo", "Dooey"]);
});

it("drops over-long terms and non-strings", () => {
expect(
sanitizeTranscriptionVocabulary([
"x".repeat(51),
"x".repeat(50),
"one two three four five six seven",
"one two three four five six",
42,
null,
]),
).toEqual(["x".repeat(50), "one two three four five six"]);
});

it("caps the term count", () => {
const terms = Array.from({ length: 150 }, (_, i) => `term${i}`);
const result = sanitizeTranscriptionVocabulary(terms);
expect(result).toHaveLength(100);
expect(result[99]).toBe("term99");
});

it("returns an empty list for unsupported input", () => {
expect(sanitizeTranscriptionVocabulary(undefined)).toEqual([]);
expect(sanitizeTranscriptionVocabulary({})).toEqual([]);
});
});
20 changes: 16 additions & 4 deletions apps/web/actions/organization/settings.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
AI_GENERATION_LANGUAGE_AUTO,
type AiGenerationLanguage,
isAiGenerationLanguage,
sanitizeTranscriptionVocabulary,
type Organisation,
} from "@cap/web-domain";
import { eq } from "drizzle-orm";
Expand All @@ -27,6 +28,7 @@ type OrganizationSettingsInput = {
hideShareableLinkCapLogo?: boolean;
shareableLinkUseOrganizationIcon?: boolean;
aiGenerationLanguage?: AiGenerationLanguage;
transcriptionVocabulary?: string[];
defaultPlaybackSpeed?: number;
};

Expand All @@ -37,6 +39,7 @@ const proOrganizationSettingKeys = [
"hideShareableLinkCapLogo",
"shareableLinkUseOrganizationIcon",
"aiGenerationLanguage",
"transcriptionVocabulary",
] as const satisfies readonly (keyof OrganizationSettingsInput)[];

const defaultProOrganizationSettings = {
Expand All @@ -46,6 +49,7 @@ const defaultProOrganizationSettings = {
hideShareableLinkCapLogo: false,
shareableLinkUseOrganizationIcon: false,
aiGenerationLanguage: AI_GENERATION_LANGUAGE_AUTO,
transcriptionVocabulary: [],
} as const satisfies Pick<
Required<OrganizationSettingsInput>,
(typeof proOrganizationSettingKeys)[number]
Expand Down Expand Up @@ -103,15 +107,23 @@ export async function updateOrganizationSettings(

await requireOrganizationSettingsManager(user.id, organizationId);

const sanitizedSettings =
settings.defaultPlaybackSpeed !== undefined
const sanitizedSettings: OrganizationSettingsInput = {
...settings,
...(settings.defaultPlaybackSpeed !== undefined
? {
...settings,
defaultPlaybackSpeed: normalizePlaybackSpeed(
settings.defaultPlaybackSpeed,
),
}
: settings;
: {}),
...(settings.transcriptionVocabulary !== undefined
? {
transcriptionVocabulary: sanitizeTranscriptionVocabulary(
settings.transcriptionVocabulary,
),
}
: {}),
};

const nextSettings = userIsPro(user)
? sanitizedSettings
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
getAiGenerationLanguageName,
isAiGenerationLanguage,
type Organisation,
sanitizeTranscriptionVocabulary,
} from "@cap/web-domain";
import { useDebounce } from "@uidotdev/usehooks";
import clsx from "clsx";
Expand Down Expand Up @@ -36,12 +37,13 @@ const defaultSettings: OrganizationSettings = {
hideShareableLinkCapLogo: false,
shareableLinkUseOrganizationIcon: false,
aiGenerationLanguage: AI_GENERATION_LANGUAGE_AUTO,
transcriptionVocabulary: [],
defaultPlaybackSpeed: DEFAULT_PLAYBACK_SPEED,
};

type BooleanOrganizationSettingKey = Exclude<
keyof OrganizationSettings,
"aiGenerationLanguage"
"aiGenerationLanguage" | "transcriptionVocabulary"
>;

const options: Array<{
Expand Down Expand Up @@ -109,8 +111,13 @@ const mergeSettings = (
aiGenerationLanguage: isAiGenerationLanguage(settings?.aiGenerationLanguage)
? settings.aiGenerationLanguage
: AI_GENERATION_LANGUAGE_AUTO,
transcriptionVocabulary: sanitizeTranscriptionVocabulary(
settings?.transcriptionVocabulary,
),
Comment on lines +114 to +116

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Settings save without edits. mergeSettings now creates a fresh transcriptionVocabulary array every time. On mount, the reset effect replaces lastSavedSettings.current, but the autosave effect still holds the earlier debounced settings. Its per-key !== check treats the two identical lists as different and sends a save without any user edit. The save refreshes the settings, which repeats this mismatch and can keep sending writes and success notices. Compare vocabulary contents, and skip stale debounced values when resetting from server settings.

Knowledge Base Used: Web product and collaborative workspace

Prompt To Fix With AI
This is a comment left during a code review.
Path: apps/web/app/(org)/dashboard/settings/organization/components/CapSettingsCard.tsx
Line: 114-116

Comment:
**Settings save without edits.** `mergeSettings` now creates a fresh `transcriptionVocabulary` array every time. On mount, the reset effect replaces `lastSavedSettings.current`, but the autosave effect still holds the earlier debounced settings. Its per-key `!==` check treats the two identical lists as different and sends a save without any user edit. The save refreshes the settings, which repeats this mismatch and can keep sending writes and success notices. Compare vocabulary contents, and skip stale debounced values when resetting from server settings.

**Knowledge Base Used:** [Web product and collaborative workspace](https://app.greptile.com/-/switch-tenant?tenant=cap&next=%2Fcustom-context%2Fknowledge-base%2Fcapsoftware%2Fcap%2F-%2Fdocs%2Fweb-product.md)

---

For each issue above, determine whether it is valid and should be fixed. If so, fix it directly.

});

const formatVocabulary = (terms?: string[]) => (terms ?? []).join(", ");

const CapSettingsCard = () => {
const { activeOrganization } = useDashboardContext();
const organizationId = activeOrganization?.organization.id;
Expand All @@ -133,6 +140,9 @@ const OrganizationCapSettings = ({
const [settings, setSettings] =
useState<OrganizationSettings>(initialSettings);
const [showLanguageMenu, setShowLanguageMenu] = useState(false);
const [vocabularyText, setVocabularyText] = useState(
formatVocabulary(initialSettings.transcriptionVocabulary),
);

const lastSavedSettings = useRef<OrganizationSettings>(initialSettings);
const languageMenuRef = useRef<HTMLDivElement>(null);
Expand All @@ -144,6 +154,7 @@ const OrganizationCapSettings = ({
useEffect(() => {
const next = mergeSettings(organizationSettings);
setSettings(next);
setVocabularyText(formatVocabulary(next.transcriptionVocabulary));
lastSavedSettings.current = next;
}, [organizationSettings]);

Expand Down Expand Up @@ -199,6 +210,11 @@ const OrganizationCapSettings = ({
return;
}

if (changedKey === "transcriptionVocabulary") {
toast.success("Transcription vocabulary saved");
return;
}

if (changedKey === "defaultPlaybackSpeed") {
toast.success(
`Default playback speed set to ${
Expand Down Expand Up @@ -267,6 +283,16 @@ const OrganizationCapSettings = ({

const selectedSpeed = settings.defaultPlaybackSpeed ?? DEFAULT_PLAYBACK_SPEED;

const handleVocabularyBlur = () => {
const terms = sanitizeTranscriptionVocabulary(vocabularyText);
setVocabularyText(formatVocabulary(terms));
setSettings((prev) =>
formatVocabulary(prev.transcriptionVocabulary) === formatVocabulary(terms)
? prev
: { ...prev, transcriptionVocabulary: terms },
);
};

const handleLanguageChange = (language: AiGenerationLanguage) => {
if (!isAiGenerationLanguage(language)) {
return;
Expand Down Expand Up @@ -386,6 +412,21 @@ const OrganizationCapSettings = ({
</div>
}
/>
<SettingsRow
title="Transcription vocabulary"
meta={<ProChip />}
description="Company names, product names, and jargon to transcribe correctly. Separate with commas or new lines."
>
<textarea
value={vocabularyText}
onChange={(event) => setVocabularyText(event.target.value)}
onBlur={handleVocabularyBlur}
disabled={!user.isPro}
rows={3}
aria-label="Transcription vocabulary"
className="p-3 mt-3 w-full text-[13px] rounded-lg border resize-y border-gray-4 bg-gray-1 text-gray-12 disabled:opacity-50 disabled:cursor-not-allowed"
/>
</SettingsRow>
</SettingsCard>
</SettingsSection>
</div>
Expand Down
6 changes: 6 additions & 0 deletions apps/web/lib/assemblyai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {
AI_GENERATION_LANGUAGE_AUTO,
type AiGenerationLanguage,
type AiGenerationLanguageCode,
sanitizeTranscriptionVocabulary,
} from "@cap/web-domain";
import { AssemblyAI } from "assemblyai";

Expand Down Expand Up @@ -49,14 +50,19 @@ export const ASSEMBLYAI_SUPPORTED_LANGUAGES = [

export function getAssemblyAITranscriptionOptions(
language: AiGenerationLanguage,
vocabulary?: readonly string[] | null,
) {
// keyterms_prompt: 1,000 terms on universal-3-5-pro, 200 on universal-2, so
// the 100-term cap is valid on both models in the speech_models fallback.
const keyterms = sanitizeTranscriptionVocabulary(vocabulary ?? []);
const baseOptions = {
speech_models: [...ASSEMBLYAI_SPEECH_MODELS],
format_text: true,
punctuate: true,
// Verbatim words: the single transcription pass feeds both the word-level
// edit transcript and the caption VTT (which strips fillers itself).
disfluencies: true,
...(keyterms.length > 0 ? { keyterms_prompt: keyterms } : {}),
};

if (language === AI_GENERATION_LANGUAGE_AUTO) {
Expand Down
8 changes: 7 additions & 1 deletion apps/web/lib/live-transcribe-farm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { Storage } from "@cap/web-backend/src/Storage/index";
import {
type AiGenerationLanguage,
parseAiGenerationLanguage,
sanitizeTranscriptionVocabulary,
Video,
} from "@cap/web-domain";
import { and, eq, sql } from "drizzle-orm";
Expand Down Expand Up @@ -171,7 +172,12 @@ export async function buildLiveSessionState(options: {
return {
lastAudioSegmentIndex,
transcribedDurationMs: artifact?.transcribedDurationMs ?? 0,
transcriptionOptions: getAssemblyAITranscriptionOptions(language),
transcriptionOptions: getAssemblyAITranscriptionOptions(
language,
sanitizeTranscriptionVocabulary(
options.orgSettings?.transcriptionVocabulary,
),
),
urls: { manifest: manifest ?? "", audioInit: audioInit ?? "", segments },
};
}
Expand Down
9 changes: 8 additions & 1 deletion apps/web/workflows/live-transcribe.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { Storage } from "@cap/web-backend/src/Storage/index";
import {
type AiGenerationLanguage,
parseAiGenerationLanguage,
sanitizeTranscriptionVocabulary,
Video,
} from "@cap/web-domain";
import { eq } from "drizzle-orm";
Expand Down Expand Up @@ -47,6 +48,7 @@ type InitResult =
transcribedDurationMs: number;
languageCode: string | null;
orgLanguage: AiGenerationLanguage;
vocabulary: string[];
};

type ChunkStepResult =
Expand Down Expand Up @@ -121,6 +123,7 @@ export async function liveTranscribeWorkflow(
lastProcessedIndex: lastIndex,
targetSeconds,
language: languageCode ?? init.orgLanguage,
vocabulary: init.vocabulary,
// After repeated failures the chunk is skipped: advance past it and
// leave a gap for the canonical transcription to fill.
skipPastFailedChunk:
Expand Down Expand Up @@ -265,6 +268,9 @@ async function initLiveTranscription(
orgLanguage: parseAiGenerationLanguage(
row.orgSettings?.aiGenerationLanguage,
),
vocabulary: sanitizeTranscriptionVocabulary(
row.orgSettings?.transcriptionVocabulary,
),
};
}

Expand All @@ -274,6 +280,7 @@ async function processNextLiveChunk(options: {
lastProcessedIndex: number;
targetSeconds: number;
language: string;
vocabulary: string[];
skipPastFailedChunk: boolean;
withContext: boolean;
}): Promise<ChunkStepResult> {
Expand Down Expand Up @@ -417,7 +424,7 @@ async function processNextLiveChunk(options: {
: ("auto" as AiGenerationLanguage);
const transcript = await client.transcripts.transcribe({
audio: audioBuffer,
...getAssemblyAITranscriptionOptions(language),
...getAssemblyAITranscriptionOptions(language, options.vocabulary),
disfluencies: true,
});

Expand Down
Loading