-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathgeminiService.ts
More file actions
823 lines (719 loc) · 30.7 KB
/
Copy pathgeminiService.ts
File metadata and controls
823 lines (719 loc) · 30.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
import { GoogleGenAI, Chat, Modality, Type } from "@google/genai";
import { z } from "zod";
import { base64ToBytes, pcmToWav } from "./audioUtils";
import { VoiceResponse, Scenario } from "../types";
import { getConversationHistory, addToHistory } from "./conversationHistory";
import { generateScenarioSystemInstruction, generateScenarioSummaryPrompt, parseHintFromResponse, parseMultiCharacterResponse } from "./scenarioService";
import { getApiKeyOrEnv } from "./apiKeyService";
// Gemini TTS output format constants
const DEFAULT_PCM_SAMPLE_RATE = 24000; // 24kHz sample rate
const DEFAULT_PCM_CHANNELS = 1; // Mono audio
// Define the system instruction to enforce the language constraint
const SYSTEM_INSTRUCTION = `
You are a friendly and patient French language tutor.
Your goal is to help the user practice speaking French.
RESPONSE FORMAT (CRITICAL):
You MUST respond with structured JSON in this exact format:
{
"french": "Your complete French response here",
"english": "The English translation here"
}
Example:
User says: "Bonjour, je suis fatigue." (User means "I am tired" but made a mistake)
You respond with JSON:
{
"french": "Bonjour! Oh, tu es fatigué ? Pourquoi es-tu fatigué aujourd'hui ?",
"english": "Hello! Oh, you are tired? Why are you tired today?"
}
GUIDELINES:
1. Understand what the user says, but don't repeat it verbatim. Briefly acknowledge understanding when needed, but focus on responding naturally.
2. If the user makes a mistake, gently correct them in your French response, but keep the conversation flowing naturally.
3. Put your COMPLETE French response in the "french" field
4. Put the COMPLETE ENGLISH translation in the "english" field
5. Keep French and English SEPARATE - do NOT combine them in one field
`;
let ai: GoogleGenAI | null = null;
let chatSession: Chat | null = null;
// Track how many messages from shared history have been synced to the session
let syncedMessageCount = 0;
// Track the active scenario for scenario-aware prompting
let activeScenario: Scenario | null = null;
// Store pending scenario and history when ai is not yet initialized
let pendingScenario: Scenario | null = null;
let pendingHistory: Array<{ role: string; content: string }> | null = null;
/**
* Max number of characters supported in multi-character scenarios
*/
const MAX_CHARACTERS = 5;
/**
* Zod schema for single-character response.
* Separates French and English for TTS control.
*/
const SingleCharacterSchema = z.object({
french: z.string().describe("The complete response in French only"),
english: z.string().describe("The English translation of the French response"),
hint: z.string().describe("Hint for what the user should say or ask next - brief description in English")
});
/**
* Zod schema for free conversation mode response.
* Separates French and English for TTS control, with optional hint.
*/
const FreeConversationSchema = z.object({
french: z.string().describe("The complete response in French only"),
english: z.string().describe("The English translation of the French response")
});
/**
* Create Zod schema for multi-character response.
* Uses fixed labels ("Character 1", "Character 2", etc.) instead of actual names
* because LLMs don't reliably use exact character names in structured output.
* The processing code maps these labels back to actual characters by index.
*/
const createMultiCharacterSchema = (scenario: Scenario) => {
const count = Math.min(scenario.characters!.length, MAX_CHARACTERS);
const labels = Array.from({ length: count }, (_, i) => `Character ${i + 1}`);
// Allow hint at top level OR inside each character response (LLMs place it inconsistently)
return z.object({
characterResponses: z.array(
z.object({
characterName: z.string().describe(`Must be one of: ${labels.join(', ')}`),
french: z.string().describe("The character's complete response in French only"),
english: z.string().describe("The English translation of the French response"),
hint: z.string().optional().describe("Optional per-character hint")
})
),
hint: z.string().optional().describe("Hint for what the user should say or ask next - brief description in English")
});
};
/**
* Convert a standard JSON Schema object (as produced by z.toJSONSchema) to the
* uppercase-typed format required by the Gemini SDK's responseSchema field.
* Gemini expects "OBJECT", "STRING", "ARRAY" etc.; z.toJSONSchema produces lowercase.
* Only passes through fields that the Gemini Schema type supports.
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
function toGeminiSchema(jsonSchema: Record<string, any>): Record<string, any> {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const result: Record<string, any> = {};
if (jsonSchema.type) result.type = (jsonSchema.type as string).toUpperCase();
if (jsonSchema.description) result.description = jsonSchema.description;
if (jsonSchema.properties) {
result.properties = Object.fromEntries(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
Object.entries(jsonSchema.properties as Record<string, Record<string, any>>).map(
([k, v]) => [k, toGeminiSchema(v)]
)
);
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
if (jsonSchema.items) result.items = toGeminiSchema(jsonSchema.items as Record<string, any>);
if (jsonSchema.required) result.required = jsonSchema.required;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
if (jsonSchema.anyOf) result.anyOf = (jsonSchema.anyOf as Record<string, any>[]).map(toGeminiSchema);
if (jsonSchema.enum) result.enum = jsonSchema.enum;
if (jsonSchema.nullable !== undefined) result.nullable = jsonSchema.nullable;
return result;
}
/**
* Gemini-format response schemas derived from the Zod schemas above.
* These are passed as responseSchema to the chat session config, which prevents
* the model from returning an unexpected JSON shape (e.g. an array of turns).
* Derived via toGeminiSchema so the shape stays in sync with the Zod definitions.
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const SINGLE_CHARACTER_RESPONSE_SCHEMA = toGeminiSchema(z.toJSONSchema(SingleCharacterSchema) as Record<string, any>);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const FREE_CONVERSATION_RESPONSE_SCHEMA = toGeminiSchema(z.toJSONSchema(FreeConversationSchema) as Record<string, any>);
const createGeminiMultiCharacterSchema = (scenario: Scenario) =>
// eslint-disable-next-line @typescript-eslint/no-explicit-any
toGeminiSchema(z.toJSONSchema(createMultiCharacterSchema(scenario)) as Record<string, any>);
/**
* Helper function to create the chat session with current state.
* Only call when ai is initialized.
*/
function createChatSession(): void {
if (!ai) {
return;
}
const systemInstruction = activeScenario
? generateScenarioSystemInstruction(activeScenario)
: SYSTEM_INSTRUCTION;
// Convert history to SDK format if provided
const historyMessages = pendingHistory ? pendingHistory.map(msg => ({
role: msg.role === 'user' ? 'user' : 'model',
parts: [{ text: msg.content }]
})) : undefined;
// Pick the response schema that matches the scenario type.
// This enforces the JSON shape at the API level, preventing the model from
// returning an array of turns instead of a single response object.
const responseSchema = activeScenario && activeScenario.characters && activeScenario.characters.length > 1
? createGeminiMultiCharacterSchema(activeScenario)
: activeScenario
? SINGLE_CHARACTER_RESPONSE_SCHEMA
: FREE_CONVERSATION_RESPONSE_SCHEMA;
chatSession = ai.chats.create({
model: 'gemini-2.0-flash-lite',
config: {
systemInstruction: systemInstruction,
// Always use JSON response format for structured French/English separation
responseMimeType: 'application/json',
responseSchema,
},
...(historyMessages && { history: historyMessages }),
});
// Update sync counter if history was provided
if (pendingHistory) {
syncedMessageCount = pendingHistory.length;
} else {
syncedMessageCount = 0;
}
// Clear pending history after successful session creation
pendingHistory = null;
}
/**
* Resets the Gemini session and sync counter.
* Should be called when clearing conversation history.
* Optionally can set a new scenario for scenario-aware prompting.
* Can optionally pass history to initialize the session with existing messages.
*
* Always persists the scenario and history in state, even if ai is not yet initialized.
* When ai is initialized later, call this again or initializeSession to create the actual session.
*/
export const resetSession = (scenario?: Scenario | null, history?: Array<{ role: string; content: string }>) => {
// Always update the module-level state, regardless of ai initialization
activeScenario = scenario || null;
pendingScenario = scenario || null;
if (history) {
pendingHistory = history;
} else {
// Only reset sync counter if no history is provided (clearing state)
syncedMessageCount = 0;
pendingHistory = null;
}
// Only create the actual chat session if ai is initialized
if (ai) {
createChatSession();
}
};
/**
* Sets the active scenario and resets the session with new instructions.
*/
export const setScenario = (scenario: Scenario | null) => {
resetSession(scenario);
};
/**
* Ensures the Gemini AI instance is initialized
*/
function ensureAiInitialized(): void {
if (!ai) {
const apiKey = getApiKeyOrEnv('gemini');
if (!apiKey) {
throw new Error("Missing Gemini API Key");
}
ai = new GoogleGenAI({ apiKey });
}
}
/**
* Gets AI's understanding/summary of a scenario description.
*/
export const processScenarioDescription = async (description: string): Promise<string> => {
ensureAiInitialized();
const response = await ai.models.generateContent({
model: 'gemini-2.0-flash-lite',
contents: [{
parts: [{ text: generateScenarioSummaryPrompt(description) }],
}],
});
return response.text || "I understand the scenario. Ready to begin when you are!";
};
/**
* Transcribes audio to text using Gemini.
*/
export const transcribeAudio = async (audioBase64: string, mimeType: string): Promise<string> => {
ensureAiInitialized();
const response = await ai.models.generateContent({
model: 'gemini-2.0-flash-lite',
contents: [{
parts: [
{ text: "Transcribe this audio exactly as spoken. Only output the transcription, nothing else." },
{
inlineData: {
data: audioBase64,
mimeType: mimeType,
},
},
],
}],
});
const text = response.text || "";
if (!text.trim()) {
throw new Error("Transcription returned empty text");
}
return text;
};
/**
* Transcribes audio and produces both a raw transcript and a cleaned-up version
* in a single LLM call using structured output.
*/
export const transcribeAndCleanupAudio = async (
audioBase64: string,
mimeType: string
): Promise<{ rawTranscript: string; cleanedTranscript: string }> => {
ensureAiInitialized();
const response = await ai!.models.generateContent({
model: 'gemini-2.0-flash-lite',
contents: [{
parts: [
{
text: `Listen to this audio and produce two versions of the transcript:
1. "rawTranscript": Transcribe the audio exactly as spoken, including all filler words, false starts, repetitions, self-corrections, and hesitations.
2. "cleanedTranscript": A cleaned-up version of the same transcript with the following removed:
- Filler words (um, uh, like, you know, so, etc.)
- False starts and repetitions
- Self-corrections and clarifications (e.g., "I mean", "actually", "wait no")
- Verbal pauses and hesitations
The cleaned version should preserve the core meaning and intent, reading smoothly while staying natural.`
},
{
inlineData: {
data: audioBase64,
mimeType: mimeType,
},
},
],
}],
config: {
responseMimeType: 'application/json',
responseSchema: {
type: Type.OBJECT,
properties: {
rawTranscript: {
type: Type.STRING,
description: 'Exact transcription of the audio as spoken, including all filler words and hesitations',
},
cleanedTranscript: {
type: Type.STRING,
description: 'Cleaned-up version with fillers, false starts, and self-corrections removed',
},
},
required: ['rawTranscript', 'cleanedTranscript'],
},
},
});
const text = response.text || "";
if (!text.trim()) {
throw new Error("Transcription returned empty response");
}
let parsed;
try {
parsed = JSON.parse(text);
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
throw new Error(`Failed to parse Gemini transcription response: ${errorMessage}. Raw response: ${text}`);
}
return {
rawTranscript: parsed.rawTranscript || "",
cleanedTranscript: parsed.cleanedTranscript || "",
};
};
/**
* Initializes the Gemini Chat session.
* Must be called with a valid API Key.
* Creates a fresh session and uses any pending scenario/history that was set before ai was initialized.
*/
export const initializeSession = async () => {
ensureAiInitialized();
// We use gemini-2.0-flash-lite for the logic/conversation as it handles audio input well,
// but we will ask for TEXT output to maintain REST compatibility, then TTS it.
// If there was a pending scenario set before ai was initialized, use it
if (pendingScenario) {
activeScenario = pendingScenario;
}
// Create session with any pending state (scenario, history)
createChatSession();
};
/**
* Generate speech audio for a specific character using Gemini TTS
* @param text The text to convert to speech
* @param voiceName The Gemini voice name to use
* @returns Blob URL for the generated audio
*/
export const generateCharacterSpeech = async (
text: string,
voiceName: string
): Promise<string> => {
if (!ai) {
ensureAiInitialized();
}
// Sanitize text to prevent breaking the delimiter
const sanitizedText = text.replace(/<\/text>/g, '<\\/text>');
const systemPrompt = `You are to read out the following text in a friendly, encouraging tone. When speaking French, use a natural French accent. You MUST output ONLY AUDIO, not TEXT. Again, ONLY AUDIO, not TEXT. Here's the text enclosed in <text> tags: <text>${sanitizedText}</text>`;
const ttsResponse = await ai!.models.generateContent({
model: 'gemini-2.5-flash-preview-tts',
contents: [{ parts: [{ text: systemPrompt }] }],
config: {
responseModalities: [Modality.AUDIO],
speechConfig: {
voiceConfig: {
prebuiltVoiceConfig: {
voiceName: voiceName
}
}
}
}
});
// Extract audio from TTS response
const candidate = ttsResponse.candidates?.[0];
const parts = candidate?.content?.parts;
if (!parts || parts.length === 0) {
throw new Error(`No content received from TTS model for character with voice ${voiceName}.`);
}
// Find the inline data part which contains the audio
const audioPart = parts.find(p => p.inlineData);
if (!audioPart || !audioPart.inlineData) {
throw new Error(`No audio data received from TTS model for character with voice ${voiceName}.`);
}
// Convert base64 to blob and create URL
const audioBytes = base64ToBytes(audioPart.inlineData.data);
// Gemini TTS returns raw PCM, convert it to WAV format
const audioBlob = pcmToWav(audioBytes, DEFAULT_PCM_SAMPLE_RATE, DEFAULT_PCM_CHANNELS);
const audioUrl = URL.createObjectURL(audioBlob);
return audioUrl;
};
/**
* Sends a user audio blob to the model and returns the response with audio and text.
*/
export const sendVoiceMessage = async (
audioBase64: string,
mimeType: string,
signal?: AbortSignal
): Promise<VoiceResponse> => {
if (!chatSession || !ai) {
if (activeScenario) {
await resetSession(activeScenario);
} else {
await initializeSession();
}
if (!chatSession || !ai) {
throw new Error("Chat session not initialized.");
}
}
try {
// Helper to make API calls cancellable via AbortSignal
const abortablePromise = <T>(promise: Promise<T>): Promise<T> => {
if (!signal) return promise;
return Promise.race([
promise,
new Promise<T>((_, reject) => {
signal.addEventListener('abort', () => {
reject(new DOMException('Request aborted', 'AbortError'));
});
})
]);
};
// Step 1: Transcribe user audio
const transcribeResponse = await abortablePromise(ai.models.generateContent({
model: 'gemini-2.0-flash-lite',
contents: [{
parts: [
{ text: "Transcribe this audio exactly as spoken. Only output the transcription, nothing else." },
{
inlineData: {
data: audioBase64,
mimeType: mimeType,
},
},
],
}],
}));
const userText = transcribeResponse.text || "";
// Validate transcription - don't proceed with empty text
if (!userText || userText.trim().length === 0) {
throw new Error("Transcription failed or returned empty text. Please try speaking again.");
}
// Sync session with shared history if needed (lazy sync when actually sending a message)
// This happens when switching back to Gemini from another provider
const sharedHistory = getConversationHistory();
// If there are unsynced messages, recreate the session with full history
// This avoids redundant API calls from replaying messages one by one
if (sharedHistory.length > syncedMessageCount) {
// Recreate session with all history passed directly to the SDK
resetSession(activeScenario, sharedHistory);
// Ensure session was created successfully
if (!chatSession) {
throw new Error("Failed to sync session with history");
}
}
// Step 2: Send User Audio to Chat Model to get Text Response
const chatResponse = await abortablePromise(chatSession.sendMessage({
message: [
{
inlineData: {
data: audioBase64,
mimeType: mimeType,
},
},
],
}));
const rawModelText = chatResponse.text; // Access text property directly
if (!rawModelText) {
throw new Error("No text response received from chat model.");
}
// Check if this is a multi-character scenario
if (activeScenario && activeScenario.characters && activeScenario.characters.length > 1) {
// Parse and validate JSON response with Zod
const MultiCharacterSchema = createMultiCharacterSchema(activeScenario);
let jsonResponse;
try {
jsonResponse = JSON.parse(rawModelText);
} catch (parseError) {
const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
throw new Error(`Failed to parse multi-character response as JSON: ${errorMessage}. Raw response: ${rawModelText}`);
}
// Use safeParse for better error handling
const validationResult = MultiCharacterSchema.safeParse(jsonResponse);
if (!validationResult.success) {
throw new Error(`Failed to validate multi-character response: ${validationResult.error.message}. Raw response: ${rawModelText}`);
}
const validated = validationResult.data;
// Map fixed character labels ("Character 1", etc.) back to actual characters by index
const characterResponses = validated.characterResponses.map(resp => {
const label = resp.characterName.trim();
// Extract the number from "Character N" label
const match = label.match(/^character\s+(\d+)$/i);
if (!match) {
throw new Error(`Unexpected character label "${label}" — expected format "Character N". Raw response: ${rawModelText}`);
}
const index = parseInt(match[1], 10) - 1; // Convert 1-based to 0-based
if (index < 0 || index >= activeScenario.characters!.length) {
throw new Error(`Character index ${index + 1} out of range (scenario has ${activeScenario.characters!.length} characters). Raw response: ${rawModelText}`);
}
const character = activeScenario.characters![index];
return {
characterId: character.id,
characterName: character.name,
french: resp.french.trim(),
english: resp.english.trim()
};
});
// Merge successive messages from the same character to reduce TTS requests
const mergedCharacterResponses = characterResponses.reduce<Array<{
characterId: string;
characterName: string;
french: string;
english: string;
}>>((acc, current) => {
if (acc.length === 0) {
return [current];
}
const lastResponse = acc[acc.length - 1];
if (lastResponse.characterId === current.characterId) {
// Same character speaking again - merge the messages
lastResponse.french = `${lastResponse.french} ${current.french}`;
lastResponse.english = `${lastResponse.english} ${current.english}`;
return acc;
}
// Different character - add as new response
return [...acc, current];
}, []);
// Extract hint: prefer top-level, fall back to last character response's hint
const hint = validated.hint
|| validated.characterResponses[validated.characterResponses.length - 1]?.hint
|| "Continue the conversation";
const parsed = {
characterResponses: mergedCharacterResponses,
hint
};
// Check if operation was cancelled before updating history
if (signal?.aborted) {
throw new DOMException('Request aborted', 'AbortError');
}
// Generate audio for each character IN PARALLEL (wrapped with abort support)
// Only use French text for TTS
const audioPromises = parsed.characterResponses.map(async (charResp) => {
const character = activeScenario.characters.find(c => c.id === charResp.characterId);
if (!character) {
throw new Error(`Character not found: ${charResp.characterName} (ID: ${charResp.characterId})`);
}
const audioUrl = await abortablePromise(generateCharacterSpeech(charResp.french, character.voiceName));
return { ...charResp, audioUrl, voiceName: character.voiceName };
});
const results = await Promise.allSettled(audioPromises);
// Process results: extract successes and mark failures
const characterAudios = results.map((result, idx) => {
if (result.status === 'rejected') {
console.error(`TTS failed for character ${parsed.characterResponses[idx].characterName}:`, result.reason);
// Return character data without audio, flagged as failed
const character = activeScenario.characters.find(c => c.id === parsed.characterResponses[idx].characterId);
return {
...parsed.characterResponses[idx],
audioUrl: '', // Use empty string instead of undefined to satisfy type
audioGenerationFailed: true,
voiceName: character?.voiceName || ''
};
}
return { ...result.value, audioGenerationFailed: false };
});
// Construct combined text for conversation history (French followed by English)
const combinedModelText = parsed.characterResponses.map(cr => `${cr.french} ${cr.english}`).join(' ');
// Check again after audio generation (user may have aborted during TTS)
if (signal?.aborted) {
// Revoke any successfully generated audio URLs
characterAudios.forEach(ca => {
if (ca.audioUrl) URL.revokeObjectURL(ca.audioUrl);
});
throw new DOMException('Request aborted', 'AbortError');
}
// Sync to shared conversation history
addToHistory("user", userText);
addToHistory("assistant", combinedModelText);
syncedMessageCount += 2;
// Return multi-character response
// Combine French and English for display
return {
audioUrl: characterAudios.map(ca => ca.audioUrl),
modelText: characterAudios.map(ca => `${ca.french} ${ca.english}`),
userText,
hint: parsed.hint, // Required field, always present
characters: characterAudios.map(ca => ({
characterId: ca.characterId,
characterName: ca.characterName,
voiceName: ca.voiceName,
audioGenerationFailed: ca.audioGenerationFailed,
frenchText: ca.french // Include French text for TTS retry
}))
};
} else {
// Single-character scenario with JSON response
if (activeScenario) {
// Parse and validate JSON response
let jsonResponse;
try {
jsonResponse = JSON.parse(rawModelText);
} catch (parseError) {
const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
throw new Error(`Failed to parse single-character response as JSON: ${errorMessage}. Raw response: ${rawModelText}`);
}
// Use safeParse for better error handling
const validationResult = SingleCharacterSchema.safeParse(jsonResponse);
if (!validationResult.success) {
throw new Error(`Failed to validate single-character response: ${validationResult.error.message}. Raw response: ${rawModelText}`);
}
const validated = validationResult.data;
// Check if operation was cancelled before generating audio
if (signal?.aborted) {
throw new DOMException('Request aborted', 'AbortError');
}
// Combine French and English for display and history
const modelText = `${validated.french} ${validated.english}`;
// Step 3: Send Text Response to TTS Model to get Audio (use ONLY French text)
// Use character voice if available, otherwise default (wrapped with abort support)
const voiceName = activeScenario?.characters?.[0]?.voiceName || "aoede";
let audioUrl = '';
try {
audioUrl = await abortablePromise(generateCharacterSpeech(validated.french, voiceName));
} catch (ttsError) {
// Re-throw aborts - user cancelled the operation
if (ttsError instanceof DOMException && ttsError.name === 'AbortError') {
throw ttsError;
}
// Log TTS failures but continue with empty audioUrl
console.error('TTS generation failed for single-character response:', ttsError);
// Will return empty audioUrl - UI shows "Audio unavailable" with retry
}
// Check if aborted after TTS (in case signal was set during generation)
if (signal?.aborted) {
// Revoke audio URL if it was generated
if (audioUrl) URL.revokeObjectURL(audioUrl);
throw new DOMException('Request aborted', 'AbortError');
}
// Update history after TTS (success or non-abort failure)
// This ensures aborted operations don't pollute history,
// but TTS failures still show text with retry button
addToHistory("user", userText);
addToHistory("assistant", modelText);
syncedMessageCount += 2;
return {
audioUrl,
userText,
modelText,
hint: validated.hint,
voiceName,
audioGenerationFailed: !audioUrl, // Empty audioUrl means TTS failed
characters: [{
characterId: activeScenario?.characters?.[0]?.id || '',
characterName: activeScenario?.characters?.[0]?.name || '',
voiceName,
audioGenerationFailed: !audioUrl,
frenchText: validated.french // Include French text for TTS retry
}]
};
} else {
// No scenario - free conversation mode with JSON response
// Parse and validate JSON response with Zod
let jsonResponse;
try {
jsonResponse = JSON.parse(rawModelText);
} catch (parseError) {
const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
throw new Error(`Failed to parse free conversation response as JSON: ${errorMessage}. Raw response: ${rawModelText}`);
}
// Use safeParse for better error handling
const validationResult = FreeConversationSchema.safeParse(jsonResponse);
if (!validationResult.success) {
throw new Error(`Failed to validate free conversation response: ${validationResult.error.message}. Raw response: ${rawModelText}`);
}
const validated = validationResult.data;
// Check if operation was cancelled before generating audio
if (signal?.aborted) {
throw new DOMException('Request aborted', 'AbortError');
}
// Combine French and English for display and history
const modelText = `${validated.french} ${validated.english}`;
// Step 3: Send ONLY French text to TTS (not the English translation)
const voiceName = "aoede";
let audioUrl = '';
try {
audioUrl = await abortablePromise(generateCharacterSpeech(validated.french, voiceName));
} catch (ttsError) {
// Re-throw aborts - user cancelled the operation
if (ttsError instanceof DOMException && ttsError.name === 'AbortError') {
throw ttsError;
}
// Log TTS failures but continue with empty audioUrl
console.error('TTS generation failed for free-conversation response:', ttsError);
// Will return empty audioUrl - UI shows "Audio unavailable" with retry
}
// Check if aborted after TTS (in case signal was set during generation)
if (signal?.aborted) {
// Revoke audio URL if it was generated
if (audioUrl) URL.revokeObjectURL(audioUrl);
throw new DOMException('Request aborted', 'AbortError');
}
// Update history after TTS (success or non-abort failure)
// This ensures aborted operations don't pollute history,
// but TTS failures still show text with retry button
addToHistory("user", userText);
addToHistory("assistant", modelText);
syncedMessageCount += 2;
return {
audioUrl,
userText,
modelText,
hint: undefined, // No hints in free conversation mode
voiceName,
audioGenerationFailed: !audioUrl, // Empty audioUrl means TTS failed
characters: [{
characterId: '',
characterName: '',
voiceName,
audioGenerationFailed: !audioUrl,
frenchText: validated.french // Include French text for TTS retry
}]
};
}
}
} catch (error) {
console.error("Error communicating with Gemini:", error);
throw error;
}
};