Feature/speech managed identity (#543)

* Bugfix - deleted duplicate enable_external_healthcheck entry

* Feature - updated Speech Service to use Managed Identity in addition to the key, added MAG functionality via Azure Speech SDK since the Fast Transcription API is not available in MAG, updated Admin Setup Walkthrough so it goes to the right place in the settings when Next is clicked, updated Speech requirements in Walkthrough, rewrote Admin Configuration docs, updated/corrected Managed Identity roles in Setup Instructions Special docs.

* Update application/single_app/templates/admin_settings.html

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update application/single_app/functions_settings.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update application/single_app/functions_documents.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update application/single_app/functions_documents.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: Paul Lizer <paullizer@microsoft.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: 3 people

application/single_app/config.py

@@ -64,6 +64,7 @@
 from io import BytesIO
 from typing import List
 
+import azure.cognitiveservices.speech as speechsdk
 from azure.cosmos import CosmosClient, PartitionKey, exceptions
 from azure.cosmos.exceptions import CosmosResourceNotFoundError
 from azure.core.credentials import AzureKeyCredential

application/single_app/functions_documents.py

@@ -4765,6 +4765,24 @@ def _split_audio_file(input_path: str, chunk_seconds: int = 540) -> List[str]:
     print(f"Produced {len(chunks)} WAV chunks: {chunks}")
     return chunks
 
+# Azure Speech SDK helper to get speech config with fresh token
+def _get_speech_config(settings, endpoint: str, locale: str):
+    """Get speech config with fresh token"""
+    if settings.get("speech_service_authentication_type") == "managed_identity":
+        credential = DefaultAzureCredential()
+        token = credential.get_token(cognitive_services_scope)
+        speech_config = speechsdk.SpeechConfig(endpoint=endpoint)
+
+        # Set the authorization token AFTER creating the config
+        speech_config.authorization_token = token.token
+    else:
+        key = settings.get("speech_service_key", "")
+        speech_config = speechsdk.SpeechConfig(endpoint=endpoint, subscription=key)
+
+    speech_config.speech_recognition_language = locale
+    print(f"[Debug] Speech config obtained successfully", flush=True)
+    return speech_config
+
 def process_audio_document(
     document_id: str,
     user_id: str,
@@ -4804,32 +4822,65 @@ def process_audio_document(
     # 3) transcribe each WAV chunk
     settings = get_settings()
     endpoint = settings.get("speech_service_endpoint", "").rstrip('/')
-    key = settings.get("speech_service_key", "")
     locale = settings.get("speech_service_locale", "en-US")
-    url = f"{endpoint}/speechtotext/transcriptions:transcribe?api-version=2024-11-15"
 
     all_phrases: List[str] = []
-    for idx, chunk_path in enumerate(chunk_paths, start=1):
-        update_callback(current_file_chunk=idx, status=f"Transcribing chunk {idx}/{len(chunk_paths)}…")
-        print(f"Transcribing WAV chunk: {chunk_path}")
-
-        with open(chunk_path, 'rb') as audio_f:
-            files = {
-                'audio': (os.path.basename(chunk_path), audio_f, 'audio/wav'),
-                'definition': (None, json.dumps({'locales':[locale]}), 'application/json')
-            }
-            headers = {'Ocp-Apim-Subscription-Key': key}
-            resp = requests.post(url, headers=headers, files=files)
-        try:
-            resp.raise_for_status()
-        except Exception as e:
-            print(f"[Error] HTTP error for {chunk_path}: {e}")
-            raise
 
-        result = resp.json()
-        phrases = result.get('combinedPhrases', [])
-        print(f"Received {len(phrases)} phrases")
-        all_phrases += [p.get('text','').strip() for p in phrases if p.get('text')]
+    # Fast Transcription API not yet available in sovereign clouds, so use SDK
+    if AZURE_ENVIRONMENT in ("usgovernment", "custom"):
+        for idx, chunk_path in enumerate(chunk_paths, start=1):
+            print(f"[Debug] Transcribing chunk {idx}: {chunk_path}")
+
+            # Get fresh config (tokens expire after ~1 hour)
+            speech_config = _get_speech_config(settings, endpoint, locale)
+
+            audio_config = speechsdk.AudioConfig(filename=chunk_path)
+            speech_recognizer = speechsdk.SpeechRecognizer(
+                speech_config=speech_config,
+                audio_config=audio_config
+            )
+
+            result = speech_recognizer.recognize_once()
+            if result.reason == speechsdk.ResultReason.RecognizedSpeech:
+                print(f"[Debug] Recognized: {result.text}")
+                all_phrases.append(result.text)
+            elif result.reason == speechsdk.ResultReason.NoMatch:
+                print(f"[Warning] No speech in {chunk_path}")
+            elif result.reason == speechsdk.ResultReason.Canceled:
+                print(f"[Error] {result.cancellation_details.reason}: {result.cancellation_details.error_details}")
+                raise RuntimeError(f"Transcription canceled for {chunk_path}: {result.cancellation_details.error_details}")
+
+    else:
+        # Use the fast-transcription API if not in sovereign or custom cloud
+        url = f"{endpoint}/speechtotext/transcriptions:transcribe?api-version=2024-11-15"
+        for idx, chunk_path in enumerate(chunk_paths, start=1):
+            update_callback(current_file_chunk=idx, status=f"Transcribing chunk {idx}/{len(chunk_paths)}…")
+            print(f"[Debug] Transcribing WAV chunk: {chunk_path}")
+
+            with open(chunk_path, 'rb') as audio_f:
+                files = {
+                    'audio': (os.path.basename(chunk_path), audio_f, 'audio/wav'),
+                    'definition': (None, json.dumps({'locales':[locale]}), 'application/json')
+                }
+                if settings.get("speech_service_authentication_type") == "managed_identity":
+                    credential = DefaultAzureCredential()
+                    token = credential.get_token(cognitive_services_scope)
+                    headers = {'Authorization': f'Bearer {token.token}'}
+                else:
+                    key = settings.get("speech_service_key", "")
+                    headers = {'Ocp-Apim-Subscription-Key': key}
+                
+                resp = requests.post(url, headers=headers, files=files)
+            try:
+                resp.raise_for_status()
+            except Exception as e:
+                print(f"[Error] HTTP error for {chunk_path}: {e}")
+                raise
+
+            result = resp.json()
+            phrases = result.get('combinedPhrases', [])
+            print(f"[Debug] Received {len(phrases)} phrases")
+            all_phrases += [p.get('text','').strip() for p in phrases if p.get('text')]
 
     # 4) cleanup WAV chunks
     for p in chunk_paths:

application/single_app/requirements.txt

@@ -52,4 +52,5 @@ cython
 pyyaml==6.0.2
 aiohttp==3.12.15
 html2text==2025.4.15
-matplotlib==3.10.7
+matplotlib==3.10.7
+azure-cognitiveservices-speech==1.47.0

application/single_app/route_frontend_admin_settings.py

@@ -668,6 +668,7 @@ def is_valid_url(url):
                 'speech_service_endpoint': form_data.get('speech_service_endpoint', '').strip(),
                 'speech_service_location': form_data.get('speech_service_location', '').strip(),
                 'speech_service_locale': form_data.get('speech_service_locale', '').strip(),
+                'speech_service_authentication_type': form_data.get('speech_service_authentication_type', 'key'),
                 'speech_service_key': form_data.get('speech_service_key', '').strip(),
 
                 'metadata_extraction_model': form_data.get('metadata_extraction_model', '').strip(),

application/single_app/static/js/admin/admin_settings.js

@@ -1654,6 +1654,15 @@ function setupToggles() {
         });
     }
 
+    const speechAuthType = document.getElementById('speech_service_authentication_type');
+    if (speechAuthType) {
+        speechAuthType.addEventListener('change', function () {
+            document.getElementById('speech_service_key_container').style.display =
+                (this.value === 'key') ? 'block' : 'none';
+            markFormAsModified();
+        });
+    }
+
     const officeAuthType = document.getElementById('office_docs_authentication_type');
     const connStrGroup = document.getElementById('office_docs_storage_conn_str_group');
     const urlGroup = document.getElementById('office_docs_storage_url_group');
@@ -3113,28 +3122,41 @@ function handleTabNavigation(stepNumber) {
         5: 'ai-models-tab',   // Embedding settings (now in AI Models tab)
         6: 'search-extract-tab', // AI Search settings
         7: 'search-extract-tab', // Document Intelligence settings
-        8: 'workspaces-tab',  // Video support
-        9: 'workspaces-tab',  // Audio support
+        8: 'search-extract-tab',  // Video support
+        9: 'search-extract-tab',  // Audio support
         10: 'safety-tab',     // Content safety
-        11: 'system-tab',     // User feedback and archiving (renamed from other-tab)
+        11: 'safety-tab',     // User feedback and archiving (changed from system-tab)
         12: 'citation-tab'    // Enhanced Citations and Image Generation
     };
 
     // Activate the appropriate tab
     const tabId = stepToTab[stepNumber];
     if (tabId) {
-        const tab = document.getElementById(tabId);
-        if (tab) {
-            // Use bootstrap Tab to show the tab
-            const bootstrapTab = new bootstrap.Tab(tab);
-            bootstrapTab.show();
-            
-            // Scroll to the relevant section after a small delay to allow tab to switch
-            setTimeout(() => {
-                // For tabs that need to jump to specific sections
-                scrollToRelevantSection(stepNumber, tabId);
-            }, 300);
+        // Check if we're using sidebar navigation or tab navigation
+        const sidebarToggle = document.getElementById('admin-settings-toggle');
+        
+        if (sidebarToggle) {
+            // Using sidebar navigation - call showAdminTab function
+            const tabName = tabId.replace('-tab', ''); // Remove '-tab' suffix
+            if (typeof showAdminTab === 'function') {
+                showAdminTab(tabName);
+            } else if (typeof window.showAdminTab === 'function') {
+                window.showAdminTab(tabName);
+            }
+        } else {
+            // Using Bootstrap tabs
+            const tab = document.getElementById(tabId);
+            if (tab) {
+                // Use bootstrap Tab to show the tab
+                const bootstrapTab = new bootstrap.Tab(tab);
+                bootstrapTab.show();
+            }
         }
+        
+        // Scroll to the relevant section after a small delay to allow tab to switch
+        setTimeout(() => {
+            scrollToRelevantSection(stepNumber, tabId);
+        }, 300);
     }
 }
 
@@ -3148,23 +3170,50 @@ function scrollToRelevantSection(stepNumber, tabId) {
     let targetElement = null;
 
     switch (stepNumber) {
+        case 1: // App title and logo
+            targetElement = document.getElementById('branding-section');
+            break;
+        case 2: // GPT settings
+            targetElement = document.getElementById('gpt-configuration');
+            break;
+        case 3: // GPT model selection
+            targetElement = document.getElementById('gpt_models_list')?.closest('.mb-3');
+            break;
         case 4: // Workspaces toggle section
-            targetElement = document.getElementById('enable_user_workspace')?.closest('.card');
+            targetElement = document.getElementById('personal-workspaces-section');
+            break;
+        case 5: // Embedding settings
+            targetElement = document.getElementById('embeddings-configuration');
+            break;
+        case 6: // AI Search settings
+            targetElement = document.getElementById('azure-ai-search-section');
+            break;
+        case 7: // Document Intelligence settings
+            targetElement = document.getElementById('document-intelligence-section');
             break;
         case 8: // Video file support
             targetElement = document.getElementById('enable_video_file_support')?.closest('.form-group');
             break;
         case 9: // Audio file support
             targetElement = document.getElementById('enable_audio_file_support')?.closest('.form-group');
             break;
+        case 10: // Content safety
+            targetElement = document.getElementById('content-safety-section');
+            break;
+        case 11: // User feedback and archiving
+            targetElement = document.getElementById('user-feedback-section');
+            break;
+        case 12: // Enhanced citations and image generation
+            targetElement = document.getElementById('enhanced-citations-section');
+            break;
         default:
             // For other steps, no specific scrolling
             break;
     }
 
     // If we found a target element, scroll to it
     if (targetElement) {
-        targetElement.scrollIntoView({ behavior: 'smooth', block: 'center' });
+        targetElement.scrollIntoView({ behavior: 'smooth', block: 'start' });
     }
 }
 
@@ -3302,9 +3351,14 @@ function isStepComplete(stepNumber) {
 
             // Otherwise check settings
             const speechEndpoint = document.getElementById('speech_service_endpoint')?.value;
-            const speechKey = document.getElementById('speech_service_key')?.value;
+            const authType = document.getElementById('speech_service_authentication_type').value;
+            const key = document.getElementById('speech_service_key').value;
 
-            return speechEndpoint && speechKey;
+            if (!speechEndpoint || (authType === 'key' && !key)) {
+                 return false;
+            } else {
+                return true;
+            }
 
         case 10: // Content safety - always complete (optional)
         case 11: // User feedback and archiving - always complete (optional)

application/single_app/templates/admin_settings.html

@@ -369,18 +369,24 @@ <h4>9. Enable Audio File Support</h4>
                             <strong>Required:</strong> Audio support configuration is required if workspaces are enabled.
                         </div>
 
-                        <p class="mb-3">Use the <strong>Audio Support</strong> settings in the Workspaces tab to enable and configure audio file processing.</p>
+                        <p class="mb-3">Use the <strong>Audio Support</strong> settings on the Search & Extract tab to enable and configure audio file processing.</p>
 
                         <ul class="list-group mb-4">
                             <li class="list-group-item d-flex justify-content-between align-items-center">
                                 <span>Speech Service Endpoint</span>
                                 <span id="audio-endpoint-badge" class="badge bg-danger">Required</span>
                             </li>
                             <li class="list-group-item d-flex justify-content-between align-items-center">
-                                <span>Speech Service Key</span>
-                                <span id="audio-key-badge" class="badge bg-danger">Required</span>
+                                <span>Authentication</span>
+                                <span id="speech-auth-badge" class="badge bg-danger">Required</span>
                             </li>
                         </ul>
+                        
+                        <div class="alert alert-info">
+                            <strong>Note:</strong> If using Managed Identity authentication ensure the Service Principal has been assigned the Cognitive Services Speech Contributor role on the Azure Speech Service and that the Speech endpoint is configured with a custom domain name.
+                            
+                            <i class="bi bi-info-circle ms-2" data-bs-toggle="tooltip" title="When a Cognitive Service has a custom domain name, the endpoint will typically look like https://<resource-name>.cognitiveservices.azure.<com or us> instead of https://<location>.cognitiveservices.azure.<com or us>"></i>
+                        </div>
                     </div>
 
                     <!-- Step 10: Content Safety -->
@@ -2831,7 +2837,7 @@ <h5>Speech Service Settings</h5>
                             <input type="text" class="form-control"
                                 id="speech_service_endpoint" name="speech_service_endpoint"
                                 value="{{ settings.speech_service_endpoint or '' }}"
-                                placeholder="https://<location>.cognitiveservices.azure.<com or us>/">
+                                placeholder="https://<location or custom domain>.cognitiveservices.azure.<com or us>/">
                         </div>
 
                         <div class="mb-3">
@@ -2849,17 +2855,41 @@ <h5>Speech Service Settings</h5>
                         </div>
 
                         <div class="mb-3">
-                            <label for="speech_service_key" class="form-label">API Key</label>
+                            <label for="speech_service_authentication_type" class="form-label">
+                                Authentication Type
+                            </label>
+                            <select class="form-select" id="speech_service_authentication_type" name="speech_service_authentication_type">
+                                <option value="key" {% if settings.speech_service_authentication_type == 'key' or not settings.speech_service_authentication_type %}selected{% endif %}>
+                                    Key
+                                </option>
+                                <option value="managed_identity" {% if settings.speech_service_authentication_type == 'managed_identity' %}selected{% endif %}>
+                                    Managed Identity
+                                </option>
+                            </select>
+                        </div>
+                        <div class="mb-3" id="speech_service_key_container" {% if settings.speech_service_authentication_type == 'managed_identity' %}style="display: none;"{% endif %}>
+                            <label for="speech_service_key" class="form-label">
+                                API Key
+                            </label>
                             <div class="input-group">
-                            <input type="password" class="form-control"
-                                    id="speech_service_key" name="speech_service_key"
-                                    value="{{ settings.speech_service_key or '' }}">
-                            <button type="button" class="btn btn-outline-secondary"
-                                    id="toggle_speech_service_key">Show</button>
+                                <input
+                                    type="password"
+                                    class="form-control"
+                                    id="speech_service_key"
+                                    name="speech_service_key"
+                                    value="{{ settings.speech_service_key or '' }}"
+                                >
+                                <button
+                                    type="button"
+                                    class="btn btn-outline-secondary"
+                                    id="toggle_speech_service_key"
+                                >
+                                    Show
+                                </button>
                             </div>
                         </div>
-                    </div>
 
+                    </div>
                     <p class="mt-2 mb-0">
                         <small class="text-muted">
                             <a href="#citation" onclick="switchTab(event, 'citation-tab')">