feat: add AI Q&A feature with TTS integration and audio playback controls
This commit adds a comprehensive Q&A feature to the NiceGUI interface: Features: - Added Q&A persona for interactive question-and-answer conversations - AI responds to user questions/statements in the whisper output - Character and word limits to reduce crosstalk (200 words/1200 chars max) - Auto-clear AI output after each response (keeps whisper for context) - Automatic TTS trigger when source is AI and TTS is enabled - Audio playback controls (play/stop) in AI output panel - Non-blocking audio playback in background thread Changes: - config/custom_personas.yaml.example: Added Q&A persona configuration - src/ai_provider.py: Extended AITextProcessor to support custom personas with limits - src/whispering_ui/state.py: Added TTS audio playback state variables - src/whispering_ui/bridge.py: Implemented Q&A mode logic and audio playback - src/whispering_ui/main.py: Pass bridge to output panels for audio controls - src/whispering_ui/components/output.py: Added play/stop buttons to AI panel The Q&A mode enables natural conversation with AI agents, making Whispering more interactive and useful for voice-driven Q&A sessions.
+243 −21 · 6 files changed
-
config/custom_personas.yaml.example+38 −0 -
src/ai_provider.py+57 −10 -
src/whispering_ui/bridge.py+111 −8 -
src/whispering_ui/components/output.py+33 −1 -
src/whispering_ui/main.py+2 −2 -
src/whispering_ui/state.py+2 −0
@@ -4,6 +4,44 @@ # Example personas: personas: + qa: + name: "Q&A" + description: "Interactive question and answer mode - AI responds to your questions and conversations" + system_prompt: | + You are a helpful AI assistant engaged in a natural conversation. Your role is to answer questions, provide information, and engage in dialogue based on what the user says. + + CRITICAL RULES: + - You ARE a conversational AI - answer questions naturally and helpfully + - Keep responses concise and focused (aim for 100-200 words maximum) + - Be clear, accurate, and friendly + - If the user asks a question, provide a direct answer + - If the user makes a statement, respond appropriately (acknowledge, ask follow-up, or provide relevant information) + - Stay on topic and maintain context from the conversation + - Use simple, easy-to-understand language + - Avoid overly long or complex explanations unless specifically asked + + Response Length Guidelines: + - For simple questions: 1-2 sentences (20-50 words) + - For moderate questions: 2-4 sentences (50-100 words) + - For complex questions: 4-8 sentences (100-200 words maximum) + - NEVER exceed 200 words unless absolutely necessary + + Examples: + User: "What's the weather like today?" + You: "I don't have access to real-time weather data, but you can check your local weather app or website for current conditions in your area." + + User: "Can you explain quantum computing?" + You: "Quantum computing uses quantum bits or 'qubits' that can exist in multiple states simultaneously, unlike regular computer bits that are either 0 or 1. This allows quantum computers to process certain complex calculations much faster than traditional computers. They're particularly useful for cryptography, drug discovery, and optimization problems." + + User: "Hello, how are you?" + You: "Hello! I'm doing well, thank you for asking. I'm here and ready to help you with any questions or tasks you have. How can I assist you today?" + + Return ONLY your response - no extra formatting, quotes, or explanations about what you're doing. + + # Optional: Specify word/character limits for this persona + max_words: 200 + max_chars: 1200 + story_narrator: name: "Story Narrator" description: "Extract and synthesize a cohesive narrative" @@ -172,7 +172,7 @@ class OpenRouterProvider: class AITextProcessor: """ - High-level text processor for proofreading and translation. + High-level text processor for proofreading, translation, and custom personas (like Q&A). """ def __init__( @@ -181,7 +181,8 @@ class AITextProcessor: model_id: Optional[str] = None, mode: str = "proofread_translate", source_lang: Optional[str] = None, - target_lang: Optional[str] = None + target_lang: Optional[str] = None, + persona_id: Optional[str] = None ): """ Initialize AI text processor. @@ -189,34 +190,74 @@ class AITextProcessor: Args: config: AIConfig instance model_id: Model to use (defaults to config default) - mode: 'proofread', 'translate', or 'proofread_translate' + mode: 'proofread', 'translate', 'proofread_translate', or 'custom' source_lang: Source language (None or 'auto' for auto-detect) target_lang: Target language code (None for proofread-only mode) + persona_id: Custom persona ID (for mode='custom') """ self.config = config self.provider = OpenRouterProvider(config, model_id) self.mode = mode self.source_lang = source_lang self.target_lang = target_lang + self.persona_id = persona_id - # Validate mode - if mode not in ['proofread', 'translate', 'proofread_translate']: - raise ValueError(f"Invalid mode: {mode}. Use 'proofread', 'translate', or 'proofread_translate'") + # Word and character limits (for custom personas) + self.max_words = None + self.max_chars = None # Get system prompt - if mode == 'proofread': + if mode == 'custom' and persona_id: + # Use custom persona prompt + self.system_prompt = config.get_persona_prompt(persona_id) + if not self.system_prompt: + raise ValueError(f"Persona not found: {persona_id}") + + # Get limits from persona config if available + if persona_id in config.custom_personas: + persona_config = config.custom_personas[persona_id] + self.max_words = persona_config.get('max_words') + self.max_chars = persona_config.get('max_chars') + elif mode == 'proofread': self.system_prompt = config.format_prompt(mode) - else: + elif mode in ['translate', 'proofread_translate']: self.system_prompt = config.format_prompt(mode, source_lang or 'auto', target_lang) + else: + raise ValueError(f"Invalid mode: {mode}. Use 'proofread', 'translate', 'proofread_translate', or 'custom'") # Get defaults defaults = config.get_defaults() self.temperature = defaults.get('temperature', 0.3) self.max_retries = defaults.get('max_retries', 2) + def _enforce_limits(self, text: str) -> str: + """ + Enforce word and character limits on the output text. + + Args: + text: Text to limit + + Returns: + Limited text + """ + if not text: + return text + + # Apply character limit first if set + if self.max_chars and len(text) > self.max_chars: + text = text[:self.max_chars].rsplit(' ', 1)[0] + '...' + + # Apply word limit if set + if self.max_words: + words = text.split() + if len(words) > self.max_words: + text = ' '.join(words[:self.max_words]) + '...' + + return text + def process(self, text: str) -> Tuple[str, Optional[str]]: """ - Process text (proofread and/or translate). + Process text (proofread, translate, or custom persona processing). Args: text: Text to process @@ -224,13 +265,19 @@ class AITextProcessor: Returns: Tuple of (processed_text, error_message) """ - return self.provider.process_text( + result, error = self.provider.process_text( text=text, system_prompt=self.system_prompt, temperature=self.temperature, max_retries=self.max_retries ) + # Enforce limits for custom personas + if not error and (self.max_words or self.max_chars): + result = self._enforce_limits(result) + + return (result, error) + if __name__ == "__main__": """Test the provider.""" @@ -298,7 +298,8 @@ class ProcessingBridge: return None selected_model_id = models[self.state.ai_model_index]['id'] - # Determine mode + # Determine mode and persona + persona_id = None if self.state.ai_translate_only: mode = "translate" else: @@ -307,7 +308,8 @@ class ProcessingBridge: if self.state.ai_persona_index >= len(personas): mode = "proofread" else: - persona_id = personas[self.state.ai_persona_index]['id'] + persona = personas[self.state.ai_persona_index] + persona_id = persona['id'] if persona_id == "proofread": if self.state.ai_translate and self.state.target_language != "none": @@ -315,11 +317,16 @@ class ProcessingBridge: else: mode = "proofread" else: - # Custom persona - treat as proofread - if self.state.ai_translate and self.state.target_language != "none": - mode = "proofread_translate" + # Custom persona (like Q&A) + if persona.get('builtin'): + # Built-in custom persona + if self.state.ai_translate and self.state.target_language != "none": + mode = "proofread_translate" + else: + mode = "proofread" else: - mode = "proofread" + # User-defined custom persona - use 'custom' mode + mode = "custom" # Get languages source = None if self.state.source_language == "auto" else self.state.source_language @@ -331,7 +338,8 @@ class ProcessingBridge: model_id=selected_model_id, mode=mode, source_lang=source, - target_lang=target + target_lang=target, + persona_id=persona_id if mode == "custom" else None ) return processor @@ -405,12 +413,92 @@ class ProcessingBridge: setattr(self.state, state_attr, preview) if new_segment: - if tts_source and self.state.tts_enabled and self.state.tts_source == tts_source: + # Check if we're in Q&A mode + is_qa_mode = (self.ai_processor and + hasattr(self.ai_processor, 'mode') and + self.ai_processor.mode == 'custom' and + hasattr(self.ai_processor, 'persona_id') and + self.ai_processor.persona_id == 'qa') + + # Handle TTS for AI output in Q&A mode + if is_qa_mode and tts_source == 'ai' and state_attr == 'ai_text': + # In Q&A mode, trigger TTS for complete AI response + if self.state.tts_enabled and self.state.tts_source == 'ai': + self._trigger_qa_tts(new_segment) + + # Clear AI output after response is complete (keep whisper for context) + # We'll do this after TTS is done speaking + + elif tts_source and self.state.tts_enabled and self.state.tts_source == tts_source: + # Normal TTS mode (non-Q&A) self.tts_session_text += new_segment + " " if autotype_mode and self.state.autotype_mode == autotype_mode and new_segment.strip(): self._autotype_text(new_segment) + def _trigger_qa_tts(self, text: str): + """Trigger TTS synthesis for Q&A response and set up playback.""" + if not self.tts_controller or not text.strip(): + return + + try: + import time + # Generate unique filename for this Q&A response + timestamp = time.strftime("%Y%m%d_%H%M%S") + filename = f"qa_response_{timestamp}" + + # Synthesize to file + output_path = self.tts_controller.synthesize_to_file( + text=text.strip(), + output_filename=filename, + file_format=self.state.tts_format, + async_mode=False # Wait for completion + ) + + if output_path: + # Store the audio file path for playback + self.state.tts_audio_file = output_path + self.state.tts_status_message = "Ready to play" + + # Auto-play the audio + self._play_tts_audio(output_path) + except Exception as e: + print(f"Q&A TTS error: {e}") + self.state.tts_status_message = f"TTS error: {str(e)[:30]}" + + def _play_tts_audio(self, audio_path: str): + """Play TTS audio file in background thread and clear AI output when done.""" + def play_audio(): + try: + import sounddevice as sd + import soundfile as sf + + # Read audio file + data, samplerate = sf.read(audio_path) + + # Update state + self.state.tts_is_playing = True + self.state.tts_status_message = "Playing..." + + # Play audio (blocking in this thread) + sd.play(data, samplerate) + sd.wait() + + # Clear AI output after playback in Q&A mode + self.state.ai_text = "" + self._ai_committed = "" + + # Update state + self.state.tts_is_playing = False + self.state.tts_status_message = "Playback complete" + except Exception as e: + print(f"Audio playback error: {e}") + self.state.tts_status_message = f"Playback error: {str(e)[:30]}" + self.state.tts_is_playing = False + + # Start playback in background thread + threading.Thread(target=play_audio, daemon=True).start() + def manual_ai_trigger(self): """Manually trigger AI processing.""" if not self.state.is_recording: @@ -420,6 +508,21 @@ class ProcessingBridge: if self.state.debug_enabled: print("[Bridge] Manual AI processing requested", flush=True) + def replay_qa_audio(self): + """Replay the last Q&A TTS audio.""" + if self.state.tts_audio_file and not self.state.tts_is_playing: + self._play_tts_audio(self.state.tts_audio_file) + + def stop_qa_audio(self): + """Stop Q&A TTS audio playback.""" + try: + import sounddevice as sd + sd.stop() + self.state.tts_is_playing = False + self.state.tts_status_message = "Stopped" + except Exception as e: + print(f"Stop audio error: {e}") + def refresh_mics(self): """Refresh the list of available microphones.""" self.state.mic_list = core.get_mic_names() @@ -8,12 +8,13 @@ from nicegui import ui from whispering_ui.state import AppState -def create_output_panels(state: AppState): +def create_output_panels(state: AppState, bridge=None): """ Create output text panels with Copy/Cut buttons. Args: state: Application state + bridge: Processing bridge (optional, needed for audio controls) Returns: The output container element for visibility control @@ -80,6 +81,26 @@ def create_output_panels(state: AppState): ) ).props('dense flat').classes('text-xs') + # Add audio playback controls for AI panel in Q&A mode + if config['key'] == 'ai' and bridge: + # Play button + play_btn = ui.button( + icon='play_arrow', + on_click=lambda b=bridge: b.replay_qa_audio() + ).props('dense flat round size=sm').classes('text-xs') + play_btn.visible = False # Initially hidden + + # Stop button + stop_btn = ui.button( + icon='stop', + on_click=lambda b=bridge: b.stop_qa_audio() + ).props('dense flat round size=sm').classes('text-xs') + stop_btn.visible = False # Initially hidden + + # Store references for visibility control + config['play_btn'] = play_btn + config['stop_btn'] = stop_btn + textareas[config['key']] = ui.textarea( placeholder=config['placeholder'] ).classes('w-full h-full flex-1 output-textarea').style('font-family: monospace; min-height: 0; height: 100%;').props('outlined readonly') @@ -94,6 +115,17 @@ def create_output_panels(state: AppState): chars, words = cfg['count_fn']() count_labels[cfg['key']].text = f'{chars} chars, {words} words' + # Update audio playback controls visibility for AI panel + if cfg['key'] == 'ai' and 'play_btn' in cfg and 'stop_btn' in cfg: + # Show controls if TTS is enabled, source is AI, and we have an audio file + show_controls = ( + state.tts_enabled and + state.tts_source == 'ai' and + state.tts_audio_file is not None + ) + cfg['play_btn'].visible = show_controls and not state.tts_is_playing + cfg['stop_btn'].visible = show_controls and state.tts_is_playing + ui.timer(0.2, update_outputs) return output_container @@ -267,8 +267,8 @@ def main(): # Sidebar on left - fixed width sidebar_container = create_sidebar(state, bridge, None).classes('sidebar-container') - # Output panels on right - flex grow - output_container = create_output_panels(state).classes('output-container') + # Output panels on right - flex grow (pass bridge for audio controls) + output_container = create_output_panels(state, bridge).classes('output-container') # Connect sidebar to output for show/hide sidebar_container._output_container = output_container @@ -48,6 +48,8 @@ class AppState: tts_voice_reference: Optional[str] = None tts_voice_display_name: str = "Default" # Display name for voice tts_status_message: str = "" # TTS status message + tts_audio_file: Optional[str] = None # Current TTS audio file for playback + tts_is_playing: bool = False # TTS audio playback state # === Autotype Settings === autotype_mode: str = "Off" # "Off", "Whisper", "Translation", "AI"