feat: Add speaker monitor capture, fix text persistence, log recovery, and copy crash
- Add speaker monitor capture with left/right channel selection - New get_monitor_names() to detect PulseAudio/PipeWire monitor devices - Channel selection: mix (default), left, or right for stereo sources - UI controls in sidebar to enable speaker capture and select channel - Fix text clearing behavior - Text now persists after stopping (previously vanished on stop) - Text clears when starting a NEW transcription session - Separated save and clear logic in bridge - Fix log recovery to load recovered text into UI - Added load_session_outputs() to session_logger - Recovery dialog now restores whisper/AI/translation text to UI - Fix copy crash during transcription - Made clipboard operations thread-safe with proper value capture - Using async JavaScript clipboard API with error handling - Added try/catch for textarea updates during active recording
+262 −51 · 9 files changed
-
src/core.py+3 −1 -
src/core_parts/audio_devices.py+68 −10 -
src/core_parts/processing.py+3 −2 -
src/session_logger.py+61 −0 -
src/whispering_ui/bridge.py+32 −17 -
src/whispering_ui/components/output.py+47 −17 -
src/whispering_ui/components/sidebar.py+25 −0 -
src/whispering_ui/main.py+17 −4 -
src/whispering_ui/state.py+6 −0
@@ -3,8 +3,10 @@ from core_parts.paragraph_detector import ParagraphDetector from core_parts.audio_devices import ( TARGET_SAMPLE_RATE, SAMPLE_WIDTH, CHUNK_DURATION, + CHANNEL_MIX, CHANNEL_LEFT, CHANNEL_RIGHT, get_preferred_hostapi_index, get_mic_names, get_default_device_index, - get_mic_index, get_device_info, audio_to_wav_bytes, resample_to_mono_16k + get_mic_index, get_device_info, audio_to_wav_bytes, resample_to_mono_16k, + get_monitor_names, get_all_input_devices ) from core_parts.processing import ( translate, parse_ai_proofread_translate, ai_translate, proc @@ -8,6 +8,11 @@ TARGET_SAMPLE_RATE = 16000 SAMPLE_WIDTH = 2 # 16-bit audio CHUNK_DURATION = 0.1 # seconds per chunk +# Channel selection options +CHANNEL_MIX = "mix" # Mix all channels to mono (default) +CHANNEL_LEFT = "left" # Use only left channel +CHANNEL_RIGHT = "right" # Use only right channel + def get_preferred_hostapi_index(): """Find the best host API: ALSA only (JACK causes crashes).""" @@ -21,9 +26,9 @@ def get_preferred_hostapi_index(): def get_mic_names(): """Get list of input device names, preferring stable devices.""" devices_list = sd.query_devices() - + mics = [] - + # First priority: ALSA virtual devices named "pipewire" or "pulse" (these route through PipeWire) for i, d in enumerate(devices_list): if d['max_input_channels'] > 0: @@ -34,7 +39,7 @@ def get_mic_names(): continue if 'pipewire' in name_lower or name_lower == 'pulse': mics.append((i, d['name'])) - + # Second priority: Simple ALSA hardware devices (limited channels, not default) for i, d in enumerate(devices_list): if d['max_input_channels'] > 0 and d['max_input_channels'] <= 8: @@ -50,10 +55,46 @@ def get_mic_names(): if 'hw:' in d['name']: # Real hardware if (i, d['name']) not in mics: mics.append((i, d['name'])) - + return mics +def get_monitor_names(): + """Get list of speaker monitor device names (for capturing system audio output). + + Monitor devices capture what's being played through speakers/headphones. + On PulseAudio/PipeWire, these typically have 'Monitor' in the name. + """ + devices_list = sd.query_devices() + + monitors = [] + + for i, d in enumerate(devices_list): + if d['max_input_channels'] > 0: + name_lower = d['name'].lower() + # Only ALSA devices (JACK crashes) + api_name = sd.query_hostapis(d['hostapi'])['name'].lower() + if 'alsa' not in api_name: + continue + # Look for monitor devices (PulseAudio/PipeWire convention) + if 'monitor' in name_lower: + monitors.append((i, d['name'])) + + return monitors + + +def get_all_input_devices(): + """Get both microphones and monitor devices, categorized. + + Returns: + dict with 'mics' and 'monitors' keys, each containing list of (index, name) tuples + """ + return { + 'mics': get_mic_names(), + 'monitors': get_monitor_names() + } + + def get_default_device_index(): """Get the best default device (pipewire or pulse, not the actual 'default').""" devices_list = sd.query_devices() @@ -123,17 +164,34 @@ def audio_to_wav_bytes(audio_data, sample_rate, sample_width, channels=1): return buffer -def resample_to_mono_16k(data, orig_rate, orig_channels): - """Convert audio to mono 16kHz for Whisper.""" +def resample_to_mono_16k(data, orig_rate, orig_channels, channel_select=CHANNEL_MIX): + """Convert audio to mono 16kHz for Whisper. + + Args: + data: Raw audio data (numpy array or bytes) + orig_rate: Original sample rate + orig_channels: Number of channels in original audio + channel_select: Which channel to use - CHANNEL_MIX (average all), + CHANNEL_LEFT (left only), or CHANNEL_RIGHT (right only) + """ # Ensure we have a copy to avoid memory issues audio = np.array(data, dtype=np.float32, copy=True) / 32768.0 - + # Convert to mono if stereo/multi-channel if orig_channels > 1 and len(audio.shape) > 1: - audio = audio.mean(axis=1) + if channel_select == CHANNEL_LEFT: + # Use only left channel (index 0) + audio = audio[:, 0] + elif channel_select == CHANNEL_RIGHT: + # Use only right channel (index 1, or last channel if mono) + channel_idx = min(1, audio.shape[1] - 1) + audio = audio[:, channel_idx] + else: + # Default: mix all channels + audio = audio.mean(axis=1) else: audio = audio.flatten() - + # Resample if needed if orig_rate != TARGET_SAMPLE_RATE: # Simple resampling using linear interpolation @@ -142,7 +200,7 @@ def resample_to_mono_16k(data, orig_rate, orig_channels): if new_length > 0: indices = np.linspace(0, len(audio) - 1, new_length) audio = np.interp(indices, np.arange(len(audio)), audio) - + # Convert back to int16 audio = (audio * 32768.0).clip(-32768, 32767).astype(np.int16) return audio.tobytes() @@ -20,6 +20,7 @@ except ImportError: from .paragraph_detector import ParagraphDetector from .audio_devices import ( TARGET_SAMPLE_RATE, SAMPLE_WIDTH, CHUNK_DURATION, + CHANNEL_MIX, CHANNEL_LEFT, CHANNEL_RIGHT, get_default_device_index, get_device_info, audio_to_wav_bytes, resample_to_mono_16k ) @@ -107,7 +108,7 @@ def ai_translate(text, ai_processor): return (text, f"AI processing error: {str(e)}") -def proc(index, model, vad, memory, patience, timeout, prompt, source, target, tsres_queue, tlres_queue, ready, device="cpu", error=None, level=None, para_detect=True, para_threshold_std=1.5, para_min_pause=0.8, para_max_chars=500, para_max_words=100, ai_processor=None, ai_process_interval=2, ai_process_words=None, ai_trigger_mode="time", silence_timeout=60, prres_queue=None, auto_stop_enabled=False, auto_stop_minutes=5, manual_trigger=None, use_google_translate=True): +def proc(index, model, vad, memory, patience, timeout, prompt, source, target, tsres_queue, tlres_queue, ready, device="cpu", error=None, level=None, para_detect=True, para_threshold_std=1.5, para_min_pause=0.8, para_max_chars=500, para_max_words=100, ai_processor=None, ai_process_interval=2, ai_process_words=None, ai_trigger_mode="time", silence_timeout=60, prres_queue=None, auto_stop_enabled=False, auto_stop_minutes=5, manual_trigger=None, use_google_translate=True, channel_select=CHANNEL_MIX): # Create paragraph detector if enabled para_detector = ParagraphDetector( threshold_std=para_threshold_std, @@ -462,7 +463,7 @@ def proc(index, model, vad, memory, patience, timeout, prompt, source, target, t level[0] = min(100, int(rms / 328 * 100)) # Convert to mono 16kHz for Whisper - mono_16k = resample_to_mono_16k(data_copy, sample_rate, channels) + mono_16k = resample_to_mono_16k(data_copy, sample_rate, channels, channel_select) frame_queue.put(mono_16k) frame_queue.put(None) @@ -306,6 +306,67 @@ class SessionLogger: return str(self.current_temp_file.name.replace(".temp_", "")) return None + def load_session_outputs(self, log_file: Path) -> Optional[Dict[str, str]]: + """ + Load the text outputs from a log file (temp or finalized). + + Args: + log_file: Path to the log file + + Returns: + Dictionary with whisper_text, ai_text, translation_text or None if failed + """ + try: + if not log_file.exists(): + return None + + outputs = { + "whisper_text": "", + "ai_text": "", + "translation_text": "" + } + + # Read the JSONL file and find the latest outputs entry + with open(log_file, 'r', encoding='utf-8') as f: + content = f.read() + + # Parse each JSON object (they're separated by newlines but may be multi-line) + # Use a simple approach: find all complete JSON objects + import re + # Match JSON objects that start with { and have proper structure + entries = [] + depth = 0 + current_obj = "" + for char in content: + if char == '{': + depth += 1 + if depth > 0: + current_obj += char + if char == '}': + depth -= 1 + if depth == 0 and current_obj: + try: + entries.append(json.loads(current_obj)) + except json.JSONDecodeError: + pass + current_obj = "" + + # Find the latest entry with non-empty outputs + for entry in reversed(entries): + if entry.get("outputs"): + entry_outputs = entry["outputs"] + if entry_outputs.get("whisper_text") or entry_outputs.get("ai_text") or entry_outputs.get("translation_text"): + outputs["whisper_text"] = entry_outputs.get("whisper_text", "") + outputs["ai_text"] = entry_outputs.get("ai_text", "") + outputs["translation_text"] = entry_outputs.get("translation_text", "") + break + + return outputs + + except Exception as e: + print(f"Error loading session outputs from {log_file}: {e}") + return None + if __name__ == "__main__": """Test the session logger.""" @@ -69,19 +69,14 @@ class ProcessingBridge: if not self._validate_settings(): return - # Reset state (but keep existing text for persistence) + # Reset state self.ready[0] = False self.error[0] = None self.state.error_message = None self.state.status_message = "Starting..." self.level[0] = 0 - # Don't clear text buffers - keep outputs persistent between start/stop - # self.state.whisper_text = "" - # self.state.translation_text = "" - # self.state.ai_text = "" - # self._whisper_committed = "" - # self._translation_committed = "" - # self._ai_committed = "" + # Clear text buffers for new transcription session + self.clear_outputs() self._stream_live = False self._stop_requested = False self._auto_stopped = False @@ -93,12 +88,26 @@ class ProcessingBridge: # Initialize AI processor if enabled self.ai_processor = self._create_ai_processor() - # Get mic index - if self.state.mic_index == 0: + # Get device index (mic or monitor based on mode) + if self.state.monitor_enabled and self.state.monitor_list: + # Use monitor device for speaker capture + if self.state.monitor_index == 0: + # First monitor in list + mic_index = self.state.monitor_list[0][0] + else: + mic_index = self.state.monitor_list[self.state.monitor_index - 1][0] + elif self.state.mic_index == 0: mic_index = core.get_default_device_index() else: mic_index = self.state.mic_list[self.state.mic_index - 1][0] + # Get channel selection constant + channel_select = core.CHANNEL_MIX + if self.state.channel_select == "left": + channel_select = core.CHANNEL_LEFT + elif self.state.channel_select == "right": + channel_select = core.CHANNEL_RIGHT + # Get translation target (None if "none") target = None if self.state.target_language == "none" else self.state.target_language source = None if self.state.source_language == "auto" else self.state.source_language @@ -162,7 +171,8 @@ class ProcessingBridge: 'auto_stop_enabled': self.state.auto_stop_enabled, 'auto_stop_minutes': self.state.auto_stop_minutes, 'manual_trigger': self.manual_trigger_requested, - 'use_google_translate': use_google_translate + 'use_google_translate': use_google_translate, + 'channel_select': channel_select }, daemon=True ).start() @@ -211,8 +221,8 @@ class ProcessingBridge: self.session_logger.finalize_session(stop_reason) self.state.current_log_request_id = None - # Write outputs to log and clear text windows on stop/auto-stop - self._flush_and_clear_outputs() + # Save outputs to log (but don't clear - text persists until next start) + self._save_outputs_to_log() # Update state self.state.is_recording = False @@ -568,6 +578,10 @@ class ProcessingBridge: """Refresh the list of available microphones.""" self.state.mic_list = core.get_mic_names() + def refresh_monitors(self): + """Refresh the list of available monitor (speaker capture) devices.""" + self.state.monitor_list = core.get_monitor_names() + def _get_config_for_logging(self) -> dict: """Get current configuration for logging purposes.""" config = { @@ -602,9 +616,8 @@ class ProcessingBridge: return config - def _flush_and_clear_outputs(self): - """Write current outputs to log and clear the three text windows.""" - # If logging is enabled, ensure current outputs are saved + def _save_outputs_to_log(self): + """Write current outputs to log (if logging enabled).""" if self.state.log_enabled: outputs = { "whisper_text": self.state.whisper_text, @@ -620,7 +633,9 @@ class ProcessingBridge: temp_request_id = self.session_logger.start_session(config) self.session_logger.update_session(outputs) self.session_logger.finalize_session("manual") - # Clear the three output buffers + + def clear_outputs(self): + """Clear the three text output buffers. Call this when starting a new transcription.""" self.state.whisper_text = "" self.state.ai_text = "" self.state.translation_text = "" @@ -141,30 +141,46 @@ def create_output_panels(state: AppState, bridge=None): def _copy_text(text: str, label: str): - """Copy text to clipboard.""" - if not text.strip(): + """Copy text to clipboard (thread-safe).""" + # Capture the text value immediately to avoid race conditions + text_to_copy = str(text) if text else "" + + if not text_to_copy.strip(): ui.notify(f'No {label} text to copy', type='warning') return try: # Use JavaScript clipboard API via ui.run_javascript import json - escaped_text = json.dumps(text) - ui.run_javascript(f'navigator.clipboard.writeText({escaped_text})') + # Escape the text safely for JavaScript + escaped_text = json.dumps(text_to_copy) + # Use async clipboard API with proper error handling + js_code = f''' + (async () => {{ + try {{ + await navigator.clipboard.writeText({escaped_text}); + }} catch (err) {{ + console.error('Clipboard write failed:', err); + }} + }})(); + ''' + ui.run_javascript(js_code) ui.notify(f'✓ Copied {label} text to clipboard', type='positive') except Exception as e: - ui.notify(f'Clipboard error: {e}', type='negative') + # Log error but don't crash + print(f"Clipboard error: {e}") + ui.notify(f'Clipboard error - try again', type='warning') def _cut_text(state: AppState, text_type: str, textarea): - """Cut text (copy and clear).""" - # Get the text + """Cut text (copy and clear) - thread-safe.""" + # Capture the text value immediately to avoid race conditions if text_type == 'whisper': - text = state.whisper_text + text = str(state.whisper_text) if state.whisper_text else "" elif text_type == 'ai': - text = state.ai_text + text = str(state.ai_text) if state.ai_text else "" elif text_type == 'translation': - text = state.translation_text + text = str(state.translation_text) if state.translation_text else "" else: return @@ -173,12 +189,21 @@ def _cut_text(state: AppState, text_type: str, textarea): return try: - # Copy to clipboard + # Copy to clipboard with async API import json escaped_text = json.dumps(text) - ui.run_javascript(f'navigator.clipboard.writeText({escaped_text})') - - # Clear the text + js_code = f''' + (async () => {{ + try {{ + await navigator.clipboard.writeText({escaped_text}); + }} catch (err) {{ + console.error('Clipboard write failed:', err); + }} + }})(); + ''' + ui.run_javascript(js_code) + + # Clear the text in state if text_type == 'whisper': state.whisper_text = "" elif text_type == 'ai': @@ -186,9 +211,14 @@ def _cut_text(state: AppState, text_type: str, textarea): elif text_type == 'translation': state.translation_text = "" - # Update textarea - textarea.value = "" + # Update textarea (wrap in try to prevent crash if textarea is being updated) + try: + textarea.value = "" + except Exception: + pass # Ignore if textarea is busy ui.notify(f'✓ Cut text to clipboard', type='positive') except Exception as e: - ui.notify(f'Clipboard error: {e}', type='negative') + # Log error but don't crash + print(f"Cut error: {e}") + ui.notify(f'Cut error - try again', type='warning') @@ -39,11 +39,36 @@ def create_sidebar(state: AppState, bridge: ProcessingBridge, output_container=N def refresh_mics(): bridge.refresh_mics() + bridge.refresh_monitors() mic_select.options = ["(system default)"] + [name for idx, name in state.mic_list] mic_select.update() + monitor_select.options = ["(first available)"] + [name for idx, name in state.monitor_list] + monitor_select.update() ui.button(icon='refresh', on_click=refresh_mics).props('flat dense round size=sm') + # === SPEAKER CAPTURE (MONITOR) SECTION === + with ui.row().classes('items-center w-full gap-1'): + monitor_cb = ui.checkbox('Speaker', value=state.monitor_enabled).props('dense') + monitor_cb.on_value_change(lambda e: setattr(state, 'monitor_enabled', e.value)) + + monitor_display = ["(first available)"] + [name for idx, name in state.monitor_list] + monitor_select = ui.select( + options=monitor_display, + value=monitor_display[0] if monitor_display else None + ).classes('flex-grow').props('dense') + monitor_select.on_value_change(lambda e: setattr(state, 'monitor_index', + monitor_select.options.index(e.value) if e.value in monitor_select.options else 0)) + + # Channel selection + with ui.row().classes('items-center w-full gap-1'): + ui.label('Channel:').classes('text-xs w-14') + channel_select = ui.select( + options=["mix", "left", "right"], + value=state.channel_select + ).classes('flex-grow').props('dense') + channel_select.on_value_change(lambda e: setattr(state, 'channel_select', e.value)) + # === CONTROL SECTION (MOVED UP) === control_btn = ui.button( 'Start', @@ -56,8 +56,9 @@ def main(): state.auto_stop_enabled = settings.get("auto_stop_enabled", False) state.auto_stop_minutes = settings.get("auto_stop_minutes", 5) - # Initialize microphone list + # Initialize microphone and monitor lists state.mic_list = core.get_mic_names() + state.monitor_list = core.get_monitor_names() # Check AI availability try: @@ -133,12 +134,24 @@ def main(): print(f"Error checking crashed sessions: {e}") def recover_session(temp_file, logger, dialog): - """Recover a crashed session.""" + """Recover a crashed session and load text into UI.""" try: + # First, load the outputs from the temp file before renaming + outputs = logger.load_session_outputs(temp_file) + + # Now recover (rename) the session final_file = logger.recover_session(temp_file) if final_file: - ui.notify('✓ Session recovered successfully', type='positive') - print(f"Recovered session: {final_file}") + # Load recovered text into UI state + if outputs: + state.whisper_text = outputs.get("whisper_text", "") + state.ai_text = outputs.get("ai_text", "") + state.translation_text = outputs.get("translation_text", "") + ui.notify('✓ Session recovered and text restored', type='positive') + print(f"Recovered session with text: {final_file}") + else: + ui.notify('✓ Session recovered (no text content)', type='positive') + print(f"Recovered session (empty): {final_file}") else: ui.notify('✗ Failed to recover session', type='negative') except Exception as e: @@ -16,6 +16,12 @@ class AppState: mic_index: int = 0 mic_list: List[Tuple[int, str]] = field(default_factory=list) + # === Monitor (Speaker Capture) Settings === + monitor_enabled: bool = False # Whether to capture speaker output instead of mic + monitor_index: int = 0 + monitor_list: List[Tuple[int, str]] = field(default_factory=list) + channel_select: str = "mix" # "mix", "left", or "right" + # === Model Settings === model: str = "large-v3" vad_enabled: bool = True