feat: integrate Kokoro TTS as a third backend option
Add KokoroTTSProvider (82M parameter model from hexgrad/kokoro) alongside existing Chatterbox and Qwen3-TTS backends. Kokoro outputs 24kHz audio, supports 54 voices across 9 languages, and requires only the kokoro pip package plus espeak-ng as a system dependency. Changes: - tts_provider.py: KokoroTTSProvider class, KOKORO_VOICES list, backend detection, factory support - tts_controller.py: kokoro_voice parameter plumbing - state.py: tts_kokoro_voice setting - sidebar.py: Kokoro voice selector in UI, backend switching - main.py: load/save kokoro settings, pass to controller - install.sh: install_kokoro function, --tts=kokoro flag, interactive menu - help.py: updated TTS help text https://claude.ai/code/session_01965m1v3f1cK6gDV1JHioA8
+230 −10 · 7 files changed
-
scripts/install.sh+55 −5 -
src/tts_controller.py+4 −0 -
src/tts_provider.py+140 −2 -
src/whispering_ui/components/help.py+3 −1 -
src/whispering_ui/components/sidebar.py+23 −2 -
src/whispering_ui/main.py+4 −0 -
src/whispering_ui/state.py+1 −0
@@ -35,6 +35,7 @@ for arg in "$@"; do echo " --tts Install TTS backends (interactive selection)" echo " --tts=chatterbox Install Chatterbox TTS backend" echo " --tts=qwen3 Install Qwen3-TTS backend" + echo " --tts=kokoro Install Kokoro TTS backend (lightweight 82M model)" echo " --tts=all Install all TTS backends" echo " -h, --help Show this help message" exit 0 @@ -425,6 +426,45 @@ except ImportError as e: echo } +install_kokoro() { + echo -e "${YELLOW}Installing Kokoro TTS...${NC}" + + # Check for espeak-ng system dependency + if ! command -v espeak-ng &> /dev/null; then + echo -e "${YELLOW}espeak-ng not found. Attempting to install...${NC}" + if command -v apt-get &> /dev/null; then + sudo apt-get install -y espeak-ng 2>/dev/null && { + echo -e "${GREEN}✓ espeak-ng installed${NC}" + } || { + echo -e "${YELLOW}⚠ Could not install espeak-ng automatically.${NC}" + echo -e "${BLUE} Please install manually: sudo apt-get install espeak-ng${NC}" + } + elif command -v pacman &> /dev/null; then + sudo pacman -S --noconfirm espeak-ng 2>/dev/null || true + elif command -v dnf &> /dev/null; then + sudo dnf install -y espeak-ng 2>/dev/null || true + else + echo -e "${YELLOW}⚠ Please install espeak-ng manually for your distribution.${NC}" + fi + else + echo -e "${GREEN}✓ espeak-ng already installed${NC}" + fi + + # Install kokoro and soundfile + $PYTHON_CMD -m pip install kokoro soundfile 2>/dev/null && { + echo -e "${GREEN}✓ Kokoro TTS installed${NC}" + } || { + echo -e "${RED}✗ Failed to install Kokoro TTS${NC}" + echo -e "${BLUE} You can try manually: $PYTHON_CMD -m pip install kokoro soundfile${NC}" + } + + # Verify installation + $PYTHON_CMD -c "from kokoro import KPipeline; print(' ✓ Kokoro import successful')" 2>/dev/null || { + echo -e "${YELLOW} Note: Kokoro import test failed. It may still work at runtime.${NC}" + } + echo +} + # Handle TTS installation if [ "$INSTALL_TTS" = true ]; then echo -e "${BLUE}========================================${NC}" @@ -440,13 +480,17 @@ if [ "$INSTALL_TTS" = true ]; then qwen3) install_qwen3 ;; + kokoro) + install_kokoro + ;; all) install_chatterbox install_qwen3 + install_kokoro ;; *) echo -e "${RED}Unknown TTS backend: $TTS_BACKEND${NC}" - echo -e "${BLUE}Available: chatterbox, qwen3, all${NC}" + echo -e "${BLUE}Available: chatterbox, qwen3, kokoro, all${NC}" ;; esac else @@ -454,10 +498,11 @@ if [ "$INSTALL_TTS" = true ]; then echo -e "${YELLOW}Which TTS backend(s) would you like to install?${NC}" echo " 1) Chatterbox TTS (ResembleAI - voice cloning, English)" echo " 2) Qwen3-TTS (Alibaba - multilingual, 10 languages, multiple voices)" - echo " 3) Both" - echo " 4) Skip TTS installation" + echo " 3) Kokoro TTS (lightweight 82M model, fast, 54 voices, 9 languages)" + echo " 4) All" + echo " 5) Skip TTS installation" echo - read -r -p "Select (1-4): " tts_choice + read -r -p "Select (1-5): " tts_choice case $tts_choice in 1) @@ -467,10 +512,14 @@ if [ "$INSTALL_TTS" = true ]; then install_qwen3 ;; 3) + install_kokoro + ;; + 4) install_chatterbox install_qwen3 + install_kokoro ;; - 4|*) + 5|*) echo -e "${BLUE}Skipping TTS installation.${NC}" echo -e "${BLUE}You can install later with: ./scripts/install.sh --tts${NC}" ;; @@ -511,6 +560,7 @@ echo -e " - For TTS: ${GREEN}./scripts/install.sh --tts${NC}" echo -e " Or install individually:" echo -e " - Chatterbox: ${GREEN}pip install chatterbox-tts --no-deps${NC}" echo -e " - Qwen3-TTS: ${GREEN}pip install qwen-tts${NC}" +echo -e " - Kokoro: ${GREEN}pip install kokoro soundfile${NC} (+ apt install espeak-ng)" echo echo -e "${YELLOW}Note: Remember to activate the virtual environment before running:${NC}" echo -e "${GREEN} source .venv/bin/activate${NC}" @@ -47,6 +47,7 @@ class TTSController: model_type: Literal["standard", "multilingual"] = "standard", qwen3_model_size: str = "1.7B", qwen3_speaker: str = "Ryan", + kokoro_voice: str = "af_heart", ): self.backend = backend self.output_dir = Path(output_dir) @@ -58,6 +59,7 @@ class TTSController: model_type=model_type, qwen3_model_size=qwen3_model_size, qwen3_speaker=qwen3_speaker, + kokoro_voice=kokoro_voice, ) self.reference_voice_path: Optional[str] = None @@ -116,6 +118,7 @@ class TTSController: model_type: str = "standard", qwen3_model_size: str = "1.7B", qwen3_speaker: str = "Ryan", + kokoro_voice: str = "af_heart", ): """Switch to a different TTS backend. Unloads the current provider.""" if self.provider: @@ -128,6 +131,7 @@ class TTSController: model_type=model_type, qwen3_model_size=qwen3_model_size, qwen3_speaker=qwen3_speaker, + kokoro_voice=kokoro_voice, ) def chunk_text(self, text: str) -> list[str]: @@ -2,7 +2,7 @@ """ TTS Provider module with multi-backend support. -Supports ChatterboxTTS and Qwen3-TTS with graceful fallback. +Supports ChatterboxTTS, Qwen3-TTS, and Kokoro with graceful fallback. """ import builtins @@ -129,6 +129,13 @@ def get_available_backends() -> dict[str, bool]: except (ImportError, ModuleNotFoundError, Exception): backends["qwen3"] = False + # Check Kokoro TTS + try: + from kokoro import KPipeline # noqa: F401 + backends["kokoro"] = True + except (ImportError, ModuleNotFoundError, Exception): + backends["kokoro"] = False + return backends @@ -603,6 +610,129 @@ class Qwen3TTSProvider(BaseTTSProvider): torch.cuda.empty_cache() +# --------------------------------------------------------------------------- +# Kokoro TTS provider +# --------------------------------------------------------------------------- + +KOKORO_VOICES = [ + # American English - Female (best: af_heart) + "af_heart", "af_bella", "af_nicole", "af_alloy", "af_aoede", + "af_kore", "af_sarah", "af_nova", "af_jessica", "af_river", "af_sky", + # American English - Male + "am_fenrir", "am_michael", "am_puck", "am_echo", "am_eric", + "am_liam", "am_onyx", "am_adam", + # British English - Female (best: bf_emma) + "bf_emma", "bf_isabella", "bf_alice", "bf_lily", + # British English - Male + "bm_fable", "bm_george", "bm_daniel", "bm_lewis", +] + + +class KokoroTTSProvider(BaseTTSProvider): + """Kokoro TTS provider — lightweight 82M parameter model. + + Uses the ``kokoro`` package (``KPipeline``) for synthesis. + Outputs 24 kHz audio. Requires ``espeak-ng`` system package. + """ + + def __init__( + self, + device: Literal["cpu", "cuda", "auto"] = "auto", + default_voice: str = "af_heart", + ): + super().__init__(device) + self.sample_rate = 24000 + self.default_voice = default_voice + self._pipeline = None + self._current_lang: str = "" + + @staticmethod + def _voice_to_lang(voice: str) -> str: + """Derive Kokoro lang_code from the voice ID prefix.""" + if voice and len(voice) >= 1: + return {"a": "a", "b": "b", "j": "j", "z": "z", + "e": "e", "f": "f", "h": "h", "i": "i", + "p": "p"}.get(voice[0], "a") + return "a" + + def _ensure_initialized(self): + if self._initialized: + return + + try: + from kokoro import KPipeline + + tts_timer_reset() + tts_log(f"Loading Kokoro TTS on {self.device}...") + + lang_code = self._voice_to_lang(self.default_voice) + self._pipeline = KPipeline(lang_code=lang_code) + self._current_lang = lang_code + self._initialized = True + tts_log("Kokoro TTS loaded") + + except ImportError as e: + raise ImportError( + "Kokoro TTS is not installed.\n" + "Install with: pip install kokoro soundfile\n" + "System dependency: apt-get install espeak-ng\n" + "Or run: ./scripts/install.sh --tts=kokoro" + ) from e + except Exception as e: + raise RuntimeError(f"Failed to load Kokoro TTS: {e}") from e + + def synthesize( + self, + text: str, + reference_audio_path: Optional[str] = None, + language: str = "en", + exaggeration: float = 0.5, + cfg: float = 0.5, + speaker: str = "", + ) -> tuple[np.ndarray, int]: + if not text.strip(): + raise ValueError("Cannot synthesize empty text") + + self._ensure_initialized() + + voice = speaker if speaker else self.default_voice + + # Recreate pipeline if the voice language changed + lang_code = self._voice_to_lang(voice) + if lang_code != self._current_lang: + from kokoro import KPipeline + self._pipeline = KPipeline(lang_code=lang_code) + self._current_lang = lang_code + + tts_log(f"Kokoro synthesize (voice={voice}): " + f"{len(text)} chars \"{text[:60]}...\"") + + audio_segments = [] + for gs, ps, audio in self._pipeline(text, voice=voice): + if audio is not None and len(audio) > 0: + audio_segments.append(audio) + + if not audio_segments: + raise RuntimeError("Kokoro produced no audio output") + + full_audio = (np.concatenate(audio_segments) + if len(audio_segments) > 1 else audio_segments[0]) + if isinstance(full_audio, torch.Tensor): + full_audio = full_audio.cpu().numpy() + if len(full_audio.shape) > 1: + full_audio = full_audio.flatten() + + dur = len(full_audio) / self.sample_rate + tts_log(f"Kokoro synthesis done: {len(full_audio)} samples, {dur:.1f}s audio") + return full_audio, self.sample_rate + + def unload(self): + if self._pipeline is not None: + del self._pipeline + self._pipeline = None + super().unload() + + # --------------------------------------------------------------------------- # Factory # --------------------------------------------------------------------------- @@ -613,16 +743,18 @@ def create_provider( model_type: str = "standard", qwen3_model_size: str = "1.7B", qwen3_speaker: str = "Ryan", + kokoro_voice: str = "af_heart", ) -> BaseTTSProvider: """ Create a TTS provider for the given backend. Args: - backend: "chatterbox" or "qwen3" + backend: "chatterbox", "qwen3", or "kokoro" device: Compute device model_type: Chatterbox model type ("standard" or "multilingual") qwen3_model_size: Qwen3 model size ("0.6B" or "1.7B") qwen3_speaker: Default Qwen3 speaker name + kokoro_voice: Default Kokoro voice ID (e.g. "af_heart") Returns: TTS provider instance @@ -633,6 +765,11 @@ def create_provider( model_size=qwen3_model_size, default_speaker=qwen3_speaker, ) + elif backend == "kokoro": + return KokoroTTSProvider( + device=device, + default_voice=kokoro_voice, + ) else: return ChatterboxProvider(device=device, model_type=model_type) @@ -663,3 +800,4 @@ if __name__ == "__main__": print("\nNo TTS backends installed. Install one with:") print(" pip install chatterbox-tts --no-deps") print(" pip install qwen-tts") + print(" pip install kokoro soundfile") @@ -59,10 +59,12 @@ See AI_SETUP.md for details.""", "tts": """**Enable TTS:** Convert text to speech -**Engine:** Chatterbox (English, voice cloning) | Qwen3-TTS (multilingual, 9 voices) +**Engine:** Chatterbox (voice cloning) | Qwen3-TTS (multilingual, 9 voices) | Kokoro (fast, 82M, 54 voices) **Qwen3 Options:** Speaker (Ryan, Aiden, Vivian, etc.) | Size (0.6B lighter, 1.7B better) +**Kokoro Options:** Voice (af_heart, am_michael, bf_emma, etc.) — prefix: a=US, b=UK, f/m=gender + **Voice:** Browse=upload ref audio for cloning | Clear=default voice **Play:** Hear TTS output through speakers in real time @@ -562,7 +562,7 @@ def create_sidebar(state: AppState, bridge: ProcessingBridge, output_container=N ui.label('Engine:').classes('text-xs w-10') # Build backend options with availability labels backend_options = {} - for bname in ["chatterbox", "qwen3"]: + for bname in ["chatterbox", "qwen3", "kokoro"]: installed = state.tts_backends_available.get(bname, False) if installed: backend_options[bname] = bname @@ -588,10 +588,12 @@ def create_sidebar(state: AppState, bridge: ProcessingBridge, output_container=N backend=selected, qwen3_model_size=state.tts_qwen3_model_size, qwen3_speaker=state.tts_qwen3_speaker, + kokoro_voice=state.tts_kokoro_voice, ) state.tts_status_message = "" - # Show/hide Qwen3 options + # Show/hide backend-specific options qwen3_row.set_visibility(selected == "qwen3") + kokoro_row.set_visibility(selected == "kokoro") tts_backend_select.on_value_change(on_backend_change) @@ -636,6 +638,25 @@ def create_sidebar(state: AppState, bridge: ProcessingBridge, output_container=N qwen3_size_select.on_value_change(on_size_change) + # Kokoro-specific options (voice selection) + kokoro_row = ui.row().classes('items-center w-full gap-1') + kokoro_row.set_visibility(state.tts_backend == "kokoro") + + with kokoro_row: + ui.label('Voice:').classes('text-xs') + from tts_provider import KOKORO_VOICES + kokoro_voice_select = register_tts(ui.select( + options=KOKORO_VOICES, + value=state.tts_kokoro_voice + ).classes('flex-grow').props('dense')) + + def on_kokoro_voice_change(e): + state.tts_kokoro_voice = e.value + if bridge.tts_controller: + bridge.tts_controller.set_parameters(speaker=e.value) + + kokoro_voice_select.on_value_change(on_kokoro_voice_change) + # Source selection - compact with ui.row().classes('items-center w-full gap-1'): ui.label('Src:').classes('text-xs w-10') @@ -107,6 +107,7 @@ def main(): state.tts_format = settings.get("tts_format", "wav") state.tts_qwen3_speaker = settings.get("tts_qwen3_speaker", "Ryan") state.tts_qwen3_model_size = settings.get("tts_qwen3_model_size", "1.7B") + state.tts_kokoro_voice = settings.get("tts_kokoro_voice", "af_heart") # If saved backend is not installed, fall back to first available if not backends.get(state.tts_backend): @@ -128,6 +129,7 @@ def main(): print(" Install with: ./scripts/install.sh --tts") print(" Or manually: pip install chatterbox-tts --no-deps") print(" pip install qwen-tts") + print(" pip install kokoro soundfile") except Exception as e: print(f"TTS features not available: {e}") state.tts_available = False @@ -146,6 +148,7 @@ def main(): backend=state.tts_backend, qwen3_model_size=state.tts_qwen3_model_size, qwen3_speaker=state.tts_qwen3_speaker, + kokoro_voice=state.tts_kokoro_voice, ) # Wire up status callbacks bridge.tts_controller.on_progress = lambda msg: setattr(state, 'tts_status_message', msg) @@ -405,6 +408,7 @@ def main(): settings.set("tts_format", state.tts_format) settings.set("tts_qwen3_speaker", state.tts_qwen3_speaker) settings.set("tts_qwen3_model_size", state.tts_qwen3_model_size) + settings.set("tts_kokoro_voice", state.tts_kokoro_voice) settings.save() @@ -54,6 +54,7 @@ class AppState: tts_is_playing: bool = False # TTS audio playback state tts_qwen3_speaker: str = "Ryan" # Qwen3-TTS speaker name tts_qwen3_model_size: str = "1.7B" # Qwen3 model size ("0.6B" or "1.7B") + tts_kokoro_voice: str = "af_heart" # Kokoro voice ID tts_backends_available: dict = field(default_factory=dict) # {name: bool} # === Autotype Settings ===