fix: make flash-attn required, install via pre-built wheels
flash-attn is REQUIRED for Qwen3-TTS, not optional. The install script now auto-detects Python/PyTorch/CUDA versions and downloads a matching pre-built wheel from github.com/mjun0812/flash-attention-prebuild-wheels before falling back to building from source. This eliminates the compilation failure that blocked installation. Also: - GPU CUDA libraries auto-detected and installed (no interactive prompt) - qwen_tts flash-attn warning suppressed during import (stdout+stderr) - Backend detection requires both qwen_tts AND flash_attn for qwen3 - Pre-built wheel sources documented in INSTALL_TTS.md https://claude.ai/code/session_01965m1v3f1cK6gDV1JHioA8
+195 −80 · 5 files changed
-
INSTALL_TTS.md+16 −13 -
requirements.txt+3 −2 -
scripts/install.sh+130 −30 -
src/tts_provider.py+45 −31 -
src/whispering_ui/main.py+1 −4
@@ -29,17 +29,20 @@ source .venv/bin/activate # Step 1: Install Qwen3-TTS pip install qwen-tts -# Step 2 (optional but recommended): Install flash-attn for faster inference -# --no-build-isolation is needed because flash-attn needs torch at build time -MAX_JOBS=4 pip install flash-attn --no-build-isolation - +# Step 2: Install flash-attn (REQUIRED) +# The install script auto-detects your system and downloads a pre-built wheel. +# If installing manually, try a pre-built wheel first (instant, no compiler): +# Visit https://github.com/mjun0812/flash-attention-prebuild-wheels/releases +# Download the wheel matching your Python/PyTorch/CUDA versions, then: +# pip install ./flash_attn-X.X.X+cuXXXtorchX.X-cpXXX-cpXXX-linux_x86_64.whl +# +# Or build from source (slow, needs CUDA toolkit + compiler): +# MAX_JOBS=4 pip install flash-attn --no-build-isolation +# # NOTE: Do NOT use "uv pip install flash-attn" - it will fail. -# Use regular pip with --no-build-isolation instead. -# If flash-attn fails to build, Qwen3-TTS will still work using -# eager attention (slower but fully functional). # Verify -python -c "from qwen_tts import Qwen3TTSModel; print('Qwen3-TTS OK')" +python -c "from qwen_tts import Qwen3TTSModel; import flash_attn; print('Qwen3-TTS OK')" ``` ### Option B: Chatterbox TTS (ResembleAI) @@ -94,11 +97,11 @@ If you see `ModuleNotFoundError: No module named 'distutils'`: ### flash-attn build failure (Qwen3-TTS) -If `pip install flash-attn --no-build-isolation` fails to compile: -- **This is not critical** - Qwen3-TTS will fall back to eager attention (slower but works) -- flash-attn requires: CUDA toolkit headers, a compatible GPU, and torch already installed -- The app will print a note at startup if flash-attn is missing -- To retry later: `MAX_JOBS=4 pip install flash-attn --no-build-isolation` +If building flash-attn from source fails, **use a pre-built wheel instead**: + +1. **Auto-install** (recommended): `./scripts/install.sh --tts=qwen3` auto-detects your Python/PyTorch/CUDA and downloads the right pre-built wheel +2. **Manual wheel**: Visit https://github.com/mjun0812/flash-attention-prebuild-wheels/releases and download the `.whl` matching your system, then `pip install ./flash_attn-....whl` +3. **Wheel finder**: Visit https://flashattn.dev to find the exact wheel for your configuration ### No audio output @@ -27,8 +27,9 @@ resemble-perth>=1.0.1 conformer>=0.3.2 # Qwen3-TTS: pip install qwen-tts (installs its own deps) -# flash-attn is required for Qwen3-TTS. Install AFTER torch: -# pip install flash-attn --no-build-isolation +# flash-attn is REQUIRED for Qwen3-TTS. Use install.sh for auto pre-built wheel: +# ./scripts/install.sh --tts=qwen3 +# Or manually: pip install flash-attn --no-build-isolation # Optional: for auto-type feature pyautogui @@ -127,23 +127,26 @@ $PYTHON_CMD -m pip install -r requirements.txt echo -e "${GREEN}✓ Base requirements installed${NC}" echo -# Check for NVIDIA GPU +# Check for NVIDIA GPU and auto-install CUDA libraries if needed echo -e "${YELLOW}Checking for NVIDIA GPU...${NC}" +HAS_GPU=false if command -v nvidia-smi &> /dev/null; then echo -e "${GREEN}✓ NVIDIA GPU detected${NC}" nvidia-smi --query-gpu=name,driver_version,memory.total --format=csv,noheader + HAS_GPU=true echo - # Ask about CUDA libraries - echo -e "${YELLOW}Do you want to install CUDA libraries for GPU acceleration? (y/N)${NC}" - read -r response - if [[ "$response" =~ ^([yY][eE][sS]|[yY])$ ]]; then - echo -e "${YELLOW}Installing CUDA libraries...${NC}" - $PYTHON_CMD -m pip install nvidia-cublas-cu12 nvidia-cudnn-cu12 - echo -e "${GREEN}✓ CUDA libraries installed${NC}" + # Auto-install CUDA libraries - check if already present first + if $PYTHON_CMD -c "import nvidia.cublas" 2>/dev/null && $PYTHON_CMD -c "import nvidia.cudnn" 2>/dev/null; then + echo -e "${GREEN}✓ CUDA libraries already installed${NC}" else - echo -e "${BLUE}Skipping CUDA libraries. You can install them later with:${NC}" - echo -e "${BLUE} pip install nvidia-cublas-cu12 nvidia-cudnn-cu12${NC}" + echo -e "${YELLOW}Installing CUDA libraries for GPU acceleration...${NC}" + $PYTHON_CMD -m pip install nvidia-cublas-cu12 nvidia-cudnn-cu12 --quiet 2>/dev/null && { + echo -e "${GREEN}✓ CUDA libraries installed${NC}" + } || { + echo -e "${YELLOW}⚠ CUDA libraries installation failed (non-critical)${NC}" + echo -e "${BLUE} GPU acceleration may still work via system CUDA.${NC}" + } fi else echo -e "${BLUE}No NVIDIA GPU detected. CPU mode will be used.${NC}" @@ -226,6 +229,104 @@ install_chatterbox() { echo } +install_flash_attn() { + # Install flash-attn (REQUIRED for Qwen3-TTS). + # Strategy: try pre-built wheel first (fast, no compiler needed), + # then fall back to building from source. + + echo -e "${YELLOW}Installing flash-attn (required for Qwen3-TTS)...${NC}" + + # Check if already installed + if $PYTHON_CMD -c "import flash_attn; print(f'flash-attn {flash_attn.__version__} already installed')" 2>/dev/null; then + echo -e "${GREEN}✓ flash-attn already installed${NC}" + return 0 + fi + + # Detect Python version, torch version, and CUDA version for pre-built wheel + PY_VER=$($PYTHON_CMD -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')" 2>/dev/null) + TORCH_VER=$($PYTHON_CMD -c "import torch; v=torch.__version__.split('+')[0].rsplit('.',1)[0]; print(v)" 2>/dev/null) + CUDA_VER=$($PYTHON_CMD -c " +import torch +cv = torch.version.cuda +if cv: + parts = cv.split('.') + print(f'cu{parts[0]}{parts[1]}') +else: + print('') +" 2>/dev/null) + + echo -e "${BLUE} Detected: Python=${PY_VER}, PyTorch=${TORCH_VER}, CUDA=${CUDA_VER}${NC}" + + FLASH_ATTN_INSTALLED=false + + # --- Attempt 1: Pre-built wheel from mjun0812 (fastest, no compiler needed) --- + if [ -n "$CUDA_VER" ] && [ -n "$TORCH_VER" ] && [ -n "$PY_VER" ]; then + echo -e "${YELLOW} Trying pre-built wheel (no compilation needed)...${NC}" + + # Try flash-attn versions from newest to oldest + for FA_VER in 2.8.3 2.7.4 2.6.3; do + WHEEL_NAME="flash_attn-${FA_VER}+${CUDA_VER}torch${TORCH_VER}-${PY_VER}-${PY_VER}-linux_x86_64.whl" + # Try multiple release tags (the repo uses incrementing tags) + for RELEASE_TAG in v0.7.16 v0.7.15 v0.7.13 v0.7.12 v0.7.11; do + WHEEL_URL="https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/${RELEASE_TAG}/${WHEEL_NAME}" + echo -e "${BLUE} Trying: flash-attn ${FA_VER} from ${RELEASE_TAG}...${NC}" + if $PYTHON_CMD -m pip install "$WHEEL_URL" 2>/dev/null; then + echo -e "${GREEN}✓ flash-attn ${FA_VER} installed from pre-built wheel${NC}" + FLASH_ATTN_INSTALLED=true + break 2 + fi + done + done + fi + + # --- Attempt 2: Build from source with --no-build-isolation --- + if [ "$FLASH_ATTN_INSTALLED" = false ]; then + echo -e "${YELLOW} Pre-built wheel not found. Building from source...${NC}" + echo -e "${BLUE} This may take several minutes to compile...${NC}" + MAX_JOBS=4 $PYTHON_CMD -m pip install -U flash-attn --no-build-isolation 2>&1 | tail -5 + if $PYTHON_CMD -c "import flash_attn" 2>/dev/null; then + echo -e "${GREEN}✓ flash-attn installed (built from source)${NC}" + FLASH_ATTN_INSTALLED=true + fi + fi + + # --- Attempt 3: Try the official Dao-AILab release wheels --- + if [ "$FLASH_ATTN_INSTALLED" = false ] && [ -n "$CUDA_VER" ] && [ -n "$TORCH_VER" ] && [ -n "$PY_VER" ]; then + echo -e "${YELLOW} Trying official GitHub release wheels...${NC}" + # The official repo uses slightly different naming + for FA_VER in 2.7.4 2.6.3; do + WHEEL_NAME="flash_attn-${FA_VER}+${CUDA_VER}torch${TORCH_VER}cxx11abiFALSE-${PY_VER}-${PY_VER}-linux_x86_64.whl" + WHEEL_URL="https://github.com/Dao-AILab/flash-attention/releases/download/v${FA_VER}/${WHEEL_NAME}" + echo -e "${BLUE} Trying: official flash-attn ${FA_VER}...${NC}" + if $PYTHON_CMD -m pip install "$WHEEL_URL" 2>/dev/null; then + echo -e "${GREEN}✓ flash-attn ${FA_VER} installed from official wheel${NC}" + FLASH_ATTN_INSTALLED=true + break + fi + # Try with cxx11abiTRUE + WHEEL_NAME="flash_attn-${FA_VER}+${CUDA_VER}torch${TORCH_VER}cxx11abiTRUE-${PY_VER}-${PY_VER}-linux_x86_64.whl" + WHEEL_URL="https://github.com/Dao-AILab/flash-attention/releases/download/v${FA_VER}/${WHEEL_NAME}" + if $PYTHON_CMD -m pip install "$WHEEL_URL" 2>/dev/null; then + echo -e "${GREEN}✓ flash-attn ${FA_VER} installed from official wheel${NC}" + FLASH_ATTN_INSTALLED=true + break + fi + done + fi + + if [ "$FLASH_ATTN_INSTALLED" = false ]; then + echo -e "${RED}✗ flash-attn installation failed${NC}" + echo -e "${RED} flash-attn is REQUIRED for Qwen3-TTS.${NC}" + echo -e "${BLUE} Manual install options:${NC}" + echo -e "${BLUE} 1. Pre-built wheel: visit https://flashattn.dev and download for your config${NC}" + echo -e "${BLUE} 2. Build from source: MAX_JOBS=4 pip install flash-attn --no-build-isolation${NC}" + echo -e "${BLUE} 3. Check https://github.com/mjun0812/flash-attention-prebuild-wheels/releases${NC}" + return 1 + fi + + return 0 +} + install_qwen3() { echo -e "${YELLOW}Installing Qwen3-TTS...${NC}" $PYTHON_CMD -m pip install qwen-tts 2>/dev/null && { @@ -246,26 +347,25 @@ install_qwen3() { rm -rf "$TEMP_DIR" } - # Install flash-attn (recommended for Qwen3-TTS performance, not required) - # Must use --no-build-isolation because flash-attn needs torch at build time - # but doesn't declare it as a build dependency - echo -e "${YELLOW}Installing flash-attn (recommended for Qwen3-TTS performance)...${NC}" - echo -e "${BLUE}This may take a few minutes to compile...${NC}" - MAX_JOBS=4 $PYTHON_CMD -m pip install -U flash-attn --no-build-isolation 2>/dev/null && { - echo -e "${GREEN}✓ flash-attn installed${NC}" - } || { - echo -e "${YELLOW}⚠ flash-attn installation failed (not critical)${NC}" - echo -e "${BLUE} Qwen3-TTS will still work using eager attention (slower).${NC}" - echo -e "${BLUE} To try again later: pip install flash-attn --no-build-isolation${NC}" - echo -e "${BLUE} Requires: compatible GPU + CUDA toolkit + torch already installed.${NC}" - } - - # Verify installation - $PYTHON_CMD -c "from qwen_tts import Qwen3TTSModel; print(' ✓ Qwen3-TTS import successful')" 2>/dev/null || { - echo -e "${YELLOW} Note: Qwen3-TTS import test failed. It may still work at runtime.${NC}" - } - $PYTHON_CMD -c "import flash_attn; print(' ✓ flash-attn import successful')" 2>/dev/null || { - echo -e "${YELLOW} Note: flash-attn not available. Qwen3-TTS will use eager attention (slower but functional).${NC}" + # Install flash-attn (REQUIRED) + install_flash_attn + + # Verify installation - suppress the qwen_tts flash-attn warning during check + $PYTHON_CMD -W ignore -c " +import warnings, io, sys +# Suppress all output from qwen_tts import (it prints a flash-attn warning) +_stderr = sys.stderr +sys.stderr = io.StringIO() +try: + from qwen_tts import Qwen3TTSModel + import flash_attn + sys.stderr = _stderr + print(' ✓ Qwen3-TTS + flash-attn verified') +except ImportError as e: + sys.stderr = _stderr + print(f' ✗ Import failed: {e}') +" 2>/dev/null || { + echo -e "${RED} ✗ Qwen3-TTS verification failed${NC}" } echo } @@ -26,6 +26,7 @@ def get_available_backends() -> dict[str, bool]: Does a deep import check (tries the actual class, not just the top-level package) so partially-installed packages don't false-positive. + Qwen3-TTS requires both qwen_tts AND flash-attn. """ backends = {} @@ -36,11 +37,19 @@ def get_available_backends() -> dict[str, bool]: except (ImportError, ModuleNotFoundError, Exception): backends["chatterbox"] = False - # Check Qwen3-TTS - only require qwen_tts itself. - # flash-attn is recommended for performance but NOT required; - # Qwen3-TTS falls back to manual PyTorch attention without it. + # Check Qwen3-TTS - requires both qwen_tts and flash-attn try: - from qwen_tts import Qwen3TTSModel # noqa: F401 + # Suppress the flash-attn warning that qwen_tts prints on import + # (it prints to stdout/stderr when flash-attn is missing during import) + import io, sys as _sys + _old_stdout, _old_stderr = _sys.stdout, _sys.stderr + _sys.stdout = io.StringIO() + _sys.stderr = io.StringIO() + try: + from qwen_tts import Qwen3TTSModel # noqa: F401 + finally: + _sys.stdout, _sys.stderr = _old_stdout, _old_stderr + import flash_attn # noqa: F401 backends["qwen3"] = True except (ImportError, ModuleNotFoundError, Exception): backends["qwen3"] = False @@ -48,15 +57,6 @@ def get_available_backends() -> dict[str, bool]: return backends -def is_flash_attn_available() -> bool: - """Check if flash-attn is installed (recommended for Qwen3-TTS performance).""" - try: - import flash_attn # noqa: F401 - return True - except (ImportError, ModuleNotFoundError, Exception): - return False - - # --------------------------------------------------------------------------- # Base provider # --------------------------------------------------------------------------- @@ -240,7 +240,15 @@ class Qwen3TTSProvider(BaseTTSProvider): return try: - from qwen_tts import Qwen3TTSModel + # Suppress the flash-attn warning that qwen_tts prints on import + import io, sys as _sys + _old_stdout, _old_stderr = _sys.stdout, _sys.stderr + _sys.stdout = io.StringIO() + _sys.stderr = io.StringIO() + try: + from qwen_tts import Qwen3TTSModel + finally: + _sys.stdout, _sys.stderr = _old_stdout, _old_stderr model_name = self._get_model_name(clone=False) print(f"Loading Qwen3-TTS ({model_name}) on {self.device}...") @@ -250,13 +258,17 @@ class Qwen3TTSProvider(BaseTTSProvider): kwargs = {"device_map": f"{self.device}:0" if self.device == "cuda" else self.device} kwargs["dtype"] = dtype - # Use flash-attn if available (faster), otherwise fall back to eager - if is_flash_attn_available(): + # flash-attn is required + try: + import flash_attn # noqa: F401 kwargs["attn_implementation"] = "flash_attention_2" - else: - print("Note: flash-attn not installed. Using eager attention (slower).") - print(" Install for better performance: pip install flash-attn --no-build-isolation") - kwargs["attn_implementation"] = "eager" + except ImportError: + raise ImportError( + "flash-attn is required for Qwen3-TTS but is not installed.\n" + "Install with: ./scripts/install.sh --tts=qwen3\n" + "Or manually: pip install flash-attn --no-build-isolation\n" + "Or use a pre-built wheel from https://github.com/mjun0812/flash-attention-prebuild-wheels/releases" + ) self.model = Qwen3TTSModel.from_pretrained(model_name, **kwargs) self._initialized = True @@ -273,7 +285,15 @@ class Qwen3TTSProvider(BaseTTSProvider): return try: - from qwen_tts import Qwen3TTSModel + import io, sys as _sys + _old_stdout, _old_stderr = _sys.stdout, _sys.stderr + _sys.stdout = io.StringIO() + _sys.stderr = io.StringIO() + try: + from qwen_tts import Qwen3TTSModel + finally: + _sys.stdout, _sys.stderr = _old_stdout, _old_stderr + import flash_attn # noqa: F401 model_name = self._get_model_name(clone=True) print(f"Loading Qwen3-TTS clone model ({model_name})...") @@ -281,20 +301,16 @@ class Qwen3TTSProvider(BaseTTSProvider): dtype = torch.bfloat16 if self.device == "cuda" else torch.float32 kwargs = {"device_map": f"{self.device}:0" if self.device == "cuda" else self.device} kwargs["dtype"] = dtype - - if is_flash_attn_available(): - kwargs["attn_implementation"] = "flash_attention_2" - else: - kwargs["attn_implementation"] = "eager" + kwargs["attn_implementation"] = "flash_attention_2" self._clone_model = Qwen3TTSModel.from_pretrained(model_name, **kwargs) print("Qwen3-TTS clone model loaded") except ImportError as e: raise ImportError( - "Qwen3-TTS is not installed.\n" - "Install with: pip install qwen-tts\n" - "Or run: ./scripts/install.sh --tts=qwen3" + "flash-attn is required for Qwen3-TTS.\n" + "Install with: ./scripts/install.sh --tts=qwen3\n" + "Or use a pre-built wheel from https://github.com/mjun0812/flash-attention-prebuild-wheels/releases" ) from e except Exception as e: raise RuntimeError(f"Failed to load Qwen3-TTS clone model: {e}") from e @@ -430,8 +446,6 @@ if __name__ == "__main__": for name, available in backends.items(): status = "INSTALLED" if available else "not installed" print(f" {name}: {status}") - flash_ok = is_flash_attn_available() - print(f" flash-attn: {'INSTALLED' if flash_ok else 'not installed (Qwen3-TTS will use eager attention)'}") # Test with first available backend for name, available in backends.items(): @@ -90,7 +90,7 @@ def main(): # tts_available=True means the TTS UI section is visible. # tts_backends_available tracks which engines are actually installed. try: - from tts_provider import get_available_backends, is_flash_attn_available + from tts_provider import get_available_backends from tts_controller import TTSController backends = get_available_backends() @@ -122,9 +122,6 @@ def main(): print(f" TTS backend {name}: {status}") if avail: has_any = True - if backends.get("qwen3"): - flash_ok = is_flash_attn_available() - print(f" flash-attn: {'available' if flash_ok else 'not installed (Qwen3 will use eager attention)'}") if not has_any: state.tts_enabled = False # Don't auto-enable if nothing is installed print(" No TTS backends installed.")