fix: install cuDNN version matching PyTorch build to prevent mismatch
Root cause of CUDNN_STATUS_SUBLIBRARY_VERSION_MISMATCH: install.sh was installing generic nvidia-cudnn-cu12 which could be a different version than what PyTorch was compiled against. Now the install script: 1. Detects PyTorch's built-in cuDNN version (torch.backends.cudnn.version()) 2. Checks if the installed nvidia-cudnn-cu12 package matches 3. Pins the install to the exact major.minor version PyTorch expects 4. Runs a smoke test (small conv2d on GPU) to verify cuDNN works Also cleaned up the runtime error messages to avoid the redundant "cuDNN error: cuDNN error:" prefix. https://claude.ai/code/session_01965m1v3f1cK6gDV1JHioA8
+68 −12 · 2 files changed
-
scripts/install.sh+59 −4 -
src/tts_provider.py+9 −8
@@ -136,11 +136,49 @@ if command -v nvidia-smi &> /dev/null; then HAS_GPU=true echo - # Auto-install CUDA libraries - check if already present first - if $PYTHON_CMD -c "import nvidia.cublas" 2>/dev/null && $PYTHON_CMD -c "import nvidia.cudnn" 2>/dev/null; then - echo -e "${GREEN}✓ CUDA libraries already installed${NC}" + # Auto-install CUDA libraries that match PyTorch's build. + # CRITICAL: version mismatch causes CUDNN_STATUS_SUBLIBRARY_VERSION_MISMATCH. + # We detect the exact cuDNN version PyTorch was compiled with and install that. + echo -e "${YELLOW}Checking CUDA library compatibility...${NC}" + CUDNN_OK=false + CUDNN_MATCH=$($PYTHON_CMD -c " +import torch, importlib, json, re, sys +# Get the cuDNN version PyTorch was built with +built = torch.backends.cudnn.version() # e.g. 90100 for 9.1.0 +if not built: + sys.exit(1) +major, minor, patch = built // 10000, (built % 10000) // 100, built % 100 +# Check currently installed nvidia-cudnn version +try: + dist = importlib.metadata.distribution('nvidia-cudnn-cu12') + installed = dist.version # e.g. '9.1.0.70' + inst_parts = installed.split('.') + if int(inst_parts[0]) == major and int(inst_parts[1]) == minor: + print('MATCH') + else: + print(f'{major}.{minor}.{patch}') +except importlib.metadata.PackageNotFoundError: + print(f'{major}.{minor}.{patch}') +" 2>/dev/null) + + if [ "$CUDNN_MATCH" = "MATCH" ]; then + echo -e "${GREEN}✓ CUDA libraries already installed and version-matched${NC}" + CUDNN_OK=true + elif [ -n "$CUDNN_MATCH" ]; then + # Need to install the matching cuDNN version + CUDNN_MAJOR_MINOR=$(echo "$CUDNN_MATCH" | cut -d. -f1,2) + echo -e "${YELLOW}Installing cuDNN ${CUDNN_MAJOR_MINOR}.x to match PyTorch build...${NC}" + # Pin to the major.minor that PyTorch expects + $PYTHON_CMD -m pip install "nvidia-cudnn-cu12>=${CUDNN_MAJOR_MINOR}.0,<${CUDNN_MAJOR_MINOR%.*}.$((${CUDNN_MAJOR_MINOR#*.}+1)).0" nvidia-cublas-cu12 --quiet 2>/dev/null && { + echo -e "${GREEN}✓ CUDA libraries installed (cuDNN ${CUDNN_MAJOR_MINOR}.x)${NC}" + CUDNN_OK=true + } || { + echo -e "${YELLOW}⚠ Pinned cuDNN install failed. Trying generic...${NC}" + $PYTHON_CMD -m pip install nvidia-cublas-cu12 nvidia-cudnn-cu12 --quiet 2>/dev/null + } else - echo -e "${YELLOW}Installing CUDA libraries for GPU acceleration...${NC}" + # No torch or no CUDA build — install generic + echo -e "${YELLOW}Installing CUDA libraries...${NC}" $PYTHON_CMD -m pip install nvidia-cublas-cu12 nvidia-cudnn-cu12 --quiet 2>/dev/null && { echo -e "${GREEN}✓ CUDA libraries installed${NC}" } || { @@ -148,6 +186,23 @@ if command -v nvidia-smi &> /dev/null; then echo -e "${BLUE} GPU acceleration may still work via system CUDA.${NC}" } fi + + # Verify cuDNN actually works with PyTorch + if [ "$CUDNN_OK" = true ]; then + $PYTHON_CMD -c " +import torch +assert torch.backends.cudnn.is_available(), 'cuDNN not available' +v = torch.backends.cudnn.version() +print(f' cuDNN version: {v // 10000}.{(v % 10000) // 100}.{v % 100}') +# Quick smoke test: run a small conv on GPU to trigger cuDNN init +if torch.cuda.is_available(): + x = torch.randn(1, 1, 8, 8, device='cuda') + torch.nn.functional.conv2d(x, torch.randn(1, 1, 3, 3, device='cuda')) + print(' cuDNN smoke test: passed') +" 2>/dev/null || { + echo -e "${YELLOW}⚠ cuDNN smoke test failed. TTS will auto-fallback to non-cuDNN mode.${NC}" + } + fi else echo -e "${BLUE}No NVIDIA GPU detected. CPU mode will be used.${NC}" fi @@ -146,15 +146,16 @@ class ChatterboxProvider(BaseTTSProvider): except RuntimeError as cuda_err: err_str = str(cuda_err).lower() if load_device == "cuda" and ("cudnn" in err_str or "cuda" in err_str): - # Retry 1: disable cuDNN (uses slower but compatible kernels) - print(f"cuDNN error: {cuda_err}") - print("Retrying with cuDNN disabled...") + # Retry 1: disable cuDNN (uses slower but compatible CUDA kernels) + print(f"CUDA error during model load: {cuda_err}") + print("Disabling cuDNN and retrying on CUDA...") torch.backends.cudnn.enabled = False try: self.model = self._load_model("cuda") - except RuntimeError: - # Retry 2: load on CPU, then move to CUDA - print("Still failing on CUDA. Loading on CPU first...") + except RuntimeError as retry_err: + # Retry 2: fall back to CPU entirely + print(f"CUDA still failing: {retry_err}") + print("Falling back to CPU...") torch.backends.cudnn.enabled = True self.model = self._load_model("cpu") self.device = "cpu" @@ -204,8 +205,8 @@ class ChatterboxProvider(BaseTTSProvider): err_str = str(e).lower() if "cudnn" in err_str or "cuda" in err_str: # cuDNN failed during inference — disable it and retry once - print(f"cuDNN error during synthesis: {e}") - print("Retrying with cuDNN disabled...") + print(f"CUDA error during synthesis: {e}") + print("Disabling cuDNN and retrying...") torch.backends.cudnn.enabled = False audio = self.model.generate( text=text,