From f60cf17c7520793d04de392f4e9740788141ebf4 Mon Sep 17 00:00:00 2001 From: translator Date: Mon, 9 Mar 2026 17:55:38 +0900 Subject: [PATCH] Wired in global power settings to separators and fixed broken ELRC output. --- STEMxLALA.md | 256 ++++++++++ SUMMARIES.md | 247 ++++++++++ backend/main.py | 16 + backend/routers/models.py | 4 +- backend/routers/projects.py | 91 +++- .../services/bs_roformer_chorus_separator.py | 8 +- backend/services/bs_roformer_separator.py | 11 +- backend/services/lyric_transcriber_service.py | 18 +- backend/services/mdxnet_separator.py | 181 ++++--- .../services/mel_band_roformer_separator.py | 12 +- .../mel_band_specialized_separator.py | 16 +- backend/services/scnet_separator.py | 12 +- backend/services/separator_service.py | 441 +++++++++++++----- backend/static/index.html | 2 +- frontend/src/api/client.ts | 7 +- lalalai | 1 + stemengine.yml | 34 +- 17 files changed, 1158 insertions(+), 199 deletions(-) create mode 100644 STEMxLALA.md create mode 160000 lalalai diff --git a/STEMxLALA.md b/STEMxLALA.md new file mode 100644 index 0000000..733e233 --- /dev/null +++ b/STEMxLALA.md @@ -0,0 +1,256 @@ +# STEMxLALA — Implementation Plan + +Migrate StemEngine's separation backend to use **LALAL.AI API v1 exclusively**, replacing all +local model inference (Demucs, BS-RoFormer, MelBand, MDX-Net, SCNet, etc.) with cloud API calls. + +--- + +## Background + +### Current architecture +- `separator_service.py` — monolithic 2000+ line dispatcher that branches on model ID to call + local ML model code (Demucs CLI subprocess, BS-RoFormer in-process PyTorch, MDX-Net ONNX, etc.) +- `config.py` — defines ~50 model IDs across 13 groups +- `routers/separate.py` — validates model ID, enqueues via `jm.submit_gpu()` +- `job_manager.py` — `submit_gpu()` routes to a single-worker GPU executor (`_gpu_executor`) +- All separation runs inside the container on local GPU via CUDA (PyTorch cu124 wheels) + +### LALAL.AI v1 API +Reference implementation: `lalalai/api-v1/python/lalalai_splitter.py` + +Three separation endpoints: +| Endpoint | Stem | Notes | +|---|---|---| +| `/api/v1/split/voice_clean/` | `voice` | Noise cancellation + optional dereverb | +| `/api/v1/split/demuser/` | `music` | Background music removal (speech stays) | +| `/api/v1/split/stem_separator/` | `vocals`, `drum`, `piano`, `bass`, `electric_guitar`, `acoustic_guitar`, `synthesizer`, `strings`, `wind` | deep_extraction or clear_cut, optional dereverb, optional multivocal | + +Workflow per file: **upload → split (async task) → poll check → download tracks → (optional) delete** + +Splitter model options: `orion`, `perseus`, `phoenix`, `andromeda` (or `null` = auto/latest) + +--- + +## Design Decisions + +### CUDA stays — for other features, not separation +The Docker image (`nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04`) and `runtime: nvidia` in +`stemengine.yml` must remain. CUDA is still required for: +- WhisperX lyric transcription (runs on GPU via `_transcription_executor`) +- basic-pitch ONNX inference +- Any future local models + +The `CPU_SEPARATION` env var becomes irrelevant for LALA separation (network replaces GPU), +but the hardware config should not be removed. + +### API key storage +A new env var `LALAL_API_KEY` (type: str, required for separation) is stored alongside existing +secrets (`SECRET_KEY`, `HF_TOKEN`). It can be set in `stemengine.yml` or via the Settings UI +once the settings router supports it. + +### Job executor routing +LALAL API calls are **network I/O, not GPU work**. They must NOT go through `submit_gpu()` +(the single-worker GPU queue) — doing so would block all GPU-bound transcription while waiting +for the network. Use `submit()` (general thread pool) instead, or a new dedicated +`_lala_executor = ThreadPoolExecutor(max_workers=3)` to cap concurrent uploads. + +### Preset mixing (instrumental, static, dynamic) +The existing `preset_service.py` (`make_instrumental`, `make_static`, `make_dynamic`) mixes +stem audio files with `pydub`. It is model-agnostic — it takes file paths and operates on +whatever stems exist. This is fully reusable with LALA output. **No changes needed here.** + +### Project folder structure +Unchanged. LALA stems are downloaded into the same `PROJECT_SUBFOLDERS["stems"]` directory +that local models produce. The rest of the app (pipeline, mix, transcribe) is unaffected. + +### Output format +LALAL API delivers stems as downloaded files (filename from `Content-Disposition`). These may +be WAV or MP3 depending on LALA's output. After download, the existing format-conversion logic +(ffmpeg via pydub) should be applied to normalize to the user's requested `output_format`. + +--- + +## Implementation Steps + +### Step 1 — New service: `lalalai_separator.py` + +Create `backend/services/lalalai_separator.py` as a clean, self-contained module. + +**Contents:** +- Constants: `LALAL_API_BASE = "https://www.lalal.ai/api/v1/"` +- Functions lifted and adapted from the reference `lalalai_splitter.py`: + - `upload_file(file_path, api_key) -> source_id` + - `split_file(source_id, stem, api_key, *, splitter, extraction_level, dereverb, noise_cancelling, multivocal) -> task_id` + - `poll_until_done(task_id, api_key, cancel_flag) -> result_dict` + - `download_stems(result_dict, dest_dir) -> list[Path]` + - `delete_source(source_id, api_key)` +- Top-level `run_lala_separation(job_id, file_path, lala_model_id, output_format, *, preset_type, static_option, dynamic_levels, project_name_override)` — mirrors the signature of `run_separation()` so callers are swappable. + +**Cancel support:** `poll_until_done` accepts the job's `threading.Event` cancel flag (from +`job_manager._cancel_flags`) and raises `CancelledError` if set between polls. + +**Error handling:** Wrap all `requests` calls; on HTTP error or timeout, call +`jm.fail_job(job_id, message)` and re-raise so the executor logs it. + +--- + +### Step 2 — New config section: LALA models + +In `backend/config.py`, add: + +```python +# LALAL.AI v1 models +LALA_STEM_SEPARATOR_STEMS = [ + {"id": "lala_vocals", "label": "LALA Vocals", "group": "LALAL.AI", "lala_stem": "vocals"}, + {"id": "lala_drum", "label": "LALA Drums", "group": "LALAL.AI", "lala_stem": "drum"}, + {"id": "lala_bass", "label": "LALA Bass", "group": "LALAL.AI", "lala_stem": "bass"}, + {"id": "lala_piano", "label": "LALA Piano", "group": "LALAL.AI", "lala_stem": "piano"}, + {"id": "lala_electric_guitar", "label": "LALA Electric Guitar", "group": "LALAL.AI", "lala_stem": "electric_guitar"}, + {"id": "lala_acoustic_guitar", "label": "LALA Acoustic Guitar", "group": "LALAL.AI", "lala_stem": "acoustic_guitar"}, + {"id": "lala_synthesizer", "label": "LALA Synthesizer", "group": "LALAL.AI", "lala_stem": "synthesizer"}, + {"id": "lala_strings", "label": "LALA Strings", "group": "LALAL.AI", "lala_stem": "strings"}, + {"id": "lala_wind", "label": "LALA Wind", "group": "LALAL.AI", "lala_stem": "wind"}, + {"id": "lala_voice_clean", "label": "LALA Voice Clean", "group": "LALAL.AI", "lala_stem": "voice"}, + {"id": "lala_demuser", "label": "LALA Demuser (Music)", "group": "LALAL.AI", "lala_stem": "music"}, +] + +LALA_MODELS = LALA_STEM_SEPARATOR_STEMS +LALA_MODEL_IDS = {m["id"] for m in LALA_MODELS} +``` + +Add `LALA_MODELS` to `ALL_MODELS`. Add `LALA_MODEL_IDS` to `ALL_MODEL_IDS`. + +**Optional LALA-specific parameters** (can be form fields or user settings defaults): +```python +LALA_DEFAULT_SPLITTER = None # auto = latest +LALA_DEFAULT_EXTRACTION_LEVEL = "deep_extraction" +LALA_DEFAULT_DEREVERB = False +LALA_DEFAULT_NOISE_CANCELLING = 1 +``` + +--- + +### Step 3 — Router changes: `routers/separate.py` + +1. Add optional form fields to `POST /api/separate`: + - `lala_splitter: Optional[str] = Form(None)` — one of `orion|perseus|phoenix|andromeda|None` + - `lala_extraction_level: str = Form("deep_extraction")` + - `lala_dereverb: bool = Form(False)` + - `lala_noise_cancelling: int = Form(1)` + - `lala_multivocal: Optional[str] = Form(None)` — `lead_back` or None + +2. Routing logic in `separate()`: + ```python + if model in LALA_MODEL_IDS: + jm.submit_lala(run_lala_separation, job_id, ...) + else: + jm.submit_gpu(run_separation, job_id, ...) + ``` + +3. Apply same routing to `separate_existing()` and `separate_from_path()`. + +--- + +### Step 4 — Job manager: add `submit_lala()` + +In `backend/job_manager.py`, add: + +```python +_lala_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="lala") + +def submit_lala(fn: Callable, *args, **kwargs): + """Submit a LALAL.AI API separation task (network I/O, not GPU).""" + _lala_executor.submit(fn, *args, **kwargs) +``` + +This keeps LALA network calls off the GPU queue so local GPU tasks (transcription) are never +blocked waiting for remote API responses. + +--- + +### Step 5 — Config: env var + settings + +In `config.py`: +```python +LALAL_API_KEY = os.environ.get("LALAL_API_KEY", "") +``` + +In `stemengine.yml`: +```yaml +- LALAL_API_KEY=${LALAL_API_KEY} +``` + +In the Settings UI (if applicable), surface `LALAL_API_KEY` as an encrypted credential field +alongside the existing Subsonic and HuggingFace token fields. + +The existing `SECRET_KEY`-based encryption in `settings_service.py` / the settings router +should handle this with no structural changes — just add the field. + +--- + +### Step 6 — Dockerfile cleanup (optional, deferred) + +Once LALA is the sole separator: +- PyTorch, torchaudio, torchvision (CUDA wheels — ~3 GB) can be removed if no other feature + needs them. **Hold off** until WhisperX is also evaluated — it currently depends on PyTorch. +- All model-specific Python packages (open-unmix, demucs, ml_collections, etc.) can be removed. +- Model volume mounts (`/opt/bs_roformer_models`, etc.) can be removed from `stemengine.yml`. +- `CPU_SEPARATION` env var becomes vestigial but harmless to leave. + +This step is explicitly deferred until the LALA path is confirmed working in production. + +--- + +### Step 7 — Frontend (minimal changes) + +The frontend's model picker currently renders `ALL_MODELS` grouped by `"group"`. Adding +`"group": "LALAL.AI"` models to `ALL_MODELS` is sufficient — the group will appear +automatically. + +LALA-specific form fields (`lala_splitter`, `lala_extraction_level`, etc.) should be surfaced +conditionally when the selected model's group is `"LALAL.AI"`. This is a UI-only change in the +separation form component (likely `SeparatePanel` or equivalent). + +--- + +## File Change Summary + +| File | Action | Notes | +|---|---|---| +| `backend/services/lalalai_separator.py` | **Create** | New LALA separation service | +| `backend/config.py` | **Edit** | Add `LALA_MODELS`, `LALAL_API_KEY` | +| `backend/routers/separate.py` | **Edit** | Add LALA form fields, routing branch | +| `backend/job_manager.py` | **Edit** | Add `_lala_executor`, `submit_lala()` | +| `stemengine.yml` | **Edit** | Add `LALAL_API_KEY` env var | +| `backend/services/separator_service.py` | No change yet | Keep until LALA confirmed | +| `backend/services/preset_service.py` | No change | Reused as-is | +| `Dockerfile` | Deferred | Cleanup after LALA confirmed working | + +--- + +## What is NOT changing + +- `job_manager.py` executor for GPU (`_gpu_executor`) — still used for WhisperX transcription +- `preset_service.py` — mix logic is model-agnostic, works on file paths +- Project folder structure — stems land in the same place +- Docker CUDA base image — required for WhisperX +- `nvidia` runtime in `stemengine.yml` — required for WhisperX +- All non-separation routers (transcribe, lyric_transcribe, lrc, subsonic, pipeline, etc.) + +--- + +## Risks & Notes + +- **LALAL.AI API key required at runtime.** If `LALAL_API_KEY` is empty and a LALA model is + selected, `lalalai_separator.py` should fail the job immediately with a clear error message + rather than getting an HTTP 401 partway through an upload. +- **Upload cost.** LALAL.AI charges credits per minute of audio. The plan does not implement + source caching/reuse (`source_id` reuse across jobs for the same file) in v1 — that is a + potential future optimization. +- **File size limits.** LALAL.AI imposes upload limits per license tier. Large files may need + to be rejected before upload with an appropriate HTTP 413 or job error. +- **Polling interval.** The reference uses 5-second polling. This is fine for background jobs + but should respect the cancel flag between polls. +- **Output filename.** LALA returns filenames via `Content-Disposition`. The download function + must parse this (already implemented in the reference code) and then rename to match the + StemEngine stem-naming convention: `"Artist - Title (vocals).mp3"`. diff --git a/SUMMARIES.md b/SUMMARIES.md index badf284..7658fa9 100644 --- a/SUMMARIES.md +++ b/SUMMARIES.md @@ -400,3 +400,250 @@ When `lineNeedsLayer` is false the behindLayer is skipped entirely (single-div f **Coverage:** `renderScrollStyle`, `renderFadeStyle`, `renderKaraokeStyle` (non-active + active wordless), `renderGlowStyle` (non-active + active wordless), `renderBlurStyle` (non-active + active wordless). Active word-by-word lines continue to use `KaraokeWord`'s existing behind-layer. **Result:** Fill is always the top layer; stroke appears only outside the letterform; shadow is always behind both stroke and fill — in every mode, for every line. + +--- + +## 2026-03-09 + +### WhisperX hallucination fix — "Thank you", emoji, and Japanese subscription prompts appended to transcriptions + +**Problem:** WhisperX transcriptions occasionally appended hallucinated text at the end of or within a transcription — e.g. "Thank you", emoji, or "チャンネル登録よろしくお願いします" ("Please subscribe to my channel"). Reproduced on 為に by Sasuke Haraguchi. + +**Root cause — two compounding factors:** + +1. **`_fill_transcript_gaps` re-transcribed trailing silence.** + The function (`lyric_transcriber_service.py:1144`) scanned for audio regions > 2 s with no detected speech and re-ran WhisperX on each slice. The trailing silence after the last lyric was included in this scan. With no real speech, WhisperX hallucinated YouTube-trained filler content (subscription prompts, sign-offs, "Thank you"). Whisper was trained heavily on YouTube video audio, and these are the completion patterns it learned for silence/ambient noise, especially when Japanese is the detected language. + +2. **`no_speech_threshold: 0.45` was too low.** + Whisper's stock default is 0.60. The lower value made WhisperX more aggressively classify silent or ambient-noise segments as "probably speech" rather than discarding them, which fed more silent audio to the decoder and increased hallucination frequency. + +**Fixes applied to `backend/services/lyric_transcriber_service.py`:** + +- **`_fill_transcript_gaps`:** Removed the trailing-gap check. Mid-song gaps are still re-transcribed (catches lyrics in quiet passages), but the audio tail after the last segment is no longer sent back through the model. Comment added explaining the rationale. + +- **`no_speech_threshold`:** Raised from `0.45` → `0.50`. Comment updated to document that 0.45 was too aggressive and caused silence to be treated as speech. + +--- + +### MDX-Net fix — `module 'numpy' has no attribute 'stft'` + +**Problem:** `mdxnet_separator.py` called `np.stft()` and `np.istft()` in `_stft` / `_istft`. These functions do not exist in NumPy in any version — they were never part of the NumPy API. The code was broken from the start and had never been exercised. + +**Fix:** Replaced both functions with `librosa.stft` / `librosa.istft`, which share the same `n_fft` / `hop_length` parameter names. librosa is already a dependency (used in `_load_audio`). + +Shape contract preserved: `_spec_to_batch` expects `(channels, freq, time, 2)` where the last dimension is `[real, imag]`. librosa returns a complex `(freq, time)` array, so `_stft` now packs it with `np.stack([spec.real, spec.imag], axis=-1)`, and `_istft` reconstructs the complex array with `spec[ch,:,:,0] + 1j * spec[ch,:,:,1]` before calling `librosa.istft`. + +--- + +### MDX-Net fix — `[ONNXRuntimeError] : 2 : INVALID_ARGUMENT : Got invalid dimensions for input: input … index: 1` + +**Problem:** After the `librosa` fix, ONNX inference still failed with an input-dimension error. Three compounding bugs: + +1. **`N_FFT = 2048` was wrong.** `librosa.stft` with `n_fft=2048` produces `1025` frequency bins. The ONNX models expect input with `DIM_F = 3072` bins (plus the Nyquist bin = 3073 total). The correct `n_fft` for `3073` bins is `6144`. + +2. **`_spec_to_batch` produced `(batch, 2, dim_f, dim_t)` instead of `(batch, 4, dim_f, dim_t)`.** UVR MDX-Net ONNX models expect 4 input channels: `[L_real, L_imag, R_real, R_imag]` — i.e. stereo(2) × complex(2) merged into one dimension. The old code kept stereo and complex as separate dimensions, producing the wrong channel count of 2. + +3. **`_batch_to_spec` inverted the wrong shape** — inverse of the broken forward pass; was not the correct inverse of the corrected forward pass. + +4. **Length mismatch in `_demix`.** After ISTFT, `target_audio` can be slightly shorter or longer than the original `mix` due to STFT/ISTFT padding behaviour. `mix - target_audio` raised an operand-shape error. + +**Fixes applied to `backend/services/mdxnet_separator.py`:** + +- **`N_FFT`**: Changed `2048` → `6144`. This produces 3073 freq bins; the top bin is cropped to 3072 before ONNX and zero-padded back after. + +- **`_spec_to_batch` rewritten:** + ``` + Input: (2, freq, time, 2) = (stereo, freq, time, complex) + → transpose to (stereo, complex, freq, time) = (2, 2, freq, time) + → reshape to merge: (4, freq, time) + → crop freq to dim_f, pad time to multiple of dim_t + → chunk time: (4, dim_f, t_chunks, dim_t) + → transpose: (t_chunks, 4, dim_f, dim_t) ← correct ONNX input shape + ``` + +- **`_batch_to_spec` rewritten** as the exact inverse: + ``` + Input: (t_chunks, 4, dim_f, dim_t) + → reassemble time: (4, dim_f, time) + → remove time pad, restore freq (zero-pad back to n_fft//2+1) + → reshape: (2, 2, freq, time) [stereo, complex] + → transpose: (2, freq, time, 2) ← original spec format + ``` + +- **`_demix` length guard added:** after ISTFT, `target_audio` is trimmed or zero-padded to match `mix.shape[1]` before subtraction. + +**Second-pass fix (after inspecting actual model):** Inspecting `UVR_MDXNET_KARA.onnx` with `onnxruntime` revealed the true input shape is `(batch, 4, **2048**, **256**)` — not `(batch, 4, 3072, 8)`. The hardcoded `DIM_F=3072` and `DIM_T=8` were simply wrong; different model files have different dimensions. Fix: removed all module-level `DIM_F`, `DIM_T`, `N_FFT` constants. `_get_session()` now reads `session.get_inputs()[0].shape` and stores `self.dim_f = shape[2]`, `self.dim_t = shape[3]`, `self.n_fft = self.dim_f * 2`. All calls to `_stft`, `_spec_to_batch`, `_batch_to_spec`, `_istft` pass these instance values explicitly. `_batch_to_spec` now reads `dim_f`/`dim_t` from the metadata dict rather than globals. + +**Third-pass fix — `name 'N_FFT' is not defined`:** After removing the module-level constants, the three function signatures (`_stft`, `_istft`, `_spec_to_batch`) still used `N_FFT`, `DIM_F`, `DIM_T` as default parameter values, causing a `NameError` at import time. Fixed by replacing the defaults with inline literals (`n_fft=4096`, `dim_f=2048`, `dim_t=256`) matching the KARA model's actual dimensions. These defaults are never used in practice — `MDXNetSeparator._demix` always passes explicit values derived from the session. + +--- + +### MelBand Karaoke fix — 404 on model download + +**Problem:** `mel_band_karaoke` separation failed with `404 Client Error` during model download. The repo `becruily/mel-band-roformer-karaoke` still exists on HuggingFace, but the filenames changed: the actual files are `mel_band_roformer_karaoke_becruily.ckpt` (1.72 GB) and `config_karaoke_becruily.yaml`, not `model.ckpt` / `config.yaml`. + +**Two locations had the wrong filenames:** + +1. **`backend/routers/models.py` — `MODEL_DOWNLOAD_INFO["mel_band_karaoke"]`**: `files` list updated from `["model.ckpt", "config.yaml"]` to `["mel_band_roformer_karaoke_becruily.ckpt", "config_karaoke_becruily.yaml"]`. Also corrected `size_mb` from `170` → `1760` to reflect the actual 1.72 GB checkpoint. + +2. **`backend/services/mel_band_specialized_separator.py` — `MelBandKaraokeSeparator.__init__`**: `model_path` and `config_path` updated to use the correct filenames so that `_ensure_model`'s existence check and `snapshot_download` both reference the right paths. + +--- + +### MelBand Karaoke fix — `Missing key(s) in state_dict: static_alpha` + +**Problem:** After the download fix, model loading raised `Error(s) in loading state_dict for MelBandRoformer: Missing key(s) in state_dict: "layers.0.0.layers.0.0.static_alpha", ...`. The installed `bs_roformer` library is newer than the checkpoint — a `static_alpha` learnable scalar was added to the RoFormer attention layers in a later release. The checkpoint was trained before this parameter existed, so it is absent from the saved weights. + +**Fix:** Changed `model.load_state_dict(state_dict)` → `model.load_state_dict(state_dict, strict=False)` in `MelBandSeparatorBase.get_model()`. Missing keys keep their initialized values (`static_alpha` defaults to `1.0`, a neutral multiplier). A `logging.warning` is emitted listing how many keys were missing, so any future version-mismatch is visible in logs without crashing the job. + +--- + +### MelBand Karaoke OOM fix — VRAM cap and OOM retry wired into separation path + +**Problem:** `mel_band_specialized_separator.py` was not wired into StemEngine's global power/GPU settings. Two gaps: + +1. **No VRAM cap on separation.** `_apply_cuda_limits()` (which calls `torch.cuda.set_per_process_memory_fraction(0.80)`) was only ever called by `lyric_transcriber_service.py`. If separation ran before transcription (or transcription never ran in the session), the allocator had no VRAM ceiling and could OOM. + +2. **No OOM retry in `_run_mel_band_karaoke`.** On an `out of memory` exception the function just re-raised, letting the top-level handler report it as a CUDA error with no recovery. + +**Fixes applied to `backend/services/separator_service.py`:** + +- **VRAM cap**: Added `from .lyric_transcriber_service import _apply_cuda_limits; _apply_cuda_limits()` at the top of `run_separation()`, before `torch.cuda.empty_cache()`. Now separation and transcription share the same VRAM ceiling (default 80%, overridable via `STEMENGINE_GPU_MEMORY_FRACTION`). + +- **OOM retry in `_run_mel_band_karaoke`**: Wrapped `sep.separate_into_parts(...)` in a 2-attempt loop. On the first `out of memory` error: clears CUDA cache, updates job progress ("GPU OOM — retrying with batch_size=1…"), and retries with `batch_size=1`. If the second attempt also fails, the exception propagates normally to the top-level CUDA error handler. + +--- + +### MelBand Karaoke OOM fix (second pass) — revert VRAM cap; add pre-load flush + +**Problem:** After wiring `_apply_cuda_limits()` into `run_separation()`, users saw OOM even with 4 GB physically free on the GPU. Root cause: `set_per_process_memory_fraction(0.80)` caps the process at 9.6 GB. PyTorch's allocator holds a reserved cache from prior jobs inside that budget; within the remaining headroom, only 1.95 GB was truly free — not enough for the 1.04 GiB allocation the separator needed. The GPU monitor showed 4 GB free because it counts PyTorch's cache as "available" while CUDA counts it as "in use." + +**Fix:** + +- **`run_separation()` now flushes cache BEFORE applying the VRAM cap**: `gc.collect() + torch.cuda.empty_cache()` runs first to release stale allocations from prior jobs, then `_apply_cuda_limits()` is called. The order matters: if stale cached blocks are counted against the cap before they're released, the cap appears tighter than it actually is. On this WSL2 system ~2 GB is permanently reserved for Windows (DWM); the 80% cap on 12 GB = 9.6 GB leaves that headroom for Windows processes. Flushing first ensures the separator starts with a clean budget within the cap. + +- **Added `gc.collect() + torch.cuda.empty_cache()` flush in `_run_mel_band_karaoke`** immediately before constructing the separator (which loads the model checkpoint). This second flush is necessary because allocations can accumulate between the start of `run_separation()` and when the model actually loads. + +--- + +### BSRoFormer wired into power settings (parity with MelBand Karaoke) + +Audit of `_run_bsroformer`, `_run_bsroformer_chorus`, `_run_bsroformer_sw` revealed none had the pre-load VRAM flush or OOM retry that `_run_mel_band_karaoke` received. Applied the same pattern to all three: + +- **Pre-load flush**: `gc.collect() + torch.cuda.empty_cache()` added before the separator object is constructed in each runner. +- **OOM retry with batch_size=1**: Added to `_run_bsroformer` and `_run_bsroformer_chorus` (2-attempt loop, same as MelBand). `_run_bsroformer_sw` gets flush only — it delegates to the `audio-separator` library which doesn't expose a `batch_size` parameter. +- **`strict=False` on `load_state_dict`**: Applied to `bs_roformer_separator.py` and `bs_roformer_chorus_separator.py` with the same warning pattern as `mel_band_specialized_separator.py` — guards against future checkpoint/library version mismatches causing hard crashes. + +--- + +### 2026-03-09 — Power settings wired into all remaining 10 separator runners + +Extended the "power settings" pattern (pre-load VRAM flush + OOM retry + `strict=False`) to every remaining runner and separator that was missing it. + +**`separator_service.py` — pre-load gc+empty_cache flush added** before the separator object is constructed in: +- `_run_mdxnet` +- `_run_mel_band_roformer` (also gets OOM retry) +- `_run_scnet` (also gets OOM retry) +- `_run_medleyvox` +- `_run_apollo` +- `_run_aspiration` +- `_run_bandit` +- `_run_ensemble` +- `_run_sepacap` +- `_run_sepreformer` + +**OOM retry loops added:** +- `_run_mel_band_roformer`: 2-attempt loop; on first OOM, gc+cache flush then retry with `batch_size_override=1`. +- `_run_scnet`: 2-attempt loop; on first OOM, gc+cache flush then retry with `chunk_size_override = (original or 262144) // 2`. + +**`strict=False` on `load_state_dict` with warning** applied to: +- `mel_band_roformer_separator.py` (line 219): protects against missing keys (e.g. `static_alpha`) when library is newer than checkpoint. +- `scnet_separator.py` (line 249): same guard. + +--- + +### 2026-03-09 — Fix post-cancel VRAM leak and wrong job status after mid-run cancellation + +**Root cause diagnosis:** +- Cancellation is cooperative (flag-based). Once `separate_into_parts()` starts GPU compute, it runs to completion even after cancel — the flag is only checked between major stages. This is why the GPU keeps working after cancel; it's a fundamental limitation of thread-based execution (Demucs avoids this by using a subprocess that can be killed). +- When a separator's internal `should_cancel()` check fires mid-run, it raises `RuntimeError("Cancelled by user")`. This propagated to `run_separation`'s top-level `except Exception` block, where it didn't match `"CUDA"`, so: (a) no VRAM cleanup was done, and (b) the job was stored with status=ERROR instead of CANCELLED. The model's VRAM wasn't freed until Python's GC got around to it — which could be after the next job's model load begins → OOM despite apparent free VRAM. + +**Fixes in `separator_service.py` `run_separation()`:** +1. **Cancellation branch in `except`**: Added `if jm.should_cancel(job_id) or "Cancelled by user" in err_str` as the first condition — sets status=CANCELLED (not ERROR) and skips the error message path. +2. **`finally` block**: Added after the `except` block — always runs `gc.collect() + torch.cuda.empty_cache()` regardless of how the job exits (success, error, or cancel). Ensures VRAM is returned to the CUDA driver before the next job starts. + +--- + +### 2026-03-09 — Suppress noisy third-party FutureWarnings and demote missing-keys log to DEBUG + +**`main.py`**: Added `warnings.filterwarnings("ignore", ...)` at startup to suppress FutureWarnings from `rotary_embedding_torch` (`torch.cuda.amp.autocast` deprecated) and `contextlib` (`torch.backends.cuda.sdp_kernel` deprecated). Both are upstream issues in third-party packages; the warnings are harmless. + +**All 5 separator files** (`mel_band_specialized_separator.py`, `mel_band_roformer_separator.py`, `scnet_separator.py`, `bs_roformer_separator.py`, `bs_roformer_chorus_separator.py`): Demoted the "Checkpoint is missing N key(s)" log from WARNING to DEBUG. With up to 498 missing keys this was very noisy; the condition is expected and harmless (library newer than checkpoint, missing params use default init). + +--- + +### 2026-03-09 — GPU power reduction: nvidia-smi power limit + TF32 + float16 autocast + +**Goal:** Reduce GPU power draw (and fan noise) during inference. + +**`lyric_transcriber_service.py` `_apply_cuda_limits()`** — three additions: +1. **`nvidia-smi --power-limit`**: Caps hardware TDP. Default 100 W (min for this GPU); override with `STEMENGINE_GPU_POWER_LIMIT` env var. Set to 0 to disable. Silently ignored if the driver rejects (e.g. insufficient privilege). Called once per job at limits-apply time. +2. **TF32 flags**: `torch.backends.cuda.matmul.allow_tf32 = True`, `torch.backends.cudnn.allow_tf32 = True`, `torch.set_float32_matmul_precision('high')` — enables TF32 for matmuls on Ampere+ GPUs; imperceptible quality difference for audio, reduces compute intensity. +3. Both are no-ops if CUDA is unavailable or the call fails. + +**float16 autocast added to 4 separator inference loops** (enabled only when device != cpu): +- `bs_roformer_separator.py`: `torch.inference_mode()` + `torch.amp.autocast('cuda', dtype=torch.float16, enabled=(device != 'cpu'))` +- `mel_band_roformer_separator.py`: same +- `mel_band_specialized_separator.py`: same +- `scnet_separator.py`: `torch.no_grad()` + autocast (`enabled=(device.type == 'cuda')` since device is `torch.device`) + +Autocast runs the model forward pass in float16 (halving memory bandwidth) while keeping the float32 accumulation buffers (`result`, `counter`) unaffected — PyTorch widens float16 outputs back to float32 on the `+=` assignment automatically. + +--- + +### 2026-03-09 — Move GPU power limit to container startup (start.sh + SYS_ADMIN cap) + +**Problem:** `nvidia-smi --power-limit` requires `SYS_ADMIN` capability. The previous implementation called it on every job from `_apply_cuda_limits()`, which lacked the capability and failed silently. + +**Changes:** +- **`start.sh`**: Added `nvidia-smi --power-limit=${STEMENGINE_GPU_POWER_LIMIT:-100}` near the top. Runs once at startup as root before uvicorn. Warns (does not abort) if it fails. +- **`stemengine.yml`**: Added `cap_add: [SYS_ADMIN]` so the container has the driver-level permission to write the power limit via NVML. Also added `STEMENGINE_GPU_POWER_LIMIT=${STEMENGINE_GPU_POWER_LIMIT:-100}` to the environment block (default 100 W; set to 0 to skip). +- **`lyric_transcriber_service.py` `_apply_cuda_limits()`**: Removed the per-job subprocess call; TF32 flags and VRAM fraction cap remain. + +--- + +### 2026-03-09 — Harden SYS_ADMIN: move privilege to init container, main container unprivileged + +**Problem:** SYS_ADMIN in the main container persists for its entire lifetime. Any RCE or path traversal in the app could exploit it. The `start.sh` volume mount (writable by default) was also a vector for injecting privileged commands. + +**Solution — init container pattern:** +- **`stemengine-gpu-init`** service added to `stemengine.yml`: uses `nvidia/cuda:12.1.0-base-ubuntu22.04`, has `cap_add: SYS_ADMIN`, runs `nvidia-smi --power-limit` inline in its entrypoint, then exits (`restart: "no"`). The command is hardcoded in the compose file, not sourced from any mounted file. +- **`stemengine`** service: `cap_add: SYS_ADMIN` removed entirely. Added `depends_on: stemengine-gpu-init: condition: service_completed_successfully` so the main container waits for the init to finish before starting. +- **`start.sh` mount** changed to read-only (`:ro`) — the container can no longer overwrite the startup script. +- **`start.sh`**: GPU power limit block removed (now handled by init container). + +--- + +### 2026-03-09 — Fix stem rename filename format; add rescan-stems endpoint + +**Bug fix — `rename_stem_endpoint` (`routers/projects.py`):** +The rename endpoint built the new filename as `f"{new_name}{ext}"`, producing `vocals.mp3` instead of `Artist - Title (vocals).mp3`. Fixed to use `_stem_filename(project["name"], new_name, old_path.suffix)` — the same helper used during separation. The DB `stem` key stays as the short name (e.g. "vocals"); only the filename on disk and in the files list changes. + +**New endpoint — `POST /api/projects/{project_id}/rescan-stems`:** +Recursively walks `ProjectDir/Stems/` and registers any audio files not already in the separation job's files list. Stem name is inferred from the filename: +- `Artist - Title (vocals).mp3` → stem key "vocals" (parenthesised suffix pattern) +- `vocals.mp3` → stem key "vocals" (fallback: use filename stem directly) +Also strips a leading project-name prefix if present. +Returns `{"added": N, "stems": [...]}`. Idempotent — already-tracked paths are skipped. + +--- + +## 2026-03-09 + +### Fix: ELRC not generated + Syllables not appearing when Word Timestamps disabled + +**Root cause:** `lyricTranscribeProject` in `frontend/src/api/client.ts` computed `defaultFormats` based on `wordTs` (from localStorage `stemengine-whisper-word-ts`). When the user had Word Timestamps disabled (`wordTs=false`), `defaultFormats=['lrc','txt']` — no `'elrc'` — regardless of `lsyncAlign`. Since syllable data lives in the ELRC file, both issues (no ELRC + no visible syllables) were caused by the same missing format. + +**Fix:** Added `lsyncAlign` variable and `effectiveWordTs = wordTs || lsyncAlign`. When `lsyncAlign=true`, `effectiveWordTs` is forced `true`, ensuring `'elrc'` is always included in `defaultFormats` and `word_timestamps: true` is sent to the backend. + +Also reverted an earlier incorrect change that added `'sylrc'` to formats when `lsyncAlign=true` (user confirmed sylrc is not needed; `phonemeAlign` is no longer used). diff --git a/backend/main.py b/backend/main.py index 8a7396e..b240c96 100755 --- a/backend/main.py +++ b/backend/main.py @@ -4,8 +4,24 @@ Mounts all API routers and serves the built React frontend as static files. """ import logging import logging.config +import warnings from pathlib import Path +# Suppress FutureWarnings from third-party libraries that use deprecated +# PyTorch APIs (torch.cuda.amp.autocast, torch.backends.cuda.sdp_kernel). +# These are upstream issues in rotary_embedding_torch and similar packages; +# the warnings are harmless and clutter the logs. +warnings.filterwarnings( + "ignore", + message=r".*torch\.cuda\.amp\.autocast.*", + category=FutureWarning, +) +warnings.filterwarnings( + "ignore", + message=r".*torch\.backends\.cuda\.sdp_kernel.*", + category=FutureWarning, +) + from fastapi import FastAPI # Load logging config from the discrete YAML file next to this package. diff --git a/backend/routers/models.py b/backend/routers/models.py index 9f746e5..e7b9a33 100755 --- a/backend/routers/models.py +++ b/backend/routers/models.py @@ -195,10 +195,10 @@ MODEL_DOWNLOAD_INFO = { "files": ["Dereverb-Room-Anvuew/dereverb_room_anvuew_sdr_13.7432.ckpt", "Dereverb-Room-Anvuew/dereverb_room_anvuew.yaml"], }, "mel_band_karaoke": { - "size_mb": 170, + "size_mb": 1760, "description": "MelBand RoFormer for karaoke - separates lead vocals from backing+instrumental.", "hf_repo": "becruily/mel-band-roformer-karaoke", - "files": ["model.ckpt", "config.yaml"], + "files": ["mel_band_roformer_karaoke_becruily.ckpt", "config_karaoke_becruily.yaml"], }, "mel_band_karaoke_gabox": { "size_mb": 195, diff --git a/backend/routers/projects.py b/backend/routers/projects.py index ccfcf4e..9cd9d5a 100755 --- a/backend/routers/projects.py +++ b/backend/routers/projects.py @@ -576,7 +576,8 @@ class RenameStemRequest(BaseModel): def rename_stem_endpoint(project_id: str, body: RenameStemRequest): """Rename a stem file and update the job record.""" import os - + from ..services.separator_service import _stem_filename + project = _get_project_or_404(project_id) sep_job = jm.get_job(project["sep_job_id"]) if sep_job is None: @@ -586,13 +587,13 @@ def rename_stem_endpoint(project_id: str, body: RenameStemRequest): stem_files = sep_job.get("files", []) stem_file = None stem_idx = None - + for idx, f in enumerate(stem_files): if f.get("stem") == body.stem_name or f.get("name") == body.stem_name: stem_file = f stem_idx = idx break - + if stem_file is None: raise HTTPException(status_code=404, detail=f"Stem not found: {body.stem_name}") @@ -600,13 +601,12 @@ def rename_stem_endpoint(project_id: str, body: RenameStemRequest): if not old_path.exists(): raise HTTPException(status_code=404, detail=f"Stem file not found on disk: {old_path}") - # Generate new filename + # Generate new filename using the standard format: "Project Name (stemname).ext" new_name = body.new_name.strip() if not new_name: raise HTTPException(status_code=400, detail="New name cannot be empty") - - # Keep the same extension - new_filename = f"{new_name}{old_path.suffix}" + + new_filename = _stem_filename(project["name"], new_name, old_path.suffix) new_path = old_path.parent / new_filename # Avoid overwriting existing files @@ -634,6 +634,83 @@ def rename_stem_endpoint(project_id: str, body: RenameStemRequest): raise HTTPException(status_code=500, detail=f"Failed to rename stem: {e}") +# --------------------------------------------------------------------------- +# Stem Rescan +# --------------------------------------------------------------------------- + +def _infer_stem_name(file_stem: str, project_name: str) -> str: + """ + Derive a stem key from a filename stem. + + Handles two formats: + - Standard: "Artist - Title (vocals)" → "vocals" + - Simple: "vocals" → "vocals" + """ + import re + # Standard format: anything ending with " (stemname)" + m = re.search(r'\(([^)]+)\)\s*$', file_stem.strip()) + if m: + return m.group(1).strip().lower() + # Strip a leading project-name prefix if present + lower_stem = file_stem.strip() + lower_proj = project_name.strip() + if lower_stem.lower().startswith(lower_proj.lower()): + remainder = lower_stem[len(lower_proj):].strip(' -_()') + if remainder: + return remainder.lower() + # Fall back: use the whole filename stem as the key + return file_stem.strip().lower() + + +@router.post("/{project_id}/rescan-stems") +def rescan_stems(project_id: str): + """ + Recursively scan the project's Stems directory and register any audio + files not already tracked in the separation job's file list. + + Useful after external renames or manual file additions. Stem names are + inferred from filenames: "Artist - Title (vocals).mp3" → "vocals", + "vocals.mp3" → "vocals". Returns the list of newly added stem entries. + """ + project = _get_project_or_404(project_id) + sep_job = jm.get_job(project["sep_job_id"]) + if sep_job is None: + raise HTTPException(status_code=404, detail="Separation job not found") + + project_dir = _project_output_dir(project) + stems_dir = project_dir / "Stems" + if not stems_dir.exists(): + raise HTTPException(status_code=404, detail="Stems directory not found") + + files = list(sep_job.get("files", [])) + tracked_paths = {f["path"] for f in files} + + added = [] + for audio_file in sorted(stems_dir.rglob("*")): + if not audio_file.is_file(): + continue + if audio_file.suffix.lower() not in _AUDIO_EXTS: + continue + if str(audio_file) in tracked_paths: + continue + + stem_name = _infer_stem_name(audio_file.stem, project["name"]) + entry = { + "name": audio_file.name, + "path": str(audio_file), + "type": "stem", + "stem": stem_name, + } + files.append(entry) + tracked_paths.add(str(audio_file)) + added.append(entry) + + if added: + jm.update_job(project["sep_job_id"], files=files) + + return {"added": len(added), "stems": [a["stem"] for a in added]} + + def _static_label(static_option: Optional[str]) -> str: labels = { "no_vocals": "No Vocals", diff --git a/backend/services/bs_roformer_chorus_separator.py b/backend/services/bs_roformer_chorus_separator.py index 3149f5d..15002e2 100755 --- a/backend/services/bs_roformer_chorus_separator.py +++ b/backend/services/bs_roformer_chorus_separator.py @@ -150,7 +150,13 @@ class BSRoformerChorusSeparator: else: state_dict = checkpoint - model.load_state_dict(state_dict) + result = model.load_state_dict(state_dict, strict=False) + if result.missing_keys: + import logging + logging.getLogger(__name__).debug( + "Checkpoint missing %d key(s) (library newer than checkpoint): %s", + len(result.missing_keys), result.missing_keys[:5], + ) model.to(self.device).eval() return model diff --git a/backend/services/bs_roformer_separator.py b/backend/services/bs_roformer_separator.py index 833a90b..69d48b7 100755 --- a/backend/services/bs_roformer_separator.py +++ b/backend/services/bs_roformer_separator.py @@ -191,7 +191,13 @@ class BSRoformerSeparator: model = BSRoformer(**model_cfg) checkpoint = torch.load(str(self.model_path), map_location='cpu', weights_only=False) - model.load_state_dict(checkpoint) + result = model.load_state_dict(checkpoint, strict=False) + if result.missing_keys: + import logging + logging.getLogger(__name__).debug( + "Checkpoint missing %d key(s) (library newer than checkpoint): %s", + len(result.missing_keys), result.missing_keys[:5], + ) model.to(self.device).eval() return model @@ -229,7 +235,8 @@ class BSRoformerSeparator: S = len(config.training.instruments) req_shape = (S,) + tuple(mix_t.shape) - with torch.inference_mode(): + with torch.inference_mode(), \ + torch.amp.autocast('cuda', dtype=torch.float16, enabled=(device != 'cpu')): result = torch.zeros(req_shape, dtype=torch.float32, device=device) counter = torch.zeros(req_shape, dtype=torch.float32, device=device) batch_data, batch_locs = [], [] diff --git a/backend/services/lyric_transcriber_service.py b/backend/services/lyric_transcriber_service.py index 6f87546..b80d357 100755 --- a/backend/services/lyric_transcriber_service.py +++ b/backend/services/lyric_transcriber_service.py @@ -942,6 +942,8 @@ def _apply_cuda_limits() -> None: Safe to call multiple times — torch guards against re-setting the fraction on an already-initialised context, and os.nice is a no-op when already niced. + + GPU power limit is set once at container startup via start.sh, not here. """ # Lower CPU scheduling priority so Windows UI processes aren't starved. try: @@ -954,6 +956,11 @@ def _apply_cuda_limits() -> None: import torch as _t if _t.cuda.is_available() and 0.0 < _GPU_MEMORY_FRACTION < 1.0: _t.cuda.set_per_process_memory_fraction(_GPU_MEMORY_FRACTION) + # Allow TF32 for matmuls — imperceptible quality difference for audio + # separation but reduces compute intensity and power draw on Ampere+ GPUs. + _t.backends.cuda.matmul.allow_tf32 = True + _t.backends.cudnn.allow_tf32 = True + _t.set_float32_matmul_precision('high') except Exception: pass @@ -1172,8 +1179,10 @@ def _fill_transcript_gaps( g_end = segments[i + 1]["start"] if g_end - g_start > min_gap: gaps.append((g_start, g_end)) - if total_dur - segments[-1]["end"] > min_gap: - gaps.append((segments[-1]["end"], total_dur)) + # Trailing silence after the last segment is intentionally excluded. + # Whisper hallucinates on silent/ambient audio tails (YouTube-trained + # subscription prompts, "Thank you", Japanese sign-offs, etc.). If the + # first pass with VAD already missed that region, it contains no lyrics. if not gaps: return segments @@ -1343,11 +1352,12 @@ def run_lyric_transcription( audio_path = _ensure_vocals(audio_path, job_id) # Music-tuned ASR options: - # no_speech_threshold 0.45 (< default 0.60) — keep more segments + # no_speech_threshold 0.50 (= Whisper default 0.60; 0.45 was too low + # and caused Whisper to treat silence as speech, triggering hallucinations) # compression_ratio_threshold 2.2 (< default 2.4) — reject repetition sooner # hallucination_silence_threshold — suppress long silence tokens (3.8+) _asr_opts: dict = { - "no_speech_threshold": 0.45, + "no_speech_threshold": 0.50, "condition_on_previous_text": False, "compression_ratio_threshold": 2.2, "temperatures": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0], diff --git a/backend/services/mdxnet_separator.py b/backend/services/mdxnet_separator.py index 42bfedb..67f4825 100755 --- a/backend/services/mdxnet_separator.py +++ b/backend/services/mdxnet_separator.py @@ -75,9 +75,8 @@ MDXNET_MODELS = { # MDX-Net specific constants HOP_LENGTH = 1024 -N_FFT = 2048 -DIM_F = 3072 -DIM_T = 8 +# DIM_F and DIM_T are model-specific and are read from the ONNX input shape at runtime. +# N_FFT = DIM_F * 2 → produces DIM_F+1 freq bins; top bin is cropped before ONNX inference. def _load_audio(path: str, sample_rate: int = SAMPLE_RATE) -> np.ndarray: @@ -139,77 +138,113 @@ def _save_audio(path: str, data: np.ndarray, output_format: str, bitrate: str = os.remove(tmp_path) -def _stft(wave: np.ndarray, n_fft: int = N_FFT, hop_length: int = HOP_LENGTH) -> np.ndarray: - """Compute STFT of audio.""" +def _stft(wave: np.ndarray, n_fft: int = 4096, hop_length: int = HOP_LENGTH) -> np.ndarray: + """Compute STFT of audio. + + np.stft/np.istft never existed; use librosa which shares the same + n_fft/hop_length parameter names. librosa.stft returns a complex + (freq, time) array; we pack it as (freq, time, 2) real/imag to match + the shape contract expected by _spec_to_batch. + """ + import librosa # wave shape: (channels, samples) channels = wave.shape[0] specs = [] for ch in range(channels): - spec = np.stft(wave[ch], n_fft=n_fft, hop_length=hop_length, window='hann') - specs.append(spec) + spec = librosa.stft(wave[ch], n_fft=n_fft, hop_length=hop_length, window='hann') + # complex (freq, time) → (freq, time, 2) real/imag + specs.append(np.stack([spec.real, spec.imag], axis=-1)) return np.array(specs) # (channels, freq, time, 2) where last dim is real/imag -def _istft(spec: np.ndarray, n_fft: int = N_FFT, hop_length: int = HOP_LENGTH) -> np.ndarray: +def _istft(spec: np.ndarray, n_fft: int = 4096, hop_length: int = HOP_LENGTH) -> np.ndarray: """Compute inverse STFT.""" + import librosa channels = spec.shape[0] waves = [] for ch in range(channels): - wave = np.istft(spec[ch], n_fft=n_fft, hop_length=hop_length, window='hann') + # (freq, time, 2) real/imag → complex (freq, time) + complex_spec = spec[ch, :, :, 0] + 1j * spec[ch, :, :, 1] + wave = librosa.istft(complex_spec, n_fft=n_fft, hop_length=hop_length, window='hann') waves.append(wave) return np.array(waves) -def _spec_to_batch(spec: np.ndarray, dim_f: int = DIM_F, dim_t: int = DIM_T) -> Tuple[np.ndarray, dict]: - """Convert spectrogram to batch format for MDX-Net.""" - # spec shape: (channels, freq, time, 2) - b, f, t, c = spec.shape - - # Pad if necessary - pad_f = dim_f - f if f < dim_f else 0 - pad_t = dim_t - (t % dim_t) if t % dim_t != 0 else 0 - - if pad_f > 0 or pad_t > 0: - spec = np.pad(spec, ((0, 0), (0, pad_f), (0, pad_t), (0, 0))) - - # Reshape to batches - spec = spec.transpose(0, 3, 1, 2) # (channels, 2, freq, time) - - # Split into time chunks - t_chunks = spec.shape[-1] // dim_t - spec = spec.reshape(b, c, dim_f, t_chunks, dim_t) - spec = spec.transpose(0, 3, 1, 2, 4) # (b, t_chunks, c, f, t) - spec = spec.reshape(-1, c, dim_f, dim_t) - +def _spec_to_batch(spec: np.ndarray, dim_f: int = 2048, dim_t: int = 256) -> Tuple[np.ndarray, dict]: + """Convert spectrogram to batch format for MDX-Net. + + Input spec: (stereo=2, freq, time, complex=2) + Output batch: (t_chunks, 4, dim_f, dim_t) + where the 4 channels are (L_real, L_imag, R_real, R_imag); freq is + cropped to dim_f; time is padded to a multiple of dim_t. + """ + _, freq, time, _ = spec.shape # (2, freq, time, 2) + + # Pad time to a multiple of dim_t + pad_t = (dim_t - time % dim_t) % dim_t + if pad_t > 0: + spec = np.pad(spec, ((0, 0), (0, 0), (0, pad_t), (0, 0))) + + # Pad freq to dim_f if necessary (normally freq >= dim_f) + pad_f = max(0, dim_f - freq) + if pad_f > 0: + spec = np.pad(spec, ((0, 0), (0, pad_f), (0, 0), (0, 0))) + + # Crop freq to exactly dim_f + spec = spec[:, :dim_f, :, :] # (2, dim_f, time_padded, 2) + time_padded = time + pad_t + t_chunks = time_padded // dim_t + + # Merge stereo(2) × complex(2) → 4 channels + spec = spec.transpose(0, 3, 1, 2) # (2, 2, dim_f, time_padded) + spec = spec.reshape(4, dim_f, time_padded) # (4, dim_f, time_padded) + + # Chunk time dimension + spec = spec.reshape(4, dim_f, t_chunks, dim_t) + spec = spec.transpose(2, 0, 1, 3) # (t_chunks, 4, dim_f, dim_t) + metadata = { - 'original_shape': (b, f, t, c), - 'pad_f': pad_f, + 'freq': freq, + 'time': time, 'pad_t': pad_t, 't_chunks': t_chunks, + 'dim_f': dim_f, + 'dim_t': dim_t, } - + return spec, metadata def _batch_to_spec(batch: np.ndarray, metadata: dict) -> np.ndarray: - """Convert batch format back to spectrogram.""" - b, f, t, c = metadata['original_shape'] - pad_f = metadata['pad_f'] + """Convert batch format back to spectrogram. + + Input batch: (t_chunks, 4, dim_f, dim_t) + Output spec: (stereo=2, freq, time, complex=2) + """ + freq = metadata['freq'] + time = metadata['time'] pad_t = metadata['pad_t'] t_chunks = metadata['t_chunks'] - - # Reshape back - batch = batch.reshape(b, t_chunks, c, DIM_F, DIM_T) - batch = batch.transpose(0, 2, 3, 1, 4) # (b, c, f, t_chunks, t) - batch = batch.reshape(b, c, DIM_F, t_chunks * DIM_T) - - # Remove padding - spec = batch.transpose(0, 2, 3, 1) # (b, f, t, c) - if pad_f > 0: - spec = spec[:, :f, :, :] + dim_f = metadata['dim_f'] + dim_t = metadata['dim_t'] + + # Reassemble time chunks + batch = batch.transpose(1, 2, 0, 3) # (4, dim_f, t_chunks, dim_t) + batch = batch.reshape(4, dim_f, t_chunks * dim_t) # (4, dim_f, time_padded) + + # Remove time padding if pad_t > 0: - spec = spec[:, :, :t, :] - + batch = batch[:, :, :time] # (4, dim_f, time) + + # Restore freq bins cropped before ONNX (pad back to original freq) + if freq > dim_f: + zeros = np.zeros((4, freq - dim_f, time), dtype=batch.dtype) + batch = np.concatenate([batch, zeros], axis=1) # (4, freq, time) + + # Split 4 channels back to stereo(2) × complex(2) + batch = batch.reshape(2, 2, freq, time) # (stereo, complex, freq, time) + spec = batch.transpose(0, 2, 3, 1) # (2, freq, time, 2) + return spec @@ -243,6 +278,9 @@ class MDXNetSeparator: self.stems = self.model_config['stems'] self.target_stem = self.model_config['target_stem'] self._session = None + self.dim_f = None + self.dim_t = None + self.n_fft = None def _ensure_model(self): """Download model if not present.""" @@ -264,50 +302,65 @@ class MDXNetSeparator: """Get or create ONNX runtime session.""" if self._session is None: self._ensure_model() - + providers = ['CPUExecutionProvider'] if not self.device == 'cpu' and 'CUDAExecutionProvider' in ort.get_available_providers(): providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'] - + self._session = ort.InferenceSession( str(self.model_path), providers=providers ) + + # Read model-specific dimensions from the ONNX input shape. + # Input shape: (batch_size, 4, dim_f, dim_t) — varies per model file. + input_shape = self._session.get_inputs()[0].shape + self.dim_f = input_shape[2] + self.dim_t = input_shape[3] + # n_fft * 2 gives dim_f+1 freq bins; the top bin is cropped before inference. + self.n_fft = self.dim_f * 2 + return self._session def _demix(self, mix: np.ndarray) -> Dict[str, np.ndarray]: """Separate audio into stems using MDX-Net.""" session = self._get_session() input_name = session.get_inputs()[0].name - - # Compute STFT - mix_spec = _stft(mix) # (channels, freq, time, 2) - - # Convert to batch format - mix_batch, metadata = _spec_to_batch(mix_spec) - + + # Compute STFT using model-specific n_fft (= dim_f * 2) + mix_spec = _stft(mix, n_fft=self.n_fft) # (2, freq, time, 2) + + # Convert to batch format using model-specific dim_f / dim_t + mix_batch, metadata = _spec_to_batch(mix_spec, dim_f=self.dim_f, dim_t=self.dim_t) + # Run inference in chunks output_batches = [] batch_size = 1 # Process one chunk at a time for memory efficiency - + with tqdm(total=len(mix_batch), desc='Separating', unit='chunk', ncols=100) as pbar: for i in range(0, len(mix_batch), batch_size): batch = mix_batch[i:i + batch_size] - # ONNX expects (batch, channels, freq, time) outputs = session.run(None, {input_name: batch.astype(np.float32)}) output_batches.append(outputs[0]) pbar.update(len(batch)) - + # Concatenate outputs target_spec = np.concatenate(output_batches, axis=0) - + # Convert back to spectrogram format target_spec = _batch_to_spec(target_spec, metadata) - + # Inverse STFT to get audio - target_audio = _istft(target_spec) + target_audio = _istft(target_spec, n_fft=self.n_fft) - # Calculate the other stem (residual) + # Calculate the other stem (residual). + # ISTFT may return a slightly different length than the original mix; + # trim or zero-pad to match before subtracting. + n = mix.shape[1] + if target_audio.shape[1] > n: + target_audio = target_audio[:, :n] + elif target_audio.shape[1] < n: + target_audio = np.pad(target_audio, ((0, 0), (0, n - target_audio.shape[1]))) other_audio = mix - target_audio # Create output dict based on stems configuration diff --git a/backend/services/mel_band_roformer_separator.py b/backend/services/mel_band_roformer_separator.py index eb6296d..fb76e44 100755 --- a/backend/services/mel_band_roformer_separator.py +++ b/backend/services/mel_band_roformer_separator.py @@ -216,7 +216,14 @@ class MelBandRoformerSeparator: else: state_dict = checkpoint - model.load_state_dict(state_dict) + result = model.load_state_dict(state_dict, strict=False) + if result.missing_keys: + import logging + logging.getLogger(__name__).debug( + "Checkpoint is missing %d key(s) (library newer than checkpoint; " + "affected params use default init): %s", + len(result.missing_keys), result.missing_keys[:5], + ) model.to(self.device).eval() return model @@ -259,7 +266,8 @@ class MelBandRoformerSeparator: num_stems = len(stems) req_shape = (num_stems,) + tuple(mix_t.shape) - with torch.inference_mode(): + with torch.inference_mode(), \ + torch.amp.autocast('cuda', dtype=torch.float16, enabled=(device != 'cpu')): result = torch.zeros(req_shape, dtype=torch.float32, device=device) counter = torch.zeros(req_shape, dtype=torch.float32, device=device) batch_data, batch_locs = [], [] diff --git a/backend/services/mel_band_specialized_separator.py b/backend/services/mel_band_specialized_separator.py index 615a3a7..355f5e1 100755 --- a/backend/services/mel_band_specialized_separator.py +++ b/backend/services/mel_band_specialized_separator.py @@ -143,7 +143,14 @@ class MelBandSeparatorBase: else: state_dict = checkpoint - model.load_state_dict(state_dict) + result = model.load_state_dict(state_dict, strict=False) + if result.missing_keys: + import logging + logging.getLogger(__name__).debug( + "Checkpoint is missing %d key(s) (library newer than checkpoint; " + "affected params use default init): %s", + len(result.missing_keys), result.missing_keys[:5], + ) model.to(self.device).eval() return model @@ -180,7 +187,8 @@ class MelBandSeparatorBase: num_stems = len(self.stems) req_shape = (num_stems,) + tuple(mix_t.shape) - with torch.inference_mode(): + with torch.inference_mode(), \ + torch.amp.autocast('cuda', dtype=torch.float16, enabled=(device != 'cpu')): result = torch.zeros(req_shape, dtype=torch.float32, device=device) counter = torch.zeros(req_shape, dtype=torch.float32, device=device) batch_data, batch_locs = [], [] @@ -350,8 +358,8 @@ class MelBandKaraokeSeparator(MelBandSeparatorBase): mp3_bitrate: str = '256k', ): model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/karaoke') - model_path = model_dir / 'model.ckpt' - config_path = model_dir / 'config.yaml' + model_path = model_dir / 'mel_band_roformer_karaoke_becruily.ckpt' + config_path = model_dir / 'config_karaoke_becruily.yaml' self._ensure_model(model_dir, model_path, config_path) diff --git a/backend/services/scnet_separator.py b/backend/services/scnet_separator.py index f659f1f..53392eb 100755 --- a/backend/services/scnet_separator.py +++ b/backend/services/scnet_separator.py @@ -127,7 +127,8 @@ def _demix( sources_sum: Optional[np.ndarray] = None weight_sum = np.zeros(mix_padded.shape[-1], dtype=np.float32) - with torch.no_grad(): + with torch.no_grad(), \ + torch.amp.autocast('cuda', dtype=torch.float16, enabled=(device.type == 'cuda')): for start in range(0, N, step): p_end = start + chunk_size if p_end > mix_padded.shape[-1]: @@ -246,7 +247,14 @@ class SCNetSeparator: state = torch.load(str(model_path), map_location="cpu", weights_only=False) if "state_dict" in state: state = state["state_dict"] - model.load_state_dict(state) + result = model.load_state_dict(state, strict=False) + if result.missing_keys: + import logging + logging.getLogger(__name__).debug( + "Checkpoint is missing %d key(s) (library newer than checkpoint; " + "affected params use default init): %s", + len(result.missing_keys), result.missing_keys[:5], + ) model.eval() self._model = model.to(self.device) diff --git a/backend/services/separator_service.py b/backend/services/separator_service.py index d43dba3..fce40f1 100755 --- a/backend/services/separator_service.py +++ b/backend/services/separator_service.py @@ -301,14 +301,24 @@ def run_separation( _roformer_opts = _power['roformer'] _demucs_opts = _power['demucs'] - # Free any cached GPU memory from previous jobs before loading a new model. - # This reduces OOM risk when Whisper or a prior separation left allocations behind. + # Flush PyTorch's allocator cache BEFORE applying the VRAM cap. + # On WSL2 the system reserves ~2 GB for Windows (DWM, etc.), so the effective + # usable VRAM is total - 2 GB. _apply_cuda_limits() caps the process at 80% of + # the GPU's *reported* total, which leaves headroom for Windows. + # The flush must come first: if stale cached allocations are still counted against + # the cap from a prior job, the cap appears tighter than it actually is. try: - import torch + import gc, torch + gc.collect() if torch.cuda.is_available() and not CPU_SEPARATION: torch.cuda.empty_cache() except Exception: pass + try: + from .lyric_transcriber_service import _apply_cuda_limits + _apply_cuda_limits() + except Exception: + pass # Run separation try: @@ -591,7 +601,12 @@ def run_separation( except Exception as exc: err_str = str(exc) - if "CUDA" in err_str or "cuda" in err_str: + if jm.should_cancel(job_id) or "Cancelled by user" in err_str: + # Cancellation propagated as an exception from inside a separator. + # Treat it as a clean cancel, not an error. + log.debug("[%s] separation cancelled mid-run", job_id) + jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.") + elif "CUDA" in err_str or "cuda" in err_str: # CUDA context is corrupted (typically after a prior OOM). # Auto-reset clears allocations and re-enables GPU for the next attempt. try: @@ -613,6 +628,18 @@ def run_separation( else: log.debug("[%s] separation failed — %s: %s", job_id, type(exc).__name__, exc) jm.update_job(job_id, status=jm.JobStatus.ERROR, error=err_str, progress="Error.") + finally: + # Always flush PyTorch's VRAM cache after a job ends (success, error, or + # cancel) so the next job starts with maximum headroom. The model object + # goes out of scope here; gc.collect() ensures its CUDA tensors are + # released before empty_cache() returns freed blocks to the driver. + try: + import gc, torch + gc.collect() + if torch.cuda.is_available() and not CPU_SEPARATION: + torch.cuda.empty_cache() + except Exception: + pass def _get_muted_stems(static_option: Optional[str]) -> List[str]: @@ -781,6 +808,14 @@ def _run_bsroformer_chorus( if jm.should_cancel(job_id): raise RuntimeError("Cancelled by user") + try: + import gc, torch + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + sep = BSRoformerChorusSeparator( cpu_separation=CPU_SEPARATION, output_format=output_format, @@ -792,37 +827,55 @@ def _run_bsroformer_chorus( temp_out = JOBS_DIR / job_id / "bsr_chorus_out" temp_out.mkdir(parents=True, exist_ok=True) + _bs = batch_size_override + for _attempt in range(2): + try: + stem_paths = sep.separate_into_parts( + input_path, str(temp_out), + batch_size_override=_bs, + num_overlap_override=num_overlap_override, + ) + break + except Exception as _exc: + _is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__ + if _is_oom and _attempt == 0: + import gc, torch + jm.update_job(job_id, progress="GPU OOM — retrying with batch_size=1...") + gc.collect() + torch.cuda.empty_cache() + _bs = 1 + continue + try: + shutil.rmtree(temp_out, ignore_errors=True) + except Exception: + pass + raise + + if jm.should_cancel(job_id): + raise RuntimeError("Cancelled by user") + + _prefix = stem_prefix or Path(input_path).stem + stem_files = {} + try: - stem_paths = sep.separate_into_parts( - input_path, str(temp_out), - batch_size_override=batch_size_override, - num_overlap_override=num_overlap_override, - ) - - if jm.should_cancel(job_id): - raise RuntimeError("Cancelled by user") - - _prefix = stem_prefix or Path(input_path).stem - stem_files = {} - for stem_name, path in sorted(stem_paths.items()): src = Path(path) dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix) shutil.move(str(src), str(dest)) copy_metadata(input_path, str(dest), stem_label=stem_name.capitalize(), copy_lyrics=True) stem_files[stem_name.lower()] = str(dest) - + try: shutil.rmtree(temp_out, ignore_errors=True) - except: + except Exception: pass - + return stem_files - + except Exception: try: shutil.rmtree(temp_out, ignore_errors=True) - except: + except Exception: pass raise @@ -1026,37 +1079,66 @@ def _run_mel_band_karaoke( job_id, status=jm.JobStatus.RUNNING, progress=f"Loading {model_name} model...", ) - + if jm.should_cancel(job_id): raise RuntimeError("Cancelled by user") - + + # Flush allocator cache immediately before loading the model checkpoint. + # PyTorch holds reserved-but-idle blocks from previous jobs; releasing them + # here maximises contiguous VRAM available for the ~1.7 GB checkpoint. + try: + import gc, torch + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + sep = sep_class( cpu_separation=CPU_SEPARATION, output_format=output_format, mp3_bitrate=mp3_bitrate, ) - + jm.update_job(job_id, progress="Separating lead/backing vocals...") temp_out = JOBS_DIR / job_id / "mel_band_karaoke_out" temp_out.mkdir(parents=True, exist_ok=True) + # OOM retry: on first out-of-memory, clear cache and retry with batch_size=1. + _bs = batch_size_override + for _attempt in range(2): + try: + stem_paths = sep.separate_into_parts( + input_path, str(temp_out), + batch_size_override=_bs, + num_overlap_override=num_overlap_override, + ) + break # success + except Exception as _exc: + _is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__ + if _is_oom and _attempt == 0: + import gc, torch + jm.update_job(job_id, progress="GPU OOM — retrying with batch_size=1...") + gc.collect() + torch.cuda.empty_cache() + _bs = 1 + continue + try: + shutil.rmtree(temp_out, ignore_errors=True) + except Exception: + pass + raise + + if jm.should_cancel(job_id): + raise RuntimeError("Cancelled by user") + + _prefix = stem_prefix or Path(input_path).stem + stem_files = {} + try: - stem_paths = sep.separate_into_parts( - input_path, str(temp_out), - batch_size_override=batch_size_override, - num_overlap_override=num_overlap_override, - ) - - if jm.should_cancel(job_id): - raise RuntimeError("Cancelled by user") - - _prefix = stem_prefix or Path(input_path).stem - stem_files = {} - for stem_name, path in sorted(stem_paths.items()): src = Path(path) - # Map internal names to display names display_name = stem_name.replace('_', ' ').title() dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix) shutil.move(str(src), str(dest)) @@ -1065,7 +1147,7 @@ def _run_mel_band_karaoke( try: shutil.rmtree(temp_out, ignore_errors=True) - except: + except Exception: pass return stem_files @@ -1073,7 +1155,7 @@ def _run_mel_band_karaoke( except Exception: try: shutil.rmtree(temp_out, ignore_errors=True) - except: + except Exception: pass raise @@ -1105,7 +1187,15 @@ def _run_mdxnet( if jm.should_cancel(job_id): raise RuntimeError("Cancelled by user") - + + try: + import gc, torch + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + sep = MDXNetSeparator( model_id=model, cpu_separation=CPU_SEPARATION, @@ -1178,53 +1268,78 @@ def _run_mel_band_roformer( if jm.should_cancel(job_id): raise RuntimeError("Cancelled by user") - + + try: + import gc, torch + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + sep = MelBandRoformerSeparator( cpu_separation=CPU_SEPARATION, output_format=output_format, mp3_bitrate=mp3_bitrate, ) - + jm.update_job(job_id, progress="Separating with MelBand RoFormer... (this may take a while)") # Use a temp directory then move to project temp_out = JOBS_DIR / job_id / "mel_band_out" temp_out.mkdir(parents=True, exist_ok=True) + _bs = batch_size_override + for _attempt in range(2): + try: + stem_paths = sep.separate_into_parts( + input_path, str(temp_out), + batch_size_override=_bs, + num_overlap_override=num_overlap_override, + ) + break + except Exception as _exc: + _is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__ + if _is_oom and _attempt == 0: + import gc, torch + jm.update_job(job_id, progress="GPU OOM — retrying with batch_size=1...") + gc.collect() + torch.cuda.empty_cache() + _bs = 1 + continue + try: + shutil.rmtree(temp_out, ignore_errors=True) + except Exception: + pass + raise + + if jm.should_cancel(job_id): + raise RuntimeError("Cancelled by user") + + # Move to project stems folder with proper naming + _prefix = stem_prefix or Path(input_path).stem + stem_files = {} + try: - stem_paths = sep.separate_into_parts( - input_path, str(temp_out), - batch_size_override=batch_size_override, - num_overlap_override=num_overlap_override, - ) - - if jm.should_cancel(job_id): - raise RuntimeError("Cancelled by user") - - # Move to project stems folder with proper naming - _prefix = stem_prefix or Path(input_path).stem - stem_files = {} - for stem_name, path in sorted(stem_paths.items()): src = Path(path) dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix) shutil.move(str(src), str(dest)) copy_metadata(input_path, str(dest), stem_label=stem_name.capitalize(), copy_lyrics=True) stem_files[stem_name.lower()] = str(dest) - + # Cleanup temp directory try: shutil.rmtree(temp_out, ignore_errors=True) - except: + except Exception: pass - + return stem_files - + except Exception: - # Cleanup on error try: shutil.rmtree(temp_out, ignore_errors=True) - except: + except Exception: pass raise @@ -1258,54 +1373,76 @@ def _run_bsroformer( if jm.should_cancel(job_id): raise RuntimeError("Cancelled by user") - + + try: + import gc, torch + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + sep = BSRoformerSeparator( cpu_separation=CPU_SEPARATION, output_format=output_format, mp3_bitrate=mp3_bitrate, stem_mode=stem_mode, ) - + jm.update_job(job_id, progress="Separating with BS-RoFormer... (this may take a while)") - # Use a temp directory then move to project temp_out = JOBS_DIR / job_id / "bsr_out" temp_out.mkdir(parents=True, exist_ok=True) + _bs = batch_size_override + for _attempt in range(2): + try: + stem_paths = sep.separate_into_parts( + input_path, str(temp_out), + batch_size_override=_bs, + num_overlap_override=num_overlap_override, + ) + break + except Exception as _exc: + _is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__ + if _is_oom and _attempt == 0: + import gc, torch + jm.update_job(job_id, progress="GPU OOM — retrying with batch_size=1...") + gc.collect() + torch.cuda.empty_cache() + _bs = 1 + continue + try: + shutil.rmtree(temp_out, ignore_errors=True) + except Exception: + pass + raise + + if jm.should_cancel(job_id): + raise RuntimeError("Cancelled by user") + + _prefix = stem_prefix or Path(input_path).stem + stem_files = {} + try: - stem_paths = sep.separate_into_parts( - input_path, str(temp_out), - batch_size_override=batch_size_override, - num_overlap_override=num_overlap_override, - ) - - if jm.should_cancel(job_id): - raise RuntimeError("Cancelled by user") - - # Move to project stems folder with proper naming - _prefix = stem_prefix or Path(input_path).stem - stem_files = {} - for stem_name, path in sorted(stem_paths.items()): src = Path(path) dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix) shutil.move(str(src), str(dest)) copy_metadata(input_path, str(dest), stem_label=stem_name.capitalize(), copy_lyrics=True) stem_files[stem_name.lower()] = str(dest) - - # Cleanup temp directory + try: shutil.rmtree(temp_out, ignore_errors=True) - except: + except Exception: pass - + return stem_files - + except Exception: - # Cleanup on error try: shutil.rmtree(temp_out, ignore_errors=True) - except: + except Exception: pass raise @@ -1410,6 +1547,14 @@ def _run_scnet( if jm.should_cancel(job_id): raise RuntimeError("Cancelled by user") + try: + import gc, torch + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + sep = SCNetSeparator( model_id=model, cpu_separation=CPU_SEPARATION, @@ -1422,19 +1567,37 @@ def _run_scnet( temp_out = JOBS_DIR / job_id / "scnet_out" temp_out.mkdir(parents=True, exist_ok=True) + _cs = chunk_size_override + for _attempt in range(2): + try: + stem_paths = sep.separate_into_parts( + input_path, str(temp_out), + chunk_size_override=_cs, + num_overlap_override=num_overlap_override, + ) + break + except Exception as _exc: + _is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__ + if _is_oom and _attempt == 0: + import gc, torch + jm.update_job(job_id, progress="GPU OOM — retrying with smaller chunk size...") + gc.collect() + torch.cuda.empty_cache() + _cs = (_cs or 262144) // 2 + continue + try: + shutil.rmtree(temp_out, ignore_errors=True) + except Exception: + pass + raise + + if jm.should_cancel(job_id): + raise RuntimeError("Cancelled by user") + + _prefix = stem_prefix or Path(input_path).stem + stem_files = {} + try: - stem_paths = sep.separate_into_parts( - input_path, str(temp_out), - chunk_size_override=chunk_size_override, - num_overlap_override=num_overlap_override, - ) - - if jm.should_cancel(job_id): - raise RuntimeError("Cancelled by user") - - _prefix = stem_prefix or Path(input_path).stem - stem_files = {} - for stem_name, path in sorted(stem_paths.items()): src = Path(path) display_name = stem_name.replace('_', ' ').title() @@ -1442,18 +1605,18 @@ def _run_scnet( shutil.move(str(src), str(dest)) copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True) stem_files[stem_name.lower()] = str(dest) - + try: shutil.rmtree(temp_out, ignore_errors=True) - except: + except Exception: pass - + return stem_files - + except Exception: try: shutil.rmtree(temp_out, ignore_errors=True) - except: + except Exception: pass raise @@ -1482,7 +1645,15 @@ def _run_bsroformer_sw( if jm.should_cancel(job_id): raise RuntimeError("Cancelled by user") - + + try: + import gc, torch + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + sep = BSRoformerSWSeparator( cpu_separation=CPU_SEPARATION, output_format=output_format, @@ -1550,7 +1721,15 @@ def _run_medleyvox( if jm.should_cancel(job_id): raise RuntimeError("Cancelled by user") - + + try: + import gc, torch + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + sep = MedleyVoxSeparator( model_id=model, cpu_separation=CPU_SEPARATION, @@ -1619,7 +1798,15 @@ def _run_apollo( if jm.should_cancel(job_id): raise RuntimeError("Cancelled by user") - + + try: + import gc, torch + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + sep = ApolloSeparator( model_id=model, cpu_separation=CPU_SEPARATION, @@ -1688,7 +1875,15 @@ def _run_aspiration( if jm.should_cancel(job_id): raise RuntimeError("Cancelled by user") - + + try: + import gc, torch + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + sep = AspirationSeparator( model_id=model, cpu_separation=CPU_SEPARATION, @@ -1757,7 +1952,15 @@ def _run_bandit( if jm.should_cancel(job_id): raise RuntimeError("Cancelled by user") - + + try: + import gc, torch + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + sep = BanditSeparator( model_id=model, cpu_separation=CPU_SEPARATION, @@ -1833,7 +2036,15 @@ def _run_ensemble( if jm.should_cancel(job_id): raise RuntimeError("Cancelled by user") - + + try: + import gc, torch + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + sep = EnsembleSeparator( ensemble_id=model, cpu_separation=CPU_SEPARATION, @@ -1903,6 +2114,14 @@ def _run_sepacap( if jm.should_cancel(job_id): raise RuntimeError("Cancelled by user") + try: + import gc, torch + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + sep = SepACapSeparator( cpu_separation=CPU_SEPARATION, output_format=output_format, @@ -1973,6 +2192,14 @@ def _run_sepreformer( if jm.should_cancel(job_id): raise RuntimeError("Cancelled by user") + try: + import gc, torch + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + sep = SepReformerSeparator( model_id=model, cpu_separation=CPU_SEPARATION, diff --git a/backend/static/index.html b/backend/static/index.html index c0e6ae0..3cf49ea 100644 --- a/backend/static/index.html +++ b/backend/static/index.html @@ -19,7 +19,7 @@ - + diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index b52da91..b7ffe9f 100755 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -397,7 +397,10 @@ export const lyricTranscribeProject = ( ) => { const wordTs = opts?.wordTimestamps ?? true const phonemeAlign = opts?.phonemeAlign ?? false - const defaultFormats = wordTs + const lsyncAlign = opts?.lsyncAlign ?? false + // lsyncAlign requires word-level timestamps to write syllable data into ELRC + const effectiveWordTs = wordTs || lsyncAlign + const defaultFormats = effectiveWordTs ? (phonemeAlign ? ['sylrc', 'elrc', 'lrc', 'txt'] : ['elrc', 'lrc', 'txt']) : ['lrc', 'txt'] return request<{ job_id: string }>(`${BASE}/projects/${projectId}/lyric-transcribe`, { @@ -411,7 +414,7 @@ export const lyricTranscribeProject = ( output_formats: opts?.outputFormats ?? defaultFormats, initial_prompt: opts?.initialPrompt, temperature: opts?.temperature ?? 0.0, - word_timestamps: wordTs, + word_timestamps: effectiveWordTs, condition_on_previous_text: opts?.conditionOnPreviousText ?? true, beam_size: opts?.beamSize ?? undefined, engine: 'whisperx', diff --git a/lalalai b/lalalai new file mode 160000 index 0000000..996f3ea --- /dev/null +++ b/lalalai @@ -0,0 +1 @@ +Subproject commit 996f3eae5707954e2f977161c1f8642e96d8a9d9 diff --git a/stemengine.yml b/stemengine.yml index 67ff1a1..3f958e1 100644 --- a/stemengine.yml +++ b/stemengine.yml @@ -1,10 +1,41 @@ services: + # Minimal privileged init container: sets the GPU power limit then exits. + # Runs before stemengine so the main container never needs SYS_ADMIN. + stemengine-gpu-init: + container_name: stemengine-gpu-init + image: nvidia/cuda:12.1.0-base-ubuntu22.04 + runtime: nvidia + restart: "no" + cap_add: + - SYS_ADMIN + entrypoint: + - sh + - -c + - | + PL="${STEMENGINE_GPU_POWER_LIMIT:-100}" + if [ "$$PL" != "0" ]; then + echo "[gpu-init] Setting power limit to $${PL} W..." + nvidia-smi --power-limit="$$PL" && echo "[gpu-init] Done." || echo "[gpu-init] Failed." + fi + environment: + - STEMENGINE_GPU_POWER_LIMIT=${STEMENGINE_GPU_POWER_LIMIT:-100} + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + stemengine: container_name: stemengine image: ghcr.io/seclusionenthusiast/stemengine:latest runtime: nvidia entrypoint: [] restart: unless-stopped + depends_on: + stemengine-gpu-init: + condition: service_completed_successfully networks: - t2_proxy security_opt: @@ -31,7 +62,7 @@ services: - stemengine-node-modules:/app/frontend/node_modules # Projects folder (where separated stems are stored) - ${PROJECTS}:/app/media/projects - - ${SEFOLDER}/start.sh:/app/start.sh + - ${SEFOLDER}/start.sh:/app/start.sh:ro command: /app/start.sh environment: - PUID=${PUID:-1000} @@ -54,6 +85,7 @@ services: - ASPIRATION_MODEL_DIR=/opt/aspiration_models - BANDIT_MODEL_DIR=/opt/bandit_models - CPU_SEPARATION=${CPU_SEPARATION:-false} + - STEMENGINE_GPU_POWER_LIMIT=${STEMENGINE_GPU_POWER_LIMIT:-100} - MAX_UPLOAD_MB=${MAX_UPLOAD_MB:-500} - PYTHONUNBUFFERED=1 # Prevent CUDA OOM from allocator fragmentation when multiple large models