From 058069d6dfa4ac3b0cf943ce4596155125adacbb Mon Sep 17 00:00:00 2001 From: translator Date: Tue, 31 Mar 2026 17:59:33 +0900 Subject: [PATCH] Myriad changes --- SUMMARIES (Claude).md | 117 ++++++ backend/config.py | 44 ++ backend/routers/models.py | 9 +- backend/services/mdx23c_arch.py | 317 +++++++++++++++ backend/services/mdx23c_separator.py | 379 ++++++++++++++++++ backend/services/mdxnet_separator.py | 12 +- .../mel_band_specialized_separator.py | 130 +++++- backend/services/separator_service.py | 149 ++++++- backend/services/transcriber_service.py | 6 +- backend/static/index.html | 4 +- .../src/components/AutoPipelineDialog.tsx | 2 +- frontend/src/pages/ProjectDetail.tsx | 32 +- frontend/src/pages/Separator.tsx | 2 +- frontend/src/pages/Settings.tsx | 15 +- frontend/src/utils/autoPipeline.ts | 1 + 15 files changed, 1179 insertions(+), 40 deletions(-) create mode 100644 backend/services/mdx23c_arch.py create mode 100644 backend/services/mdx23c_separator.py diff --git a/SUMMARIES (Claude).md b/SUMMARIES (Claude).md index 6aa8106..9668103 100644 --- a/SUMMARIES (Claude).md +++ b/SUMMARIES (Claude).md @@ -1940,3 +1940,120 @@ Added end-to-end stem deletion: - **`frontend/src/pages/ProjectDetail.tsx`**: new `StemDeleteButton` component — shows trashcan icon, prompts inline confirmation (checkmark / X) before calling the API. Added to each stem row in `StemsSection` alongside the existing rename and reprocess buttons. Also updated SUMMARIES path in memory from `SUMMARIES.md` → `SUMMARIES (Claude).md`. + +--- + +### 2026-03-31 — Implement MDX23C TFC-TDF runner for mdxnet_drumsep + +Fixed ONNX `INVALID_PROTOBUF` crash when running drum separation. Root cause: `MDX23C-DrumSep-aufr33-jarredou.ckpt` is a PyTorch ZIP checkpoint but was being loaded via `ort.InferenceSession()` (the ONNX path). + +**Architecture investigation:** +- Fully decoded checkpoint structure by parsing the pickle byte stream directly on the host +- Confirmed TFC-TDF U-Net (ZFTurbo MSST `TFC_TDF_net`) — NOT an ONNX model +- Fetched the companion config YAML from HuggingFace to confirm all hyperparameters: + `n_fft=2048`, `dim_f=1024`, `num_subbands=4`, `num_channels=128`, `growth=128`, `bottleneck_factor=4`, `num_scales=5`, `norm=InstanceNorm`, `act=gelu` +- 6 output stems: kick, snare, toms, hh, ride, crash + +**New files created:** +- `backend/services/mdx23c_arch.py` — vendored `TFC_TDF_net` architecture (STFT, TFC_TDF blocks, Upscale/Downscale, full U-Net); `prefer_target_instrument` replaced by direct config attribute lookup +- `backend/services/mdx23c_separator.py` — `MDX23CSeparator` class: HF download of checkpoint + config YAML, overlapping-window inference (`_demix`), `separate_into_parts` / `separate_into_parts_chunked`, OOM retry logic + +**Files modified:** +- `backend/services/mdxnet_separator.py` — removed `mdxnet_drumsep` entry from the ONNX model dict; replaced with explanatory comment +- `backend/services/separator_service.py` — added `if model == 'mdxnet_drumsep'` routing (before general `_run_mdxnet`) to dispatch to new `_run_mdx23c()`; added `_run_mdx23c()` function (mirrors `_run_scnet` pattern with OOM retry and chunking support) +- `backend/routers/models.py` — fixed `mdxnet_drumsep` entry: corrected `hf_repo` to `miercolesv/MVSEP-Central-Backup`, added config YAML to download files list, updated description to reflect 6-stem output + +--- + +## 2026-03-31 + +### Fix ensemble vocal separation: `config.yaml` not found after `snapshot_download` + +`MelBandVocalsSeparator`, `MelBandDereverbSeparator`, and `MelBandKaraokeSeparator` all used `snapshot_download` without `local_dir_use_symlinks=False`. By default, `huggingface_hub` creates symlinks from the `local_dir` pointing into `~/.cache/huggingface/hub/`. In Docker, the cache is ephemeral, so after a container restart or cache clear the symlinks broke — causing `FileNotFoundError: /opt/mel_band_models/vocals/config.yaml` even though the directory existed. + +Root cause was wrong hardcoded filenames — the HF repo filenames didn't match what the code expected, so `_ensure_model` always failed its existence check (silently), then `super().__init__` tried to open nonexistent paths. `local_dir_use_symlinks` is deprecated in current huggingface_hub (files are always copied directly now). + +- `MelBandVocalsSeparator`: expected `model.ckpt` / `config.yaml` → actual: `mel_band_roformer_vocals_becruily.ckpt` / `config_vocals_becruily.yaml` +- `MelBandDereverbSeparator`: expected `config.yaml` → actual: `dereverb_mel_band_roformer_anvuew.yaml` (model filenames were already correct) +- `MelBandKaraokeSeparator`: filenames were already correct, no change needed + +Fix applied live (no rebuild): `docker cp` updated file + manually triggered `snapshot_download` with correct filenames + `docker restart stemengine-stemengine-1`. + +**Files modified:** +- `backend/services/mel_band_specialized_separator.py` — corrected `model_path` / `config_path` filenames in `MelBandVocalsSeparator.__init__` and `MelBandDereverbSeparator.__init__`; added `local_dir_use_symlinks=False` to all three `snapshot_download` calls (harmless, param is now deprecated/ignored) + +### 2026-03-31 — Fixed SCNet size-mismatch crash in Ensemble vocal separation + +**Problem:** `ensemble_vocal_best` (and standalone `scnet_xl`/`scnet_xl_ihf`) crashed with `size mismatch for encoder.0.SDlayer.convs.0.weight` when loading the checkpoint. Root cause: the cached config YAMLs (`config_musdb18_scnet_xl.yaml`) specified `dims: [4, 64, 128, 256]` and `num_dplayer: 8`, but the actual checkpoint (`scnet_checkpoint_musdb18.ckpt`, md5 `6038de19`) was trained with `dims: [4, 32, 64, 128]` and only 6 DP layers. The configs had the wrong architecture dimensions for this checkpoint. + +**Fix:** Updated the two on-disk config YAMLs inside the running container (volume-mounted, no rebuild needed): +- `/opt/scnet_models/scnet_xl/config_musdb18_scnet_xl.yaml` +- `/opt/scnet_models/scnet_xl_ihf/config_musdb18_scnet_xl.yaml` + +Changes in both files: +- `dims: [4, 64, 128, 256]` → `dims: [4, 32, 64, 128]` +- `num_dplayer: 8` → `num_dplayer: 6` + +After fix: both models load with 0 missing / 0 unexpected keys. No code changes required. + +### 2026-03-31 — Added Monophonic mode for Basic Pitch MIDI transcription + +**Files modified:** +- `backend/config.py` — added `"monophonic"` entry to `TRANSCRIBE_MODES` (label "Monophonic", engine "basic-pitch") and corresponding preset to `TRANSCRIBE_PRESETS` (`onset_threshold=0.6`, `frame_threshold=0.5`, `minimum_note_length=100.0`, `melodia_trick=True`, `multiple_pitch_bends=False`) +- `backend/services/transcriber_service.py` — updated docstring to list `monophonic` as a valid mode +- `frontend/src/pages/ProjectDetail.tsx` — added `{ id: 'monophonic', label: 'Monophonic', engine: 'basic-pitch', available: true }` to `FALLBACK_MODES` + +Frontend rebuilt and backend restarted. + +### 2026-03-31 — Fixed karaoke BS-RoFormer models producing two vocal tracks instead of vocals + instrumental + +**Root cause (two bugs):** + +1. **Broadcasting bug in `_demix`**: All karaoke configs set `num_stems: 1`. When `num_stems=1`, BS-RoFormer outputs `(batch, channels=2, time)` — no stems dimension. The base class allocated `result` with shape `(len(self.stems)=2, channels, time)`. Adding `x[j]` of shape `(2, time)` into `(2, 2, time)` triggered PyTorch broadcasting (prepends 1 → `(1, 2, time)` → fills both rows), so both stem slots received identical audio. Fix: read `model_out_stems` from `config.model.num_stems` and allocate `result` with that count; compute remaining stems as `mix − sum(direct outputs)`. + +2. **Wrong class for Anvuew**: `bs_roformer_karaoke_anvuew` was dispatched to `MelBandKaraokeGaboxSeparator`, loading `bs_karaoke_gabox_IS.ckpt` instead of the correct `karaoke_bs_roformer_anvuew.ckpt` (which was already downloaded alongside its config). + +**Files modified:** +- `backend/services/mel_band_specialized_separator.py`: + - `_demix`: allocate `result` lazily from first batch output shape (`x.shape[1]`), then compute residual stems as `mix − sum(direct_stems)` for any additional requested stems + - Changed all karaoke separator `stems=['vocals', 'karaoke']` → `stems=['vocals', 'instrumental']` (naming fix) + - Added `BSRoformerKaraokeAnvuewSeparator` class loading `karaoke_bs_roformer_anvuew.ckpt` / `karaoke_bs_roformer_anvuew.yaml` +- `backend/services/separator_service.py` — fixed `bs_roformer_karaoke_anvuew` dispatch to use `BSRoformerKaraokeAnvuewSeparator` +- `backend/services/separation/output_layout.py` — reverted the earlier `"karaoke"` alias (no longer needed) + +### 2026-03-31 — Added "instrumental" to per-stem refinement options + +Added `"instrumental"` to `INSTRUMENT_MODEL_KEYS` in `backend/config.py`, `INSTRUMENT_KEYS` in `frontend/src/pages/Separator.tsx`, and `COMMON_STEMS` in `frontend/src/pages/Settings.tsx` (Auto-Pipeline). Positioned between `vocals` and `backing` in all three. Frontend rebuilt, backend restarted. + +### 2026-03-31 — Karaoke separators: renamed "vocals" stem to "leading" + +Changed `stems=['vocals', 'instrumental']` → `stems=['leading', 'instrumental']` on the six karaoke separator classes only (`MelBandKaraokeGaboxSeparator`, `BSRoformerKaraokeAnvuewSeparator`, `MelBandKaraokeFrazerBecruilySeparator`, `MelBandKaraokeGaboxSmallSeparator`, `MelBandKaraokeGaboxV2ExperimentalSeparator`, `MelBandKaraokeGaboxV1Separator`). Standard vocals separators and `MelBandDeuxBecruilySeparator` unchanged. + +### 2026-03-31 — Added Guitar and Bass MIDI transcription modes + +**Files modified:** +- `backend/config.py` — added `"guitar"` and `"bass"` entries to `TRANSCRIBE_MODES` and `TRANSCRIBE_PRESETS`. Guitar: min_freq=82 Hz, max_freq=1300 Hz, pitch bends on, polyphonic. Bass: min_freq=41 Hz, max_freq=300 Hz, pitch bends on, polyphonic. Both use standard onset/frame thresholds and melodia_trick=False. +- `backend/services/transcriber_service.py` — updated docstrings to list both new modes +- `frontend/src/pages/ProjectDetail.tsx` — added Guitar and Bass to FALLBACK_MODES + +Frontend rebuilt and backend restarted. + +--- + +## 2026-03-31 + +### Lyric transcription stem selector (vocals / leading / backing) + +Added a stem picker so users can choose which audio stem to transcribe, rather than always using 'vocals'. + +**Auto-pipeline changes:** +- `frontend/src/utils/autoPipeline.ts` — added `stem?: string` field to `AutoStepLyrics` interface +- `frontend/src/pages/Settings.tsx` — added `stem: 'vocals'` to default lyrics step; added "Stem" selector dropdown (Vocals / Leading / Backing) as first control in the lyrics step UI panel +- `frontend/src/components/AutoPipelineDialog.tsx` — changed hardcoded `'vocals'` to `step.stem ?? 'vocals'` when calling `lyricTranscribeProject` + +**Project detail changes:** +- `frontend/src/pages/ProjectDetail.tsx` + - Expanded `vocalStems` filter from `.includes('vocal')` to `/^(vocal|leading|backing)/i` so karaoke 'leading' and 'backing' stems now show a Lyrics Sync row + - Added `txStemName` state inside `StemLyricsRow` (defaults to the row's `stemName` prop) + - Added dynamic stem selector in Transcribe mode: computed from `stems` prop filtered to vocals/leading/backing; only rendered when >1 option exists + - `handleTranscribe` and `handleAlign` now use `txStemName` instead of the fixed `stemName` prop diff --git a/backend/config.py b/backend/config.py index 001cf50..8050c75 100755 --- a/backend/config.py +++ b/backend/config.py @@ -221,6 +221,7 @@ ALL_MODEL_IDS = {m["id"] for m in ALL_MODELS} # Canonical stem keys for per-instrument model refinement. INSTRUMENT_MODEL_KEYS = ( "vocals", + "instrumental", "backing", "drums", "bass", @@ -254,6 +255,24 @@ TRANSCRIBE_MODES = [ "engine": "basic-pitch", "description": "Single-voice / vocal melody extraction — basic-pitch with melodia trick", }, + { + "id": "monophonic", + "label": "Monophonic", + "engine": "basic-pitch", + "description": "Strict single-voice output — melodia trick + higher thresholds + no pitch bends", + }, + { + "id": "guitar", + "label": "Guitar", + "engine": "basic-pitch", + "description": "Guitar-optimised — pitch bends enabled, frequency range E2–D6 (82–1300 Hz)", + }, + { + "id": "bass", + "label": "Bass", + "engine": "basic-pitch", + "description": "Bass-optimised — pitch bends enabled, frequency range E1–G4 (41–300 Hz)", + }, # ── Omnizart modes ────────────────────────────────────────────────────── # Require OMNIZART_ENABLED=true; served by the omnizart-sidecar container # (or a local Omnizart installation when OMNIZART_SIDECAR_URL is unset). @@ -301,6 +320,31 @@ TRANSCRIBE_PRESETS = { "minimum_frequency": 65.0, # Hz — below typical singing range "melodia_trick": True, }, + "monophonic": { + "onset_threshold": 0.6, + "frame_threshold": 0.5, + "minimum_note_length": 100.0, + "melodia_trick": True, + "multiple_pitch_bends": False, + }, + "guitar": { + "onset_threshold": 0.5, + "frame_threshold": 0.3, + "minimum_note_length": 58.0, + "minimum_frequency": 82.0, + "maximum_frequency": 1300.0, + "melodia_trick": False, + "multiple_pitch_bends": True, + }, + "bass": { + "onset_threshold": 0.5, + "frame_threshold": 0.3, + "minimum_note_length": 58.0, + "minimum_frequency": 41.0, + "maximum_frequency": 300.0, + "melodia_trick": False, + "multiple_pitch_bends": True, + }, } # Hardware diff --git a/backend/routers/models.py b/backend/routers/models.py index 99591b7..8fe812a 100755 --- a/backend/routers/models.py +++ b/backend/routers/models.py @@ -298,9 +298,12 @@ MODEL_DOWNLOAD_INFO = { }, "mdxnet_drumsep": { "size_mb": 448, - "description": "MDX23C DrumSep - specialized drum separation model.", - "hf_repo": "Politrees/UVR_resources", - "files": ["models/MDX23C/MDX23C-DrumSep-aufr33-jarredou.ckpt"], + "description": "MDX23C DrumSep (6-stem: kick/snare/toms/hh/ride/crash) by aufr33-jarredou.", + "hf_repo": "miercolesv/MVSEP-Central-Backup", + "files": [ + "Politrees/UVR_resources/models/MDX23C/MDX23C-DrumSep-aufr33-jarredou.ckpt", + "Politrees/UVR_resources/models/MDX23C/config_drumsep_mdx23c.yaml", + ], }, # SCNet XL models (2024-2025 state-of-the-art) diff --git a/backend/services/mdx23c_arch.py b/backend/services/mdx23c_arch.py new file mode 100644 index 0000000..1950784 --- /dev/null +++ b/backend/services/mdx23c_arch.py @@ -0,0 +1,317 @@ +""" +MDX23C TFC-TDF U-Net architecture — vendored from ZFTurbo MSST (MIT licence). +Source: https://github.com/ZFTurbo/Music-Source-Separation-Training + +Only the inference-relevant classes are kept; training scaffolding is removed. +`prefer_target_instrument` is replaced with a direct attribute lookup on the +config namespace produced by `mdx23c_separator._load_config()`. +""" +from functools import partial + +import torch +import torch.nn as nn + + +# --------------------------------------------------------------------------- +# STFT helpers +# --------------------------------------------------------------------------- + +class STFT: + def __init__(self, config): + self.n_fft = config.n_fft + self.hop_length = config.hop_length + self.window = torch.hann_window(window_length=self.n_fft, periodic=True) + self.dim_f = config.dim_f + + def __call__(self, x): + window = self.window.to(x.device) + batch_dims = x.shape[:-2] + c, t = x.shape[-2:] + x = x.reshape([-1, t]) + x = torch.stft( + x, + n_fft=self.n_fft, + hop_length=self.hop_length, + window=window, + center=True, + return_complex=True, + ) + x = torch.view_as_real(x) + x = x.permute([0, 3, 1, 2]) + x = x.reshape([*batch_dims, c, 2, -1, x.shape[-1]]).reshape( + [*batch_dims, c * 2, -1, x.shape[-1]] + ) + return x[..., : self.dim_f, :] + + def inverse(self, x): + window = self.window.to(x.device) + batch_dims = x.shape[:-3] + c, f, t = x.shape[-3:] + n = self.n_fft // 2 + 1 + f_pad = torch.zeros([*batch_dims, c, n - f, t]).to(x.device) + x = torch.cat([x, f_pad], -2) + x = x.reshape([*batch_dims, c // 2, 2, n, t]).reshape([-1, 2, n, t]) + x = x.permute([0, 2, 3, 1]) + x = x[..., 0] + x[..., 1] * 1.0j + x = torch.istft(x, n_fft=self.n_fft, hop_length=self.hop_length, window=window, center=True) + x = x.reshape([*batch_dims, 2, -1]) + return x + + +# --------------------------------------------------------------------------- +# Norm / activation factories +# --------------------------------------------------------------------------- + +def get_norm(norm_type): + def norm(c, norm_type): + if norm_type == "BatchNorm": + return nn.BatchNorm2d(c) + elif norm_type == "InstanceNorm": + return nn.InstanceNorm2d(c, affine=True) + elif "GroupNorm" in norm_type: + g = int(norm_type.replace("GroupNorm", "")) + return nn.GroupNorm(num_groups=g, num_channels=c) + else: + return nn.Identity() + + return partial(norm, norm_type=norm_type) + + +def get_act(act_type): + if act_type == "gelu": + return nn.GELU() + elif act_type == "relu": + return nn.ReLU() + elif act_type[:3] == "elu": + alpha = float(act_type.replace("elu", "")) + return nn.ELU(alpha) + else: + raise ValueError(f"Unknown activation type: {act_type!r}") + + +# --------------------------------------------------------------------------- +# Building blocks +# --------------------------------------------------------------------------- + +class Upscale(nn.Module): + def __init__(self, in_c, out_c, scale, norm, act): + super().__init__() + self.conv = nn.Sequential( + norm(in_c), + act, + nn.ConvTranspose2d( + in_channels=in_c, out_channels=out_c, + kernel_size=scale, stride=scale, bias=False, + ), + ) + + def forward(self, x): + return self.conv(x) + + +class Downscale(nn.Module): + def __init__(self, in_c, out_c, scale, norm, act): + super().__init__() + self.conv = nn.Sequential( + norm(in_c), + act, + nn.Conv2d( + in_channels=in_c, out_channels=out_c, + kernel_size=scale, stride=scale, bias=False, + ), + ) + + def forward(self, x): + return self.conv(x) + + +class TFC_TDF(nn.Module): + """Time-Frequency Convolution + Time-Domain Filtering block stack.""" + + def __init__(self, in_c, c, l, f, bn, norm, act): + """ + Args: + in_c: input channel count for the first block. + c: output channel count (all blocks). + l: number of TFC-TDF sub-blocks. + f: frequency dimension (per subband) at this stage. + bn: bottleneck factor for the TDF linear layers (f // bn). + norm: normalisation factory from get_norm(). + act: activation module instance. + """ + super().__init__() + + self.blocks = nn.ModuleList() + for _ in range(l): + block = nn.Module() + block.tfc1 = nn.Sequential( + norm(in_c), + act, + nn.Conv2d(in_c, c, 3, 1, 1, bias=False), + ) + block.tdf = nn.Sequential( + norm(c), + act, + nn.Linear(f, f // bn, bias=False), + norm(c), + act, + nn.Linear(f // bn, f, bias=False), + ) + block.tfc2 = nn.Sequential( + norm(c), + act, + nn.Conv2d(c, c, 3, 1, 1, bias=False), + ) + block.shortcut = nn.Conv2d(in_c, c, 1, 1, 0, bias=False) + self.blocks.append(block) + in_c = c + + def forward(self, x): + for block in self.blocks: + s = block.shortcut(x) + x = block.tfc1(x) + x = x + block.tdf(x) + x = block.tfc2(x) + x = x + s + return x + + +# --------------------------------------------------------------------------- +# Full TFC-TDF U-Net (MDX23C architecture) +# --------------------------------------------------------------------------- + +class TFC_TDF_net(nn.Module): + """ + TFC-TDF U-Net as used by ZFTurbo MSST for MDX23C checkpoints. + + Config must be a namespace (use `mdx23c_separator._load_config` which + calls `_to_ns`) with the following attributes: + + config.audio.num_channels int — stereo=2 + config.audio.dim_f int — STFT frequency bins used (1024) + config.audio.n_fft int — STFT window size (2048) + config.audio.hop_length int — STFT hop (512) + config.model.norm str — "InstanceNorm" / "BatchNorm" / … + config.model.act str — "gelu" / "relu" / … + config.model.num_subbands int — 4 + config.model.num_scales int — encoder/decoder depth (5) + config.model.scale list — [2, 2] + config.model.num_channels int — base channel count (128) + config.model.growth int — channel growth per scale (128) + config.model.bottleneck_factor int — TDF reduction factor (4) + config.model.num_blocks_per_scale int — TFC-TDF blocks per stage (2) + config.training.instruments list — output stem names + """ + + def __init__(self, config): + super().__init__() + self.config = config + + norm = get_norm(norm_type=config.model.norm) + act = get_act(act_type=config.model.act) + + instruments = config.training.instruments + self.num_target_instruments = len(instruments) + self.num_subbands = config.model.num_subbands + + dim_c = self.num_subbands * config.audio.num_channels * 2 + n = config.model.num_scales + scale = config.model.scale + l = config.model.num_blocks_per_scale + c = config.model.num_channels + g = config.model.growth + bn = config.model.bottleneck_factor + f = config.audio.dim_f // self.num_subbands + + self.first_conv = nn.Conv2d(dim_c, c, 1, 1, 0, bias=False) + + self.encoder_blocks = nn.ModuleList() + for _ in range(n): + block = nn.Module() + block.tfc_tdf = TFC_TDF(c, c, l, f, bn, norm, act) + block.downscale = Downscale(c, c + g, scale, norm, act) + f = f // scale[1] + c += g + self.encoder_blocks.append(block) + + self.bottleneck_block = TFC_TDF(c, c, l, f, bn, norm, act) + + self.decoder_blocks = nn.ModuleList() + for _ in range(n): + block = nn.Module() + block.upscale = Upscale(c, c - g, scale, norm, act) + f = f * scale[1] + c -= g + block.tfc_tdf = TFC_TDF(2 * c, c, l, f, bn, norm, act) + self.decoder_blocks.append(block) + + self.final_conv = nn.Sequential( + nn.Conv2d(c + dim_c, c, 1, 1, 0, bias=False), + act, + nn.Conv2d(c, self.num_target_instruments * dim_c, 1, 1, 0, bias=False), + ) + + self.stft = STFT(config.audio) + + # ------------------------------------------------------------------ + # Subband reshape helpers + # ------------------------------------------------------------------ + + def cac2cws(self, x): + """Channel-as-channels → channel-with-subbands.""" + k = self.num_subbands + b, c, f, t = x.shape + x = x.reshape(b, c, k, f // k, t) + x = x.reshape(b, c * k, f // k, t) + return x + + def cws2cac(self, x): + """Channel-with-subbands → channel-as-channels.""" + k = self.num_subbands + b, c, f, t = x.shape + x = x.reshape(b, c // k, k, f, t) + x = x.reshape(b, c // k, f * k, t) + return x + + # ------------------------------------------------------------------ + + def forward(self, x): + """ + Args: + x: (B, 2, T_audio) stereo waveform tensor + Returns: + (B, num_targets, 2, T_audio) + """ + x = self.stft(x) # (B, C*2, dim_f, T_frames) + mix = x = self.cac2cws(x) # (B, C*2*num_subbands, dim_f/ns, T_frames) + + first_conv_out = x = self.first_conv(x) + + x = x.transpose(-1, -2) # swap freq and time axes + + encoder_outputs = [] + for block in self.encoder_blocks: + x = block.tfc_tdf(x) + encoder_outputs.append(x) + x = block.downscale(x) + + x = self.bottleneck_block(x) + + for block in self.decoder_blocks: + x = block.upscale(x) + x = torch.cat([x, encoder_outputs.pop()], 1) + x = block.tfc_tdf(x) + + x = x.transpose(-1, -2) # restore freq/time order + + x = x * first_conv_out # artifact suppression gate + + x = self.final_conv(torch.cat([mix, x], 1)) + + x = self.cws2cac(x) + + b, c, f, t = x.shape + x = x.reshape(b, self.num_target_instruments, -1, f, t) + + x = self.stft.inverse(x) # (B, num_targets, 2, T_audio) + return x diff --git a/backend/services/mdx23c_separator.py b/backend/services/mdx23c_separator.py new file mode 100644 index 0000000..00dcf33 --- /dev/null +++ b/backend/services/mdx23c_separator.py @@ -0,0 +1,379 @@ +""" +MDX23C (TFC-TDF U-Net) separator — native PyTorch inference. +Architecture vendored from ZFTurbo MSST (MIT licence): + https://github.com/ZFTurbo/Music-Source-Separation-Training + +Handles the `mdxnet_drumsep` model (MDX23C-DrumSep-aufr33-jarredou) which +is a 6-stem drum separator: kick, snare, toms, hh, ride, crash. +""" +import shutil +import types +from pathlib import Path +from typing import Dict, Optional + +import numpy as np +import torch +import torchaudio +import yaml + +from .mdx23c_arch import TFC_TDF_net +from ..config import MDXNET_MODEL_DIR, CPU_SEPARATION + + +# --------------------------------------------------------------------------- +# Model registry +# --------------------------------------------------------------------------- + +MDX23C_MODELS: Dict[str, dict] = { + "mdxnet_drumsep": { + "model_file": "MDX23C-DrumSep-aufr33-jarredou.ckpt", + "config_file": "config_drumsep_mdx23c.yaml", + "hf_repo": "miercolesv/MVSEP-Central-Backup", + "hf_model_path": "Politrees/UVR_resources/models/MDX23C/MDX23C-DrumSep-aufr33-jarredou.ckpt", + "hf_config_path": "Politrees/UVR_resources/models/MDX23C/config_drumsep_mdx23c.yaml", + }, +} + + +# --------------------------------------------------------------------------- +# Config helpers +# --------------------------------------------------------------------------- + +def _to_ns(d): + """Recursively convert dicts to SimpleNamespace for attribute access.""" + if isinstance(d, dict): + return types.SimpleNamespace(**{k: _to_ns(v) for k, v in d.items()}) + if isinstance(d, list): + return [_to_ns(i) for i in d] + return d + + +def _load_config(config_path: Path) -> types.SimpleNamespace: + """Load YAML config, tolerate !!python/tuple tags.""" + loader = yaml.SafeLoader + loader.add_constructor( + "tag:yaml.org,2002:python/tuple", + lambda ldr, node: tuple(ldr.construct_sequence(node)), + ) + with open(config_path) as f: + raw = yaml.load(f, Loader=loader) + return _to_ns(raw) + + +# --------------------------------------------------------------------------- +# Audio I/O +# --------------------------------------------------------------------------- + +def _load_audio(path: str, sample_rate: int) -> np.ndarray: + """Load audio and return (2, samples) float32 array at target sample rate.""" + waveform, sr = torchaudio.load(path) + if sr != sample_rate: + waveform = torchaudio.functional.resample(waveform, sr, sample_rate) + if waveform.shape[0] == 1: + waveform = waveform.repeat(2, 1) + elif waveform.shape[0] > 2: + waveform = waveform[:2] + return waveform.numpy().astype(np.float32) + + +def _save_audio(path: str, audio: np.ndarray, sample_rate: int, output_format: str): + """Save (channels, samples) numpy array via torchaudio.""" + tensor = torch.tensor(audio, dtype=torch.float32) + fmt = output_format.lower() + if fmt == "mp3": + torchaudio.save(path, tensor, sample_rate, format="mp3") + elif fmt == "flac": + torchaudio.save(path, tensor, sample_rate, format="flac") + else: + torchaudio.save(path, tensor, sample_rate) + + +# --------------------------------------------------------------------------- +# Overlapping-window inference (MSST style) +# --------------------------------------------------------------------------- + +def _build_window(chunk_size: int, step: int) -> np.ndarray: + """Linear fade-in/flat/fade-out window; consecutive windows sum to 1.""" + fade = step + fade_in = np.linspace(0.0, 1.0, fade, endpoint=False, dtype=np.float32) + fade_out = np.linspace(1.0, 0.0, fade, endpoint=False, dtype=np.float32) + flat_len = chunk_size - 2 * fade + if flat_len > 0: + return np.concatenate([fade_in, np.ones(flat_len, dtype=np.float32), fade_out]) + return np.concatenate([fade_in, fade_out]) + + +def _demix( + model: TFC_TDF_net, + mix: np.ndarray, + chunk_size: int, + num_overlap: int, + device: torch.device, +) -> np.ndarray: + """ + MSST-style overlapping-window inference. + + Args: + mix: (2, samples) float32 stereo audio + Returns: + (num_sources, 2, samples) float32 + """ + C, N = mix.shape + step = chunk_size // num_overlap + border = chunk_size - step + + mix_padded = np.pad(mix, ((0, 0), (border, border)), mode="reflect") + + window = _build_window(chunk_size, step) + sources_sum: Optional[np.ndarray] = None + weight_sum = np.zeros(mix_padded.shape[-1], dtype=np.float32) + + with torch.no_grad(), \ + torch.amp.autocast("cuda", dtype=torch.float16, enabled=(device.type == "cuda")): + for start in range(0, N, step): + p_end = start + chunk_size + if p_end > mix_padded.shape[-1]: + break + chunk = mix_padded[:, start:p_end] + + mean = float(chunk.mean()) + std = max(float(chunk.std()), 1e-8) + chunk_norm = (chunk - mean) / std + + inp = torch.tensor(chunk_norm[np.newaxis], dtype=torch.float32, device=device) + out = model(inp) # (1, num_sources, 2, chunk_size) + out_np = out.squeeze(0).cpu().numpy() # (num_sources, 2, chunk_size) + out_np = out_np * std + mean + + if sources_sum is None: + num_sources = out_np.shape[0] + sources_sum = np.zeros( + (num_sources, C, mix_padded.shape[-1]), dtype=np.float32 + ) + + sources_sum[:, :, start:p_end] += out_np * window[np.newaxis, np.newaxis, :] + weight_sum[start:p_end] += window + + if sources_sum is None: + raise RuntimeError("Audio is too short for MDX23C chunked inference.") + + weight_sum = np.maximum(weight_sum, 1e-8) + sources = sources_sum / weight_sum[np.newaxis, np.newaxis, :] + return sources[:, :, border : border + N] + + +# --------------------------------------------------------------------------- +# Separator class +# --------------------------------------------------------------------------- + +class MDX23CSeparator: + """Native PyTorch inference wrapper for MDX23C TFC-TDF U-Net models.""" + + def __init__( + self, + model_id: str = "mdxnet_drumsep", + cpu_separation: bool = CPU_SEPARATION, + output_format: str = "mp3", + mp3_bitrate: str = "256k", + ): + if model_id not in MDX23C_MODELS: + raise ValueError( + f"Unknown MDX23C model: {model_id!r}. " + f"Available: {list(MDX23C_MODELS)}" + ) + + self.model_id = model_id + self.model_cfg = MDX23C_MODELS[model_id] + self.cpu_separation = cpu_separation + self.output_format = output_format + self.mp3_bitrate = mp3_bitrate + self.device = torch.device( + "cpu" if cpu_separation else ("cuda" if torch.cuda.is_available() else "cpu") + ) + + self._model: Optional[TFC_TDF_net] = None + self._config = None + self._instruments = None + + # ------------------------------------------------------------------ + # Model management + # ------------------------------------------------------------------ + + def _ensure_files(self) -> tuple[Path, Path]: + """ + Locate or download model checkpoint and config YAML. + + The checkpoint may already exist at MDXNET_MODEL_DIR (downloaded by + the pre-existing ONNX download path). Config YAML is downloaded next + to it if absent. + """ + from urllib.request import urlretrieve + from huggingface_hub import hf_hub_download + + model_path = MDXNET_MODEL_DIR / self.model_cfg["model_file"] + config_path = MDXNET_MODEL_DIR / self.model_cfg["config_file"] + + if not model_path.exists(): + print(f"[MDX23C] Downloading {self.model_cfg['model_file']} …") + MDXNET_MODEL_DIR.mkdir(parents=True, exist_ok=True) + cached = hf_hub_download( + repo_id=self.model_cfg["hf_repo"], + filename=self.model_cfg["hf_model_path"], + ) + shutil.copy2(cached, model_path) + + if not config_path.exists(): + print(f"[MDX23C] Downloading {self.model_cfg['config_file']} …") + MDXNET_MODEL_DIR.mkdir(parents=True, exist_ok=True) + cached = hf_hub_download( + repo_id=self.model_cfg["hf_repo"], + filename=self.model_cfg["hf_config_path"], + ) + shutil.copy2(cached, config_path) + + return model_path, config_path + + def _load_model(self) -> None: + """Load config, instantiate TFC_TDF_net, load checkpoint weights.""" + model_path, config_path = self._ensure_files() + + cfg = _load_config(config_path) + self._config = cfg + self._instruments = cfg.training.instruments + + model = TFC_TDF_net(cfg) + + state = torch.load(str(model_path), map_location="cpu", weights_only=False) + if "state_dict" in state: + state = state["state_dict"] + result = model.load_state_dict(state, strict=True) + model.eval() + self._model = model.to(self.device) + print( + f"[MDX23C] Loaded {self.model_id} ({len(self._instruments)} stems: " + f"{', '.join(self._instruments)}) on {self.device}" + ) + + def _get_model(self) -> TFC_TDF_net: + if self._model is None: + self._load_model() + return self._model + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def separate_into_parts( + self, + input_path: str, + output_dir: str, + chunk_size_override: Optional[int] = None, + num_overlap_override: Optional[int] = None, + ) -> Dict[str, str]: + """ + Separate audio into per-drum-component stems. + + Returns: + {stem_name: absolute_output_path} + """ + model = self._get_model() + cfg = self._config + + sample_rate: int = cfg.audio.sample_rate + inf_cfg = cfg.inference + chunk_size: int = ( + chunk_size_override + if chunk_size_override is not None + else getattr(cfg.audio, "chunk_size", 130560) + ) + num_overlap: int = ( + num_overlap_override + if num_overlap_override is not None + else getattr(inf_cfg, "num_overlap", 4) + ) + + mix = _load_audio(input_path, sample_rate) # (2, N) + + print(f"[MDX23C] Running {self.model_id} on {self.device} …") + sources_np = _demix(model, mix, chunk_size, num_overlap, self.device) + # sources_np: (num_sources, 2, N) + + out_dir = Path(output_dir) + out_dir.mkdir(parents=True, exist_ok=True) + output_paths: Dict[str, str] = {} + + for i, stem in enumerate(self._instruments): + if i >= sources_np.shape[0]: + break + ext = self.output_format.lower() + out_path = str(out_dir / f"{stem}.{ext}") + _save_audio(out_path, sources_np[i], sample_rate, self.output_format) + output_paths[stem] = out_path + print(f"[MDX23C] Saved {stem} → {out_path}") + + return output_paths + + def separate_into_parts_chunked( + self, + input_path: str, + output_dir: str, + chunk_plan, + chunk_size_override: Optional[int] = None, + num_overlap_override: Optional[int] = None, + progress_cb=None, + ) -> Dict[str, str]: + """Chunked variant for long tracks (overlap-add reassembly).""" + from .separation.chunking import extract_chunk, overlap_add_reconstruct + + model = self._get_model() + cfg = self._config + + sample_rate: int = cfg.audio.sample_rate + inf_cfg = cfg.inference + chunk_size: int = ( + chunk_size_override + if chunk_size_override is not None + else getattr(cfg.audio, "chunk_size", 130560) + ) + num_overlap: int = ( + num_overlap_override + if num_overlap_override is not None + else getattr(inf_cfg, "num_overlap", 4) + ) + + audio = _load_audio(input_path, sample_rate) + n_chunks = len(chunk_plan.chunks) + chunk_outputs = [] + + for i, (start, end) in enumerate(chunk_plan.chunks): + ts_start = start / chunk_plan.sample_rate + ts_end = min(end, chunk_plan.total_samples) / chunk_plan.sample_rate + msg = f"Chunk {i + 1}/{n_chunks} ({ts_start:.0f} s – {ts_end:.0f} s)..." + if progress_cb: + progress_cb(msg) + + chunk_audio = extract_chunk(audio, start, end) + sources_chunk = _demix(model, chunk_audio, chunk_size, num_overlap, self.device) + stem_dict = { + self._instruments[j]: sources_chunk[j] + for j in range(min(len(self._instruments), sources_chunk.shape[0])) + } + chunk_outputs.append(stem_dict) + + stem_names = list(chunk_outputs[0].keys()) + full_sources = overlap_add_reconstruct(chunk_outputs, chunk_plan, stem_names) + + out_dir = Path(output_dir) + out_dir.mkdir(parents=True, exist_ok=True) + output_paths: Dict[str, str] = {} + + for stem in stem_names: + if stem not in full_sources: + continue + ext = self.output_format.lower() + out_path = str(out_dir / f"{stem}.{ext}") + _save_audio(out_path, full_sources[stem], sample_rate, self.output_format) + output_paths[stem] = out_path + print(f"[MDX23C chunked] Saved {stem} → {out_path}") + + return output_paths diff --git a/backend/services/mdxnet_separator.py b/backend/services/mdxnet_separator.py index 4e38600..2f30bb7 100755 --- a/backend/services/mdxnet_separator.py +++ b/backend/services/mdxnet_separator.py @@ -64,13 +64,11 @@ MDXNET_MODELS = { 'target_stem': 'dry_vocals', 'use_melband': True, # Uses MelBand architecture, not pure ONNX }, - 'mdxnet_drumsep': { - 'repo_id': 'miercolesv/MVSEP-Central-Backup', - 'path': 'Politrees/UVR_resources/models/MDX23C/MDX23C-DrumSep-aufr33-jarredou.ckpt', - 'stems': ['drums', 'no_drums'], - 'target_stem': 'drums', - 'use_melband': True, - }, + # mdxnet_drumsep is intentionally absent here — it is a PyTorch TFC-TDF + # checkpoint (MDX23C architecture), not an ONNX model. Loading it with + # ort.InferenceSession raises INVALID_PROTOBUF. It is handled instead + # by mdx23c_separator.MDX23CSeparator via _run_mdx23c() in + # separator_service.py. } # MDX-Net specific constants diff --git a/backend/services/mel_band_specialized_separator.py b/backend/services/mel_band_specialized_separator.py index 9c8f8a0..b21f5e7 100755 --- a/backend/services/mel_band_specialized_separator.py +++ b/backend/services/mel_band_specialized_separator.py @@ -283,7 +283,12 @@ class MelBandSeparatorBase: window_middle[-fade_size:] *= fadeout num_stems = len(self.stems) - req_shape = (num_stems,) + tuple(mix_t.shape) + # num_stems: 1 models output (batch, channels, time) — no stems dim. + # num_stems: N>1 models output (batch, N, channels, time). + # Read the true stem count from the model config so we allocate the + # result buffer correctly and avoid the PyTorch broadcasting trap. + model_out_stems = int(getattr(config.model, 'num_stems', num_stems)) + req_shape = (model_out_stems,) + tuple(mix_t.shape) with torch.inference_mode(), \ torch.amp.autocast('cuda', dtype=torch.float16, enabled=(device != 'cpu')): @@ -325,8 +330,17 @@ class MelBandSeparatorBase: estimated = estimated[..., (C - step):-(C - step)] sources = {} - for idx, name in enumerate(self.stems): - sources[name] = estimated[idx].cpu().numpy() + for idx in range(model_out_stems): + sources[self.stems[idx]] = estimated[idx].cpu().numpy() + + # Models with num_stems=1 output only the primary stem (e.g. lead vocals). + # Compute any additional requested stems as mix − sum(direct outputs). + if model_out_stems < num_stems: + mix_np = mix[:, :estimated.shape[-1]] + residual = mix_np - sum(sources.values()) + for idx in range(model_out_stems, num_stems): + sources[self.stems[idx]] = residual + return sources def separate_into_parts(self, input_path: str, output_dir: str, @@ -464,8 +478,8 @@ class MelBandVocalsSeparator(MelBandSeparatorBase): segment_secs_override: Optional[float] = None, ): model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/vocals') - model_path = model_dir / 'model.ckpt' - config_path = model_dir / 'config.yaml' + model_path = model_dir / 'mel_band_roformer_vocals_becruily.ckpt' + config_path = model_dir / 'config_vocals_becruily.yaml' # Ensure model is downloaded self._ensure_model(model_dir, model_path, config_path) @@ -492,6 +506,7 @@ class MelBandVocalsSeparator(MelBandSeparatorBase): repo_id=self.HF_REPO_ID, revision=self.HF_REVISION, local_dir=str(model_dir), + local_dir_use_symlinks=False, ) @@ -520,10 +535,10 @@ class MelBandDereverbSeparator(MelBandSeparatorBase): model_path = model_dir / 'dereverb_mel_band_roformer_less_aggressive_anvuew_sdr_18.8050.ckpt' else: model_path = model_dir / 'dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt' - config_path = model_dir / 'config.yaml' - + config_path = model_dir / 'dereverb_mel_band_roformer_anvuew.yaml' + self._ensure_model(model_dir, model_path, config_path) - + super().__init__( model_path=model_path, config_path=config_path, @@ -546,6 +561,7 @@ class MelBandDereverbSeparator(MelBandSeparatorBase): repo_id=self.HF_REPO_ID, revision=self.HF_REVISION, local_dir=str(model_dir), + local_dir_use_symlinks=False, ) @@ -594,6 +610,7 @@ class MelBandKaraokeSeparator(MelBandSeparatorBase): repo_id=self.HF_REPO_ID, revision=self.HF_REVISION, local_dir=str(model_dir), + local_dir_use_symlinks=False, ) @@ -892,21 +909,21 @@ class MelBandKaraokeGaboxSeparator(MelBandSeparatorBase): cpu_separation=cpu_separation, output_format=output_format, mp3_bitrate=mp3_bitrate, - stems=['vocals', 'karaoke'], + stems=['leading', 'instrumental'], diagnostics_enabled=diagnostics_enabled, diagnostics_cb=diagnostics_cb, dim_t_override=dim_t_override, segment_secs_override=segment_secs_override, ) - + def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path): if model_path.exists() and config_path.exists(): return model_dir.mkdir(parents=True, exist_ok=True) - + try: from huggingface_hub import hf_hub_download - + if not model_path.exists(): hf_hub_download( repo_id=self.HF_REPO_ID, @@ -949,6 +966,87 @@ class MelBandKaraokeGaboxSeparator(MelBandSeparatorBase): return model +class BSRoformerKaraokeAnvuewSeparator(MelBandSeparatorBase): + """BS-RoFormer Karaoke Anvuew checkpoint (karaoke_bs_roformer_anvuew).""" + + HF_REPO_ID = 'miercolesv/MVSEP-Central-Backup' + HF_SUBFOLDER = 'MelBandRoformers/bsroformers' + + def __init__( + self, + model_dir: Optional[str] = None, + cpu_separation: bool = True, + output_format: str = 'mp3', + mp3_bitrate: str = '256k', + diagnostics_enabled: bool = False, + diagnostics_cb=None, + dim_t_override: Optional[int] = None, + segment_secs_override: Optional[float] = None, + ): + model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/karaoke_gabox') + model_path = model_dir / 'karaoke_bs_roformer_anvuew.ckpt' + config_path = model_dir / 'karaoke_bs_roformer_anvuew.yaml' + + self._ensure_model(model_dir, model_path, config_path) + + super().__init__( + model_path=model_path, + config_path=config_path, + cpu_separation=cpu_separation, + output_format=output_format, + mp3_bitrate=mp3_bitrate, + stems=['leading', 'instrumental'], + diagnostics_enabled=diagnostics_enabled, + diagnostics_cb=diagnostics_cb, + dim_t_override=dim_t_override, + segment_secs_override=segment_secs_override, + ) + + def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path): + if model_path.exists() and config_path.exists(): + return + model_dir.mkdir(parents=True, exist_ok=True) + try: + from huggingface_hub import hf_hub_download + if not model_path.exists(): + hf_hub_download( + repo_id=self.HF_REPO_ID, + filename=f"{self.HF_SUBFOLDER}/karaoke_bs_roformer_anvuew.ckpt", + local_dir=model_dir, + local_dir_use_symlinks=False, + ) + if not config_path.exists(): + hf_hub_download( + repo_id=self.HF_REPO_ID, + filename=f"{self.HF_SUBFOLDER}/karaoke_bs_roformer_anvuew.yaml", + local_dir=model_dir, + local_dir_use_symlinks=False, + ) + except Exception as e: + raise RuntimeError(f"Failed to download BS-RoFormer Karaoke Anvuew model: {e}") + + def get_model(self): + """Anvuew karaoke checkpoint uses BS-RoFormer architecture.""" + config = self._load_config() + model_cfg = OmegaConf.to_container(config.model, resolve=True) + model = _build_roformer_model( + BSRoformer, + model_cfg, + config_path=self.config_path, + model_label="BSRoformer", + ) + checkpoint = torch.load(str(self.model_path), map_location='cpu', weights_only=False) + _load_roformer_checkpoint( + model, + checkpoint, + model_path=self.model_path, + config_path=self.config_path, + model_label="BSRoformer Karaoke Anvuew", + ) + model.to(self.device).eval() + return model + + class MelBandKaraokeFrazerBecruilySeparator(MelBandSeparatorBase): """BS-RoFormer Karaoke Frazer+Becruily checkpoint.""" @@ -977,7 +1075,7 @@ class MelBandKaraokeFrazerBecruilySeparator(MelBandSeparatorBase): cpu_separation=cpu_separation, output_format=output_format, mp3_bitrate=mp3_bitrate, - stems=['vocals', 'karaoke'], + stems=['leading', 'instrumental'], diagnostics_enabled=diagnostics_enabled, diagnostics_cb=diagnostics_cb, dim_t_override=dim_t_override, @@ -1064,7 +1162,7 @@ class MelBandKaraokeGaboxSmallSeparator(MelBandSeparatorBase): cpu_separation=cpu_separation, output_format=output_format, mp3_bitrate=mp3_bitrate, - stems=['vocals', 'karaoke'], + stems=['leading', 'instrumental'], diagnostics_enabled=diagnostics_enabled, diagnostics_cb=diagnostics_cb, dim_t_override=dim_t_override, @@ -1129,7 +1227,7 @@ class MelBandKaraokeGaboxV2ExperimentalSeparator(MelBandSeparatorBase): cpu_separation=cpu_separation, output_format=output_format, mp3_bitrate=mp3_bitrate, - stems=['vocals', 'karaoke'], + stems=['leading', 'instrumental'], diagnostics_enabled=diagnostics_enabled, diagnostics_cb=diagnostics_cb, dim_t_override=dim_t_override, @@ -1193,7 +1291,7 @@ class MelBandKaraokeGaboxV1Separator(MelBandSeparatorBase): cpu_separation=cpu_separation, output_format=output_format, mp3_bitrate=mp3_bitrate, - stems=['vocals', 'karaoke'], + stems=['leading', 'instrumental'], diagnostics_enabled=diagnostics_enabled, diagnostics_cb=diagnostics_cb, dim_t_override=dim_t_override, diff --git a/backend/services/separator_service.py b/backend/services/separator_service.py index 2a7b8c0..9b261d1 100755 --- a/backend/services/separator_service.py +++ b/backend/services/separator_service.py @@ -635,6 +635,17 @@ def _run_model_separation( chunk_duration_secs=chunk_duration_secs, chunk_overlap_secs=chunk_overlap_secs, ) + if model == 'mdxnet_drumsep': + return _run_mdx23c( + job_id, input_path, model, normalised_format, stems_dir, + mp3_bitrate=mp3_bitrate, metadata=metadata, + stem_prefix=stem_prefix, + chunk_size_override=power_opts.get('scnet', {}).get('chunk_size'), + num_overlap_override=power_opts.get('scnet', {}).get('num_overlap'), + chunking_mode=chunking_mode, + chunk_duration_secs=chunk_duration_secs, + chunk_overlap_secs=chunk_overlap_secs, + ) return _run_mdxnet( job_id, input_path, model, normalised_format, stems_dir, mp3_bitrate=mp3_bitrate, metadata=metadata, @@ -1944,8 +1955,8 @@ def _run_mel_band_karaoke( sep_class = MelBandKaraokeGaboxSeparator model_name = "MelBand Karaoke Gabox" elif model == 'bs_roformer_karaoke_anvuew': - from .mel_band_specialized_separator import MelBandKaraokeGaboxSeparator - sep_class = MelBandKaraokeGaboxSeparator + from .mel_band_specialized_separator import BSRoformerKaraokeAnvuewSeparator + sep_class = BSRoformerKaraokeAnvuewSeparator model_name = "BS-RoFormer Karaoke Anvuew" elif model == 'bs_roformer_karaoke_frazer_becruily': from .mel_band_specialized_separator import MelBandKaraokeFrazerBecruilySeparator @@ -2845,6 +2856,140 @@ def _run_scnet( +# --------------------------------------------------------------------------- +# MDX23C TFC-TDF U-Net (Python API) +# --------------------------------------------------------------------------- + +def _run_mdx23c( + job_id: str, + input_path: str, + model: str, + output_format: str, + stems_dir: Path, + mp3_bitrate: str = '256k', + metadata: Optional[Dict] = None, + stem_prefix: Optional[str] = None, + chunk_size_override: Optional[int] = None, + num_overlap_override: Optional[int] = None, + chunking_mode: Optional[str] = None, + chunk_duration_secs: Optional[int] = None, + chunk_overlap_secs: Optional[int] = None, +) -> Dict[str, str]: + """Run MDX23C TFC-TDF U-Net separation (drum component separation).""" + from .mdx23c_separator import MDX23CSeparator + from .separation.chunking import plan_chunks, should_chunk + + jm.update_job( + job_id, status=jm.JobStatus.RUNNING, + progress=f"Loading MDX23C model ({model})...", + ) + + if jm.should_cancel(job_id): + raise RuntimeError("Cancelled by user") + + try: + import gc, torch + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + + sep = MDX23CSeparator( + model_id=model, + cpu_separation=CPU_SEPARATION, + output_format=output_format, + mp3_bitrate=mp3_bitrate, + ) + + jm.update_job(job_id, progress="Separating with MDX23C... (this may take a while)") + + temp_out = JOBS_DIR / job_id / "mdx23c_out" + temp_out.mkdir(parents=True, exist_ok=True) + + _dur = get_audio_duration_secs(input_path) + _chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE) + _chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS) + _chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS) + _use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS) + _cs = chunk_size_override + + for _attempt in range(2): + try: + if _use_chunking: + _total_samples = max(1, int(round(_dur * 44100))) + _chunk_plan = plan_chunks( + _total_samples, 44100, + _chunk_secs, + _chunk_ov, + ) + _n = len(_chunk_plan.chunks) + jm.update_job( + job_id, + progress=( + f"Chunked mode: {_n} chunk(s) × {_chunk_secs} s " + f"with {_chunk_ov} s crossfade overlap..." + ), + ) + stem_paths = sep.separate_into_parts_chunked( + input_path, str(temp_out), + chunk_plan=_chunk_plan, + chunk_size_override=_cs, + num_overlap_override=num_overlap_override, + progress_cb=lambda msg: jm.update_job(job_id, progress=msg), + ) + else: + stem_paths = sep.separate_into_parts( + input_path, str(temp_out), + chunk_size_override=_cs, + num_overlap_override=num_overlap_override, + ) + break + except Exception as _exc: + _is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__ + if _is_oom and _attempt == 0: + import gc, torch + jm.update_job(job_id, progress="GPU OOM — retrying with smaller chunk size...") + gc.collect() + torch.cuda.empty_cache() + _cs = (_cs or 130560) // 2 + continue + try: + shutil.rmtree(temp_out, ignore_errors=True) + except Exception: + pass + raise + + if jm.should_cancel(job_id): + raise RuntimeError("Cancelled by user") + + _prefix = stem_prefix or Path(input_path).stem + stem_files = {} + + try: + for stem_name, path in sorted(stem_paths.items()): + src = Path(path) + display_name = stem_name.replace('_', ' ').title() + dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix) + shutil.move(str(src), str(dest)) + copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True) + stem_files[stem_name.lower()] = str(dest) + + try: + shutil.rmtree(temp_out, ignore_errors=True) + except Exception: + pass + + return stem_files + + except Exception: + try: + shutil.rmtree(temp_out, ignore_errors=True) + except Exception: + pass + raise + + # --------------------------------------------------------------------------- # BS-RoFormer SW (Swin Transformer variant) (Python API) # --------------------------------------------------------------------------- diff --git a/backend/services/transcriber_service.py b/backend/services/transcriber_service.py index de847fd..6b2c97c 100755 --- a/backend/services/transcriber_service.py +++ b/backend/services/transcriber_service.py @@ -2,7 +2,7 @@ MIDI transcription dispatcher. Routes to the appropriate engine based on the mode prefix: - 'polyphonic' | 'melody' → basic-pitch (in-process, fast) + 'polyphonic' | 'melody' | 'monophonic' | 'guitar' | 'bass' → basic-pitch (in-process, fast) 'omnizart-*' → Omnizart CLI or sidecar HTTP (requires OMNIZART_ENABLED=true) """ import logging @@ -18,8 +18,8 @@ def run_transcription(job_id: str, input_path: str, mode: str) -> None: """ Background task: transcribe audio to MIDI. - mode: 'polyphonic' | 'melody' → basic-pitch - 'omnizart-music' → Omnizart music mode + mode: 'polyphonic' | 'melody' | 'monophonic' | 'guitar' | 'bass' → basic-pitch + 'omnizart-music' → Omnizart music mode 'omnizart-vocal' → Omnizart vocal mode 'omnizart-vocal-contour' → Omnizart vocal-contour mode """ diff --git a/backend/static/index.html b/backend/static/index.html index ec74f9d..7af2dcf 100644 --- a/backend/static/index.html +++ b/backend/static/index.html @@ -19,8 +19,8 @@ - - + +
diff --git a/frontend/src/components/AutoPipelineDialog.tsx b/frontend/src/components/AutoPipelineDialog.tsx index 6f1c3fb..9f4566a 100755 --- a/frontend/src/components/AutoPipelineDialog.tsx +++ b/frontend/src/components/AutoPipelineDialog.tsx @@ -232,7 +232,7 @@ export default function AutoPipelineDialog({ target, onClose, onComplete, onOpen const modelId = step.model ?? step.whisperModel ?? (engine === 'parakeet' ? 'parakeet-1.1b-rnnt-multilingual-asr' : 'large-v2') await withRetry(async () => { patchStep(stepIdx, { detail: `${engine === 'parakeet' ? 'Parakeet' : 'WhisperX'} ${modelId}…` }) - const { job_id: transcribeJobId } = await lyricTranscribeProject(projectId, 'vocals', { + const { job_id: transcribeJobId } = await lyricTranscribeProject(projectId, step.stem ?? 'vocals', { engine, model: modelId, wordTimestamps: wordTs, diff --git a/frontend/src/pages/ProjectDetail.tsx b/frontend/src/pages/ProjectDetail.tsx index d6222fd..3149bf0 100755 --- a/frontend/src/pages/ProjectDetail.tsx +++ b/frontend/src/pages/ProjectDetail.tsx @@ -1340,6 +1340,9 @@ function MixesSection({ const FALLBACK_MODES: ModeOption[] = [ { id: 'polyphonic', label: 'Polyphonic', engine: 'basic-pitch', available: true }, { id: 'melody', label: 'Melody/Vocal', engine: 'basic-pitch', available: true }, + { id: 'monophonic', label: 'Monophonic', engine: 'basic-pitch', available: true }, + { id: 'guitar', label: 'Guitar', engine: 'basic-pitch', available: true }, + { id: 'bass', label: 'Bass', engine: 'basic-pitch', available: true }, ] /** Render a grouped setTxStemName(e.target.value)} + className="rounded-lg ui-surface border ui-border px-2.5 py-1.5 text-xs ui-text focus:outline-none focus:border-violet-400 ui-focus-ring" + title="Stem to transcribe" + > + {txStemOptions.map(n => ( + + ))} + + ) : null + })()} patchStep(idx, { stem: e.target.value } as Partial)} + className={AP_SELECT} + > + + + + +