Commit
First push
This commit is contained in:
commit
c4a020c191
146 files changed
+40796
No files matched your search
Executable
+9
@@ -0,0 +1,9 @@
|
||||
frontend/node_modules
|
||||
backend/__pycache__
|
||||
backend/**/__pycache__
|
||||
**/*.pyc
|
||||
*.db
|
||||
media/
|
||||
uploads/
|
||||
jobs/
|
||||
imports/
|
||||
Executable
+70
@@ -0,0 +1,70 @@
|
||||
# =============================================================================
|
||||
# StemEngine Git Ignore
|
||||
# =============================================================================
|
||||
|
||||
# Environment files (contain secrets)
|
||||
.env
|
||||
.env.local
|
||||
.env.*.local
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
|
||||
# Node.js / Frontend
|
||||
node_modules/
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
# Generated static files (built by frontend)
|
||||
backend/static/assets/
|
||||
backend/static/*.js
|
||||
backend/static/*.css
|
||||
backend/static/*.map
|
||||
|
||||
# Data directories (user content)
|
||||
jobs/
|
||||
uploads/
|
||||
lrc_sync/
|
||||
imports/
|
||||
db/
|
||||
./db/
|
||||
projects/
|
||||
|
||||
# Model caches (large files, downloaded at runtime)
|
||||
# These are typically mounted as volumes
|
||||
*.pt
|
||||
*.pth
|
||||
*.onnx
|
||||
*.bin
|
||||
*.safetensors
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Logs
|
||||
logs/
|
||||
*.log
|
||||
|
||||
# Model storage volumes (large binaries, mounted at runtime)
|
||||
opt/
|
||||
stemengine-models/
|
||||
|
||||
# Downloaded packages
|
||||
*.deb
|
||||
Executable
+99
@@ -0,0 +1,99 @@
|
||||
# ============================================================
|
||||
# Single stage: Python runtime + Node.js
|
||||
# ============================================================
|
||||
# CUDA 12.4 + cuDNN 9 runtime — provides GPU support for:
|
||||
# • PyTorch (cu124 wheels bundle their own cuBLAS/cuDNN)
|
||||
# • onnxruntime-gpu (requires system libcudnn; uses CUDAExecutionProvider for
|
||||
# MDX-Net separation and basic-pitch ONNX inference)
|
||||
# • libcudnn8 is also installed alongside cuDNN 9 because WhisperX's phoneme
|
||||
# alignment model (wav2vec2 via pyannote / CTranslate2) links against the
|
||||
# cuDNN 8 ABI (.so.8 filenames) which cuDNN 9 no longer ships.
|
||||
#
|
||||
# Node.js is also installed here so that start.sh can run
|
||||
# `vite build --watch`, keeping the frontend live without a Docker rebuild.
|
||||
# node_modules are installed once at first container start and persisted in
|
||||
# the stemengine-node-modules named volume.
|
||||
FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# ---------- System dependencies ----------
|
||||
# Add deadsnakes PPA for Python 3.11 (not available in default Ubuntu 22.04 repos).
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
gnupg \
|
||||
&& mkdir -p /etc/apt/keyrings \
|
||||
&& curl -fsSL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xF23C5A6CF475977595C89F51BA6932366A755776" \
|
||||
| gpg --dearmor -o /etc/apt/keyrings/deadsnakes.gpg \
|
||||
&& echo "deb [signed-by=/etc/apt/keyrings/deadsnakes.gpg] https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy main" \
|
||||
> /etc/apt/sources.list.d/deadsnakes.list \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
ffmpeg \
|
||||
abcmidi \
|
||||
libcudnn8 \
|
||||
python3.11 \
|
||||
python3.11-dev \
|
||||
python3.11-venv \
|
||||
build-essential \
|
||||
git \
|
||||
libsndfile1 \
|
||||
libsndfile1-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Bootstrap pip for Python 3.11
|
||||
RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11
|
||||
|
||||
# ---------- Node.js 20.x ----------
|
||||
# Required for `vite build --watch` which runs at container startup via start.sh.
|
||||
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
|
||||
&& apt-get install -y --no-install-recommends nodejs \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# ---------- Main Python 3.11 dependencies ----------
|
||||
WORKDIR /app
|
||||
|
||||
COPY backend/requirements.txt /app/backend/requirements.txt
|
||||
RUN python3.11 -m pip install --no-cache-dir -r /app/backend/requirements.txt
|
||||
|
||||
# Install CUDA-enabled PyTorch (CUDA 12.4 wheels; self-contained, does not need system CUDA 12.4)
|
||||
# torch 2.6+ is required — earlier versions have a serious torch.load vulnerability (even with
|
||||
# weights_only=True). cu121 wheels were discontinued after torch 2.4; use cu124 going forward.
|
||||
RUN python3.11 -m pip install --no-cache-dir \
|
||||
torch==2.6.0 torchaudio==2.6.0 torchvision==0.21.0 \
|
||||
--index-url https://download.pytorch.org/whl/cu124
|
||||
|
||||
# Note: Model weights are downloaded on-demand at runtime on first use.
|
||||
# This keeps the Docker image smaller and avoids build-time network dependencies.
|
||||
|
||||
# ---------- Application source ----------
|
||||
COPY backend/ /app/backend/
|
||||
COPY lyrics-sync/ /app/lyrics-sync/
|
||||
|
||||
# Frontend source is bind-mounted at runtime (${SEFOLDER}/frontend:/app/frontend).
|
||||
# Copy it here so the image has a fallback if the bind-mount is absent,
|
||||
# and so COPY frontend/package*.json is available for the layer cache.
|
||||
COPY frontend/ /app/frontend/
|
||||
|
||||
# ---------- Startup script ----------
|
||||
COPY start.sh /app/start.sh
|
||||
RUN chmod +x /app/start.sh
|
||||
|
||||
# ---------- Runtime ----------
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
ENV MEDIA_DIR=/app/media
|
||||
ENV CPU_SEPARATION=true
|
||||
ENV BS_ROFORMER_MODEL_DIR=/opt/bs_roformer_models
|
||||
ENV MEL_BAND_MODEL_DIR=/opt/mel_band_models
|
||||
ENV MDXNET_MODEL_DIR=/opt/mdxnet_models
|
||||
ENV SCNET_MODEL_DIR=/opt/scnet_models
|
||||
ENV MEDLEYVOX_MODEL_DIR=/opt/medleyvox_models
|
||||
ENV APOLLO_MODEL_DIR=/opt/apollo_models
|
||||
ENV ASPIRATION_MODEL_DIR=/opt/aspiration_models
|
||||
ENV BANDIT_MODEL_DIR=/opt/bandit_models
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
CMD ["/app/start.sh"]
|
||||
@@ -0,0 +1,142 @@
|
||||
# StemEngine - Karaoke Anything?
|
||||
|
||||
> This is a **vibe-coded** project based on the work of people who actually know what they're doing. But now I can theoretically karaoke anything and so can you.
|
||||
|
||||
---
|
||||
|
||||
## Features
|
||||
|
||||
- **Stem Separation** — 40+ models across 10 model families
|
||||
- **MIDI Transcription** — basic-pitch with polyphonic and melody modes
|
||||
- **ABC Notation** — convert MIDI to ABC notation for sheet music
|
||||
- **Lyrics Sync** — forced-alignment to produce ELRC files with word-level timestamps and a live animated preview
|
||||
- **Project Management** — persistent project tabs, per-stem reprocessing, stem renaming
|
||||
- **Mix Presets** — Instrumental, Static (remove elements), Dynamic (per-stem volume)
|
||||
- **Subsonic Integration** — browse and import from a Subsonic-compatible server
|
||||
- **Model Downloader** — download model weights on-demand from the Settings UI
|
||||
|
||||
---
|
||||
|
||||
## Separation Models
|
||||
|
||||
| Family | Models |
|
||||
|---|---|
|
||||
| **Demucs** | v4, v4 Fine-tuned, v3 Extra Quantized, v3 |
|
||||
| **BS-RoFormer** | 4-stem, 5-stem (Guitar), 5-stem (Piano), 6-stem, Chorus, Anvuew, Karaoke Anvuew, SW 6-stem |
|
||||
| **MelBand RoFormer** | 4-stem, Vocals, Vocals Anvuew, Dereverb, Dereverb Anvuew, Dereverb Room, Karaoke, Karaoke Gabox, Deux Becruily |
|
||||
| **MDX-Net** | Vocals FT, Inst HQ 1, Inst HQ 2, Karaoke, Dereverb, Drum Sep |
|
||||
| **SCNet** | XL (2-stem), XL IHF, Large (4-stem), Large Starrytong |
|
||||
| **MedleyVox** | Base, 55/66/88/100 Epoch variants |
|
||||
| **Apollo** | Vocals, EDM, EDM Big |
|
||||
| **Aspiration** | Full, Light |
|
||||
| **Bandit v2** | Multi-Language, English |
|
||||
| **Ensemble** | Vocal Best Quality, 4-Stem, 6-Stem Full |
|
||||
|
||||
Model weights are downloaded on first use and cached in `/opt/`.
|
||||
|
||||
---
|
||||
|
||||
## Requirements
|
||||
|
||||
- Docker with the NVIDIA Container Toolkit (for GPU acceleration)
|
||||
- NVIDIA GPU with CUDA 12.4 support
|
||||
- Docker Compose
|
||||
|
||||
CPU-only mode is available via `CPU_SEPARATION=true` but separation will be significantly slower.
|
||||
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
1. Copy `.env.example` to `.env` and fill in the required values (see Configuration below).
|
||||
|
||||
2. Build and start:
|
||||
|
||||
```bash
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
3. Open `http://localhost:8000` in your browser.
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
All configuration is via environment variables. The `docker-compose.yml` accepts these via a `.env` file in the same directory.
|
||||
|
||||
### Required
|
||||
|
||||
| Variable | Description |
|
||||
|---|---|
|
||||
| `SEFOLDER` | Host path for persistent runtime data (uploads, jobs, DB, LRC sync) |
|
||||
| `ENGINEFOLDER` | Host path for model weight caches |
|
||||
| `PROJECTS` | Host path for separated project output |
|
||||
| `SECRET_KEY` | Random hex key for encrypting Subsonic credentials at rest. Generate with: `python3 -c "import secrets; print(secrets.token_hex(32))"` |
|
||||
|
||||
### Optional
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `CPU_SEPARATION` | `false` | Set to `true` to force CPU inference |
|
||||
| `MAX_UPLOAD_MB` | `500` | Maximum upload file size in MB |
|
||||
| `TZ` | `UTC` | Timezone |
|
||||
| `NAVSERVER` | — | Subsonic server URL (can also be set in the UI) |
|
||||
| `NAVUSER` | — | Subsonic username |
|
||||
| `NAVPASS` | — | Subsonic password |
|
||||
|
||||
### Model Directories
|
||||
|
||||
These are set automatically in `docker-compose.yml` and point inside the `/opt/` volume. Override only if you need a custom layout.
|
||||
|
||||
`BS_ROFORMER_MODEL_DIR`, `MEL_BAND_MODEL_DIR`, `MDXNET_MODEL_DIR`, `SCNET_MODEL_DIR`, `MEDLEYVOX_MODEL_DIR`, `APOLLO_MODEL_DIR`, `ASPIRATION_MODEL_DIR`, `BANDIT_MODEL_DIR`
|
||||
|
||||
---
|
||||
|
||||
## Project Folder Structure
|
||||
|
||||
Each separation job creates a folder under `PROJECTS_DIR`:
|
||||
|
||||
```
|
||||
Artist - Title/
|
||||
Artist - Title.mp3 # Original source audio (copy)
|
||||
Artist - Title.elrc # ELRC lyrics with word-level timestamps (if aligned)
|
||||
Artist - Title.lrc # Plain LRC (if aligned)
|
||||
Stems/
|
||||
vocals.mp3
|
||||
drums.mp3
|
||||
bass.mp3
|
||||
other.mp3
|
||||
Instrumental/ # Only when Instrumental preset selected
|
||||
Artist - Title - Instrumental.mp3
|
||||
Static/ # Only when Static preset selected
|
||||
Artist - Title - No Vocals.mp3
|
||||
Dynamic/ # Only when Dynamic preset selected
|
||||
Artist - Title - Dynamic Mix.mp3
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Tech Stack
|
||||
|
||||
- **Frontend** — React 18, TypeScript, Vite, TailwindCSS
|
||||
- **Backend** — FastAPI, Python 3.11, SQLAlchemy
|
||||
- **Separation** — Demucs, BS-RoFormer, MelBand RoFormer, MDX-Net (onnxruntime-gpu), SCNet, MedleyVox, Apollo, Aspiration, Bandit
|
||||
- **Transcription** — basic-pitch (ONNX, CUDAExecutionProvider)
|
||||
- **Lyrics Sync** — wav2vec2-based forced alignment via custom `lsync` module
|
||||
- **Runtime** — NVIDIA CUDA 12.4 + cuDNN 9, PyTorch 2.6 (cu124 wheels)
|
||||
- **Container** — `nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04`
|
||||
|
||||
---
|
||||
|
||||
## GPU Notes
|
||||
|
||||
The container requires the NVIDIA Container Toolkit. On the host:
|
||||
|
||||
```bash
|
||||
# Verify GPU is visible inside the container
|
||||
docker run --rm --gpus all nvidia/cuda:12.4.1-base-ubuntu22.04 nvidia-smi
|
||||
```
|
||||
|
||||
PyTorch wheels are self-contained (cu124) and do not require a matching CUDA toolkit on the host — only the driver.
|
||||
|
||||
`onnxruntime-gpu` is used for MDX-Net and basic-pitch ONNX inference; it auto-selects `CUDAExecutionProvider` when a GPU is available.
|
||||
Executable
Whitespace-only changes.
Executable
+265
@@ -0,0 +1,265 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# Base paths — each can be overridden independently via its own env var,
|
||||
# or all inherit from MEDIA_DIR when left unset.
|
||||
MEDIA_DIR = Path(os.environ.get("MEDIA_DIR", "/app/media"))
|
||||
UPLOAD_DIR = Path(os.environ.get("UPLOAD_DIR", str(MEDIA_DIR / "uploads")))
|
||||
JOBS_DIR = Path(os.environ.get("JOBS_DIR", str(MEDIA_DIR / "jobs")))
|
||||
PROJECTS_DIR = Path(os.environ.get("PROJECTS_DIR", str(MEDIA_DIR / "projects")))
|
||||
DB_PATH = Path(os.environ.get("DB_PATH", str(MEDIA_DIR / "db" / "stemengine.db")))
|
||||
LRC_SYNC_DIR = Path(os.environ.get("LRC_SYNC_DIR", str(MEDIA_DIR / "lrc_sync")))
|
||||
|
||||
# BS-RoFormer model directory
|
||||
BS_ROFORMER_MODEL_DIR = Path(os.environ.get("BS_ROFORMER_MODEL_DIR", "/opt/bs_roformer_models"))
|
||||
|
||||
# MelBand RoFormer model directory
|
||||
MEL_BAND_MODEL_DIR = Path(os.environ.get("MEL_BAND_MODEL_DIR", "/opt/mel_band_models"))
|
||||
|
||||
# MDX-Net model directory
|
||||
MDXNET_MODEL_DIR = Path(os.environ.get("MDXNET_MODEL_DIR", "/opt/mdxnet_models"))
|
||||
|
||||
# SCNet XL model directory
|
||||
SCNET_MODEL_DIR = Path(os.environ.get("SCNET_MODEL_DIR", "/opt/scnet_models"))
|
||||
|
||||
# MedleyVox model directory
|
||||
MEDLEYVOX_MODEL_DIR = Path(os.environ.get("MEDLEYVOX_MODEL_DIR", "/opt/medleyvox_models"))
|
||||
|
||||
# Apollo model directory
|
||||
APOLLO_MODEL_DIR = Path(os.environ.get("APOLLO_MODEL_DIR", "/opt/apollo_models"))
|
||||
|
||||
# Bandit model directory
|
||||
BANDIT_MODEL_DIR = Path(os.environ.get("BANDIT_MODEL_DIR", "/opt/bandit_models"))
|
||||
|
||||
# Aspiration model directory
|
||||
ASPIRATION_MODEL_DIR = Path(os.environ.get("ASPIRATION_MODEL_DIR", "/opt/aspiration_models"))
|
||||
|
||||
# Ensemble mode cache directory
|
||||
ENSEMBLE_CACHE_DIR = Path(os.environ.get("ENSEMBLE_CACHE_DIR", "/app/media/ensemble_cache"))
|
||||
|
||||
# Separation models — Demucs + BS-RoFormer
|
||||
DEMUCS_MODELS = [
|
||||
{"id": "htdemucs", "label": "Demucs v4", "group": "Demucs"},
|
||||
{"id": "htdemucs_ft", "label": "Demucs v4 Fine-tuned", "group": "Demucs"},
|
||||
{"id": "mdx_extra_q", "label": "Demucs v3 Extra Quantized","group": "Demucs"},
|
||||
{"id": "mdx", "label": "Demucs v3", "group": "Demucs"},
|
||||
]
|
||||
|
||||
BS_ROFORMER_MODELS = [
|
||||
{"id": "bs_roformer_4stem", "label": "BS-RoFormer 4-stem", "group": "BS-RoFormer"},
|
||||
{"id": "bs_roformer_5stem_guitar", "label": "BS-RoFormer 5-stem (+ Guitar)", "group": "BS-RoFormer"},
|
||||
{"id": "bs_roformer_5stem_piano", "label": "BS-RoFormer 5-stem (+ Piano)", "group": "BS-RoFormer"},
|
||||
{"id": "bs_roformer_6stem", "label": "BS-RoFormer 6-stem (full)", "group": "BS-RoFormer"},
|
||||
]
|
||||
|
||||
# MelBand RoFormer models
|
||||
MEL_BAND_MODELS = [
|
||||
{"id": "mel_band_roformer_4stem", "label": "MelBand RoFormer 4-stem", "group": "MelBand RoFormer"},
|
||||
{"id": "mel_band_vocals", "label": "MelBand Vocals (2-stem)", "group": "MelBand RoFormer"},
|
||||
{"id": "mel_band_vocals_anvuew", "label": "MelBand Vocals Anvuew", "group": "MelBand RoFormer"},
|
||||
{"id": "mel_band_dereverb", "label": "MelBand Dereverb", "group": "MelBand RoFormer"},
|
||||
{"id": "mel_band_dereverb_anvuew", "label": "MelBand Dereverb Anvuew", "group": "MelBand RoFormer"},
|
||||
{"id": "mel_band_dereverb_room", "label": "MelBand Dereverb Room", "group": "MelBand RoFormer"},
|
||||
{"id": "mel_band_karaoke", "label": "MelBand Karaoke (Lead/Back)", "group": "MelBand RoFormer"},
|
||||
{"id": "mel_band_karaoke_gabox", "label": "MelBand Karaoke Gabox", "group": "MelBand RoFormer"},
|
||||
{"id": "mel_band_deux_becruily", "label": "MelBand RoFormer Deux Becruily", "group": "MelBand RoFormer"},
|
||||
]
|
||||
|
||||
# BS-RoFormer specialized models
|
||||
BS_ROFORMER_SPECIALIZED_MODELS = [
|
||||
{"id": "bs_roformer_chorus", "label": "BS-RoFormer Male/Female Chorus", "group": "BS-RoFormer"},
|
||||
{"id": "bs_roformer_anvuew", "label": "BS-RoFormer Anvuew (Vocals)", "group": "BS-RoFormer"},
|
||||
{"id": "bs_roformer_karaoke_anvuew", "label": "BS-RoFormer Karaoke Anvuew", "group": "BS-RoFormer"},
|
||||
]
|
||||
|
||||
# MDX-Net models (from Ultimate Vocal Remover)
|
||||
MDXNET_MODELS = [
|
||||
{"id": "mdxnet_voc_ft", "label": "MDX-Net Vocals FT", "group": "MDX-Net"},
|
||||
{"id": "mdxnet_inst_hq_1", "label": "MDX-Net Inst HQ 1", "group": "MDX-Net"},
|
||||
{"id": "mdxnet_inst_hq_2", "label": "MDX-Net Inst HQ 2", "group": "MDX-Net"},
|
||||
{"id": "mdxnet_karaoke", "label": "MDX-Net Karaoke", "group": "MDX-Net"},
|
||||
{"id": "mdxnet_dereverb", "label": "MDX-Net Dereverb", "group": "MDX-Net"},
|
||||
{"id": "mdxnet_drumsep", "label": "MDX-Net Drum Separation", "group": "MDX-Net"},
|
||||
]
|
||||
|
||||
# SCNet XL models (state-of-the-art 2024-2025)
|
||||
SCNET_MODELS = [
|
||||
{"id": "scnet_xl", "label": "SCNet XL (2-stem)", "group": "SCNet"},
|
||||
{"id": "scnet_xl_ihf", "label": "SCNet XL IHF (2-stem)", "group": "SCNet"},
|
||||
{"id": "scnet_large", "label": "SCNet Large (4-stem)", "group": "SCNet"},
|
||||
{"id": "scnet_large_starrytong", "label": "SCNet Large Starrytong", "group": "SCNet"},
|
||||
]
|
||||
|
||||
# BS-RoFormer SW (Swin Transformer variant)
|
||||
BS_ROFORMER_SW_MODELS = [
|
||||
{"id": "bs_roformer_sw_6stem", "label": "BS-RoFormer SW 6-stem", "group": "BS-RoFormer SW"},
|
||||
]
|
||||
|
||||
# MedleyVox models (specialized vocal separation)
|
||||
MEDLEYVOX_MODELS = [
|
||||
{"id": "medleyvox", "label": "MedleyVox (Vocals)", "group": "MedleyVox"},
|
||||
{"id": "medleyvox_55ep", "label": "MedleyVox 55 Epochs", "group": "MedleyVox"},
|
||||
{"id": "medleyvox_66ep", "label": "MedleyVox 66 Epochs", "group": "MedleyVox"},
|
||||
{"id": "medleyvox_88ep", "label": "MedleyVox 88 Epochs", "group": "MedleyVox"},
|
||||
{"id": "medleyvox_100ep", "label": "MedleyVox 100 Epochs", "group": "MedleyVox"},
|
||||
]
|
||||
|
||||
# Apollo models (vocal enhancement/reverb removal)
|
||||
APOLLO_MODELS = [
|
||||
{"id": "apollo_vocals", "label": "Apollo Vocals (Enhancement)", "group": "Apollo"},
|
||||
{"id": "apollo_edm", "label": "Apollo EDM", "group": "Apollo"},
|
||||
{"id": "apollo_edm_big", "label": "Apollo EDM Big", "group": "Apollo"},
|
||||
]
|
||||
|
||||
# Aspiration models (breath/aspiration removal)
|
||||
ASPIRATION_MODELS = [
|
||||
{"id": "aspiration_full", "label": "Aspiration Removal (Full)", "group": "Aspiration"},
|
||||
{"id": "aspiration_light", "label": "Aspiration Removal (Light)", "group": "Aspiration"},
|
||||
]
|
||||
|
||||
# Bandit-v2 models (multi-language/multi-stem)
|
||||
BANDIT_MODELS = [
|
||||
{"id": "bandit_v2_multi", "label": "Bandit v2 Multi-Language", "group": "Bandit"},
|
||||
{"id": "bandit_v2_eng", "label": "Bandit v2 English", "group": "Bandit"},
|
||||
]
|
||||
|
||||
# Ensemble modes (combines multiple models for best quality)
|
||||
ENSEMBLE_MODELS = [
|
||||
{"id": "ensemble_vocal_best", "label": "Ensemble - Vocal Best Quality", "group": "Ensemble"},
|
||||
{"id": "ensemble_4stem", "label": "Ensemble - 4-Stem", "group": "Ensemble"},
|
||||
{"id": "ensemble_6stem", "label": "Ensemble - 6-Stem Full", "group": "Ensemble"},
|
||||
]
|
||||
|
||||
ALL_MODELS = (
|
||||
DEMUCS_MODELS +
|
||||
BS_ROFORMER_MODELS +
|
||||
BS_ROFORMER_SW_MODELS +
|
||||
BS_ROFORMER_SPECIALIZED_MODELS +
|
||||
MEL_BAND_MODELS +
|
||||
MDXNET_MODELS +
|
||||
SCNET_MODELS +
|
||||
MEDLEYVOX_MODELS +
|
||||
APOLLO_MODELS +
|
||||
ASPIRATION_MODELS +
|
||||
BANDIT_MODELS +
|
||||
ENSEMBLE_MODELS
|
||||
)
|
||||
DEFAULT_DEMUCS_MODEL = "htdemucs"
|
||||
DEFAULT_MODEL = DEFAULT_DEMUCS_MODEL
|
||||
|
||||
ALL_MODEL_IDS = {m["id"] for m in ALL_MODELS}
|
||||
|
||||
# Output formats
|
||||
OUTPUT_FORMATS = [
|
||||
{"id": "mp3", "label": "MP3 (256 kbps)"},
|
||||
{"id": "mp3_320", "label": "MP3 (320 kbps)"},
|
||||
{"id": "flac", "label": "FLAC (lossless)"},
|
||||
{"id": "wav", "label": "WAV (lossless)"},
|
||||
]
|
||||
DEFAULT_OUTPUT_FORMAT = "mp3"
|
||||
|
||||
# basic-pitch transcription modes (mapped to inference parameter presets)
|
||||
TRANSCRIBE_MODES = [
|
||||
{"id": "polyphonic", "label": "Polyphonic"},
|
||||
{"id": "melody", "label": "Melody / Vocal"},
|
||||
]
|
||||
DEFAULT_TRANSCRIBE_MODE = "polyphonic"
|
||||
|
||||
# basic-pitch inference parameter presets per mode
|
||||
TRANSCRIBE_PRESETS = {
|
||||
"polyphonic": {
|
||||
"onset_threshold": 0.5,
|
||||
"frame_threshold": 0.3,
|
||||
"minimum_note_length": 58.0,
|
||||
"melodia_trick": False,
|
||||
},
|
||||
"melody": {
|
||||
"onset_threshold": 0.5,
|
||||
"frame_threshold": 0.3,
|
||||
"minimum_note_length": 58.0,
|
||||
"minimum_frequency": 65.0, # Hz — below typical singing range
|
||||
"melodia_trick": True,
|
||||
},
|
||||
}
|
||||
|
||||
# Hardware
|
||||
CPU_SEPARATION = os.environ.get("CPU_SEPARATION", "true").lower() != "false"
|
||||
|
||||
# Limits
|
||||
MAX_UPLOAD_BYTES = int(os.environ.get("MAX_UPLOAD_MB", "500")) * 1024 * 1024
|
||||
POLL_INTERVAL_SECONDS = 5.0
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Presets for stem mixing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Available preset types
|
||||
PRESET_TYPES = [
|
||||
{"id": "stems_only", "label": "Stems Only", "description": "Generate separated stems only"},
|
||||
{"id": "static", "label": "Static Mix", "description": "Remove specific elements completely"},
|
||||
{"id": "instrumental", "label": "Instrumental", "description": "Remove vocals only"},
|
||||
{"id": "dynamic", "label": "Dynamic Mix", "description": "Adjust volume levels of each element"},
|
||||
]
|
||||
|
||||
# Static preset options (what to remove)
|
||||
STATIC_OPTIONS = [
|
||||
{"id": "no_vocals", "label": "No Vocals", "description": "Remove vocals (keep drums, bass, other)"},
|
||||
{"id": "no_drums", "label": "No Drums", "description": "Remove drums (keep vocals, bass, other)"},
|
||||
{"id": "no_bass", "label": "No Bass", "description": "Remove bass (keep vocals, drums, other)"},
|
||||
{"id": "no_vocals_no_bass", "label": "No Vocals + No Bass", "description": "Remove vocals and bass"},
|
||||
{"id": "no_drums_no_bass", "label": "No Drums + No Bass", "description": "Remove drums and bass (vocals + other)"},
|
||||
]
|
||||
|
||||
# Default volume levels for dynamic preset (0.0 to 2.0, where 1.0 = original)
|
||||
DEFAULT_DYNAMIC_LEVELS = {
|
||||
"vocals": 1.0,
|
||||
"drums": 1.0,
|
||||
"bass": 1.0,
|
||||
"other": 1.0,
|
||||
}
|
||||
|
||||
# Available stems and their labels (varies by model)
|
||||
STEM_LABELS = {
|
||||
"vocals": "Vocals",
|
||||
"drums": "Drums",
|
||||
"bass": "Bass",
|
||||
"other": "Other",
|
||||
"guitar": "Guitar",
|
||||
"piano": "Piano",
|
||||
"lead": "Lead Vocals",
|
||||
"backing": "Backing Vocals",
|
||||
"male": "Male Vocals",
|
||||
"female": "Female Vocals",
|
||||
"instrumental": "Instrumental",
|
||||
"dry_vocals": "Dry Vocals",
|
||||
"reverb": "Reverb",
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Project folder structure
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
PROJECT_SUBFOLDERS = {
|
||||
"stems": "Stems", # Always generated - contains separated stems
|
||||
"instrumental": "Instrumental", # Contains instrumental mixes (no vocals)
|
||||
"static": "Static", # Contains static mixes (specific elements removed)
|
||||
"dynamic": "Dynamic", # Contains dynamic mixes (volume adjusted)
|
||||
"transcriptions": "Transcriptions", # Contains transcription outputs
|
||||
"lrc": "LRC", # Contains synced lyrics files
|
||||
}
|
||||
|
||||
# Subsonic integration
|
||||
IMPORTS_DIR = MEDIA_DIR / "imports"
|
||||
SUBSONIC_CONFIG_PATH = MEDIA_DIR / "subsonic.json"
|
||||
|
||||
# Lyric transcription (Whisper models)
|
||||
WHISPER_MODEL_DIR = Path(os.environ.get("WHISPER_MODEL_DIR", "/opt/whisper_models"))
|
||||
DEFAULT_WHISPER_MODEL = os.environ.get("DEFAULT_WHISPER_MODEL", "large-v2")
|
||||
|
||||
# Create directories on import
|
||||
UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
||||
JOBS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
PROJECTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
LRC_SYNC_DIR.mkdir(parents=True, exist_ok=True)
|
||||
IMPORTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
Executable
+401
@@ -0,0 +1,401 @@
|
||||
"""
|
||||
Job manager: SQLite-backed job state + ThreadPoolExecutor for background tasks.
|
||||
"""
|
||||
import re
|
||||
import uuid
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import create_engine, Column, String, DateTime, JSON, Text, text, event
|
||||
from sqlalchemy.orm import DeclarativeBase, Session
|
||||
from sqlalchemy.orm.attributes import flag_modified
|
||||
|
||||
from .config import DB_PATH
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DB setup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
engine = create_engine(
|
||||
f"sqlite:///{DB_PATH}",
|
||||
connect_args={"check_same_thread": False, "timeout": 30},
|
||||
)
|
||||
|
||||
|
||||
@event.listens_for(engine, "connect")
|
||||
def _set_sqlite_pragmas(dbapi_connection, _connection_record):
|
||||
"""Enable WAL mode and set busy timeout so concurrent writes don't deadlock."""
|
||||
cursor = dbapi_connection.cursor()
|
||||
cursor.execute("PRAGMA journal_mode=WAL")
|
||||
cursor.execute("PRAGMA synchronous=NORMAL")
|
||||
cursor.execute("PRAGMA busy_timeout=30000")
|
||||
cursor.close()
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class JobRecord(Base):
|
||||
__tablename__ = "jobs"
|
||||
|
||||
id = Column(String, primary_key=True)
|
||||
job_type = Column(String, nullable=False) # separate | transcribe | abc-convert | pipeline | mix
|
||||
status = Column(String, default="queued") # queued | running | done | error | cancelled
|
||||
progress = Column(String, default="")
|
||||
error = Column(Text, default="")
|
||||
files = Column(JSON, default=list) # list of {name, path} dicts
|
||||
result = Column(JSON, default=dict) # arbitrary result metadata
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
|
||||
class ProjectRecord(Base):
|
||||
__tablename__ = "projects"
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid4()))
|
||||
name = Column(String, nullable=False)
|
||||
tab_label = Column(String, default='') # display label for tab e.g. "Demucs v4_MP3 (256 kbps)"
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
sep_job_id = Column(String, nullable=False)
|
||||
trans_jobs = Column(JSON, default=dict) # stem_name → job_id
|
||||
abc_jobs = Column(JSON, default=dict) # stem_name → job_id
|
||||
lrc_jobs = Column(JSON, default=dict) # stem_name → job_id
|
||||
mix_job_id = Column(String, nullable=True)
|
||||
|
||||
|
||||
Base.metadata.create_all(engine)
|
||||
|
||||
# Idempotent migrations: add missing columns to existing tables
|
||||
for _migration_sql in (
|
||||
"ALTER TABLE jobs ADD COLUMN result JSON",
|
||||
"ALTER TABLE projects ADD COLUMN trans_jobs JSON",
|
||||
"ALTER TABLE projects ADD COLUMN abc_jobs JSON",
|
||||
"ALTER TABLE projects ADD COLUMN lrc_jobs JSON",
|
||||
"ALTER TABLE projects ADD COLUMN tab_label TEXT DEFAULT ''",
|
||||
):
|
||||
try:
|
||||
with engine.connect() as _conn:
|
||||
_conn.execute(text(_migration_sql))
|
||||
_conn.commit()
|
||||
except Exception:
|
||||
pass # Column already exists
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cancellation flags: job_id → threading.Event (set = cancel requested)
|
||||
# ---------------------------------------------------------------------------
|
||||
_cancel_flags: dict[str, threading.Event] = {}
|
||||
_flags_lock = threading.Lock()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Thread pools
|
||||
# Two executors:
|
||||
# _general_executor — multi-worker for CPU-bound tasks (e.g. ABC conversion)
|
||||
# _gpu_executor — single-worker for GPU-exclusive tasks (PyTorch + TF)
|
||||
# serialises Demucs/BS-RoFormer and Omnizart so they
|
||||
# never compete for VRAM; FIFO by submission order.
|
||||
# ---------------------------------------------------------------------------
|
||||
_general_executor = ThreadPoolExecutor(max_workers=4)
|
||||
_gpu_executor = ThreadPoolExecutor(max_workers=1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class JobStatus(str, Enum):
|
||||
QUEUED = "queued"
|
||||
RUNNING = "running"
|
||||
DONE = "done"
|
||||
ERROR = "error"
|
||||
CANCELLED = "cancelled"
|
||||
|
||||
|
||||
def create_job(job_type: str) -> str:
|
||||
job_id = str(uuid.uuid4())
|
||||
with Session(engine) as session:
|
||||
session.add(JobRecord(id=job_id, job_type=job_type))
|
||||
session.commit()
|
||||
with _flags_lock:
|
||||
_cancel_flags[job_id] = threading.Event()
|
||||
return job_id
|
||||
|
||||
|
||||
def get_job(job_id: str) -> Optional[dict]:
|
||||
with Session(engine) as session:
|
||||
record = session.get(JobRecord, job_id)
|
||||
if record is None:
|
||||
return None
|
||||
return _record_to_dict(record)
|
||||
|
||||
|
||||
def list_jobs() -> list[dict]:
|
||||
with Session(engine) as session:
|
||||
records = session.query(JobRecord).order_by(JobRecord.created_at.desc()).all()
|
||||
return [_record_to_dict(r) for r in records]
|
||||
|
||||
|
||||
def list_all_jobs() -> list[dict]:
|
||||
"""Return every job record regardless of type."""
|
||||
return list_jobs()
|
||||
|
||||
|
||||
def delete_job(job_id: str) -> bool:
|
||||
"""Permanently delete a job record. Returns True if the record existed."""
|
||||
with Session(engine) as session:
|
||||
record = session.get(JobRecord, job_id)
|
||||
if record is None:
|
||||
return False
|
||||
session.delete(record)
|
||||
session.commit()
|
||||
with _flags_lock:
|
||||
_cancel_flags.pop(job_id, None)
|
||||
return True
|
||||
|
||||
|
||||
def update_job(job_id: str, **kwargs):
|
||||
with Session(engine) as session:
|
||||
record = session.get(JobRecord, job_id)
|
||||
if record is None:
|
||||
return
|
||||
for key, value in kwargs.items():
|
||||
setattr(record, key, value)
|
||||
record.updated_at = datetime.utcnow()
|
||||
session.commit()
|
||||
|
||||
|
||||
def append_job_files(job_id: str, new_files: list):
|
||||
"""Append file entries to an existing job's file list without overwriting."""
|
||||
with Session(engine) as session:
|
||||
record = session.get(JobRecord, job_id)
|
||||
if record is None:
|
||||
return
|
||||
existing = list(record.files or [])
|
||||
existing_names = {f.get("name") for f in existing}
|
||||
for f in new_files:
|
||||
if f.get("name") not in existing_names:
|
||||
existing.append(f)
|
||||
record.files = existing
|
||||
record.updated_at = datetime.utcnow()
|
||||
session.commit()
|
||||
|
||||
|
||||
def cancel_job(job_id: str):
|
||||
with _flags_lock:
|
||||
flag = _cancel_flags.get(job_id)
|
||||
if flag:
|
||||
flag.set()
|
||||
update_job(job_id, status=JobStatus.CANCELLED)
|
||||
|
||||
|
||||
def should_cancel(job_id: str) -> bool:
|
||||
with _flags_lock:
|
||||
flag = _cancel_flags.get(job_id)
|
||||
return flag.is_set() if flag else False
|
||||
|
||||
|
||||
def submit(fn: Callable, *args, **kwargs):
|
||||
"""Submit a CPU-bound background task (multi-worker)."""
|
||||
_general_executor.submit(fn, *args, **kwargs)
|
||||
|
||||
|
||||
def submit_gpu(fn: Callable, *args, **kwargs):
|
||||
"""Submit a GPU-exclusive background task (single-worker FIFO queue).
|
||||
|
||||
Use for any job that loads PyTorch or TensorFlow models onto the GPU:
|
||||
stem separation (Demucs / BS-RoFormer), MIDI transcription (Omnizart),
|
||||
lyrics alignment (wav2vec2 + Demucs vocal extraction). Tasks run
|
||||
strictly one-at-a-time in the order they were submitted.
|
||||
"""
|
||||
_gpu_executor.submit(fn, *args, **kwargs)
|
||||
|
||||
|
||||
def _record_to_dict(record: JobRecord) -> dict:
|
||||
return {
|
||||
"id": record.id,
|
||||
"job_type": record.job_type,
|
||||
"status": record.status,
|
||||
"progress": record.progress,
|
||||
"error": record.error,
|
||||
"files": record.files or [],
|
||||
"result": record.result or {},
|
||||
"created_at": record.created_at.isoformat() if record.created_at else None,
|
||||
"updated_at": record.updated_at.isoformat() if record.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Project CRUD
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def find_or_create_project(name: str, sep_job_id: str, tab_label: str = '') -> dict:
|
||||
"""Upsert a project record by sep_job_id. Safe to call multiple times."""
|
||||
with Session(engine) as session:
|
||||
record = session.query(ProjectRecord).filter_by(sep_job_id=sep_job_id).first()
|
||||
if record is None:
|
||||
record = ProjectRecord(
|
||||
id=str(uuid4()),
|
||||
name=name,
|
||||
sep_job_id=sep_job_id,
|
||||
tab_label=tab_label,
|
||||
)
|
||||
session.add(record)
|
||||
session.commit()
|
||||
return _project_to_dict(record)
|
||||
|
||||
|
||||
def get_project(project_id: str) -> Optional[dict]:
|
||||
with Session(engine) as session:
|
||||
record = session.get(ProjectRecord, project_id)
|
||||
return _project_to_dict(record) if record else None
|
||||
|
||||
|
||||
def get_project_by_sep_job(sep_job_id: str) -> Optional[dict]:
|
||||
with Session(engine) as session:
|
||||
record = session.query(ProjectRecord).filter_by(sep_job_id=sep_job_id).first()
|
||||
return _project_to_dict(record) if record else None
|
||||
|
||||
|
||||
def update_project(project_id: str, **kwargs):
|
||||
with Session(engine) as session:
|
||||
record = session.get(ProjectRecord, project_id)
|
||||
if record is None:
|
||||
return
|
||||
for key, value in kwargs.items():
|
||||
setattr(record, key, value)
|
||||
session.commit()
|
||||
|
||||
|
||||
def list_projects() -> list:
|
||||
"""
|
||||
Return all projects newest-first. Also materialises any completed separation
|
||||
jobs that don't yet have a project record (idempotent migration).
|
||||
"""
|
||||
with Session(engine) as session:
|
||||
# Find all completed separation/pipeline jobs
|
||||
completed_sep_jobs = (
|
||||
session.query(JobRecord)
|
||||
.filter(
|
||||
JobRecord.job_type.in_(["separate", "pipeline"]),
|
||||
JobRecord.status == "done",
|
||||
)
|
||||
.all()
|
||||
)
|
||||
|
||||
existing_sep_ids = {
|
||||
r.sep_job_id for r in session.query(ProjectRecord).all()
|
||||
}
|
||||
|
||||
for job in completed_sep_jobs:
|
||||
if job.id not in existing_sep_ids:
|
||||
proj_name = _extract_project_name(job)
|
||||
new_proj = ProjectRecord(
|
||||
id=str(uuid4()),
|
||||
name=proj_name,
|
||||
sep_job_id=job.id,
|
||||
created_at=job.created_at,
|
||||
)
|
||||
session.add(new_proj)
|
||||
existing_sep_ids.add(job.id)
|
||||
|
||||
session.commit()
|
||||
|
||||
records = (
|
||||
session.query(ProjectRecord)
|
||||
.order_by(ProjectRecord.created_at.desc())
|
||||
.all()
|
||||
)
|
||||
return [_project_to_dict(r) for r in records]
|
||||
|
||||
|
||||
def _extract_project_name(job: JobRecord) -> str:
|
||||
"""Derive a human-readable project name from a separation job record."""
|
||||
# 1. Prefer result.project_name (set for jobs after result column was added)
|
||||
if job.result and isinstance(job.result, dict):
|
||||
name = job.result.get("project_name") or ""
|
||||
if name:
|
||||
return name
|
||||
pp = job.result.get("project_path") or ""
|
||||
if pp:
|
||||
return Path(pp).name
|
||||
|
||||
# 2. Derive from stem file path structure: .../ProjectName/Stems/file.mp3
|
||||
for f in (job.files or []):
|
||||
file_path = f.get("path", "")
|
||||
if not file_path:
|
||||
continue
|
||||
p = Path(file_path)
|
||||
parts = p.parts
|
||||
# Find the "Stems" component
|
||||
for i, part in enumerate(parts):
|
||||
if part == "Stems" and i > 0:
|
||||
return parts[i - 1]
|
||||
|
||||
# 3. Derive from first file name (e.g. "input_stem - Vocals.mp3" → "input_stem")
|
||||
if job.files:
|
||||
first_name = (job.files[0] or {}).get("name", "")
|
||||
stem = Path(first_name).stem # "input_stem - Vocals"
|
||||
parts = stem.rsplit(" - ", 1)
|
||||
if len(parts) == 2:
|
||||
return parts[0]
|
||||
return stem or job.id[:8]
|
||||
|
||||
return job.id[:8]
|
||||
|
||||
|
||||
def _project_to_dict(record: ProjectRecord) -> dict:
|
||||
return {
|
||||
"id": record.id,
|
||||
"name": record.name,
|
||||
"tab_label": record.tab_label or '',
|
||||
"created_at": record.created_at.isoformat() if record.created_at else None,
|
||||
"sep_job_id": record.sep_job_id,
|
||||
"trans_jobs": record.trans_jobs or {},
|
||||
"abc_jobs": record.abc_jobs or {},
|
||||
"lrc_jobs": record.lrc_jobs or {},
|
||||
"mix_job_id": record.mix_job_id,
|
||||
}
|
||||
|
||||
|
||||
def set_project_trans_job(project_id: str, stem_name: str, job_id: str):
|
||||
"""Store a per-stem transcription job_id in the project's trans_jobs dict."""
|
||||
with Session(engine) as session:
|
||||
record = session.get(ProjectRecord, project_id)
|
||||
if record is None:
|
||||
return
|
||||
jobs = dict(record.trans_jobs or {})
|
||||
jobs[stem_name] = job_id
|
||||
record.trans_jobs = jobs
|
||||
flag_modified(record, "trans_jobs")
|
||||
session.commit()
|
||||
|
||||
|
||||
def set_project_abc_job(project_id: str, stem_name: str, job_id: str):
|
||||
"""Store a per-stem ABC job_id in the project's abc_jobs dict."""
|
||||
with Session(engine) as session:
|
||||
record = session.get(ProjectRecord, project_id)
|
||||
if record is None:
|
||||
return
|
||||
jobs = dict(record.abc_jobs or {})
|
||||
jobs[stem_name] = job_id
|
||||
record.abc_jobs = jobs
|
||||
flag_modified(record, "abc_jobs")
|
||||
session.commit()
|
||||
|
||||
|
||||
def set_project_lrc_job(project_id: str, stem_name: str, job_id: str):
|
||||
"""Store a per-stem LRC alignment job_id in the project's lrc_jobs dict."""
|
||||
with Session(engine) as session:
|
||||
record = session.get(ProjectRecord, project_id)
|
||||
if record is None:
|
||||
return
|
||||
jobs = dict(record.lrc_jobs or {})
|
||||
jobs[stem_name] = job_id
|
||||
record.lrc_jobs = jobs
|
||||
flag_modified(record, "lrc_jobs")
|
||||
session.commit()
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
"""
|
||||
StemEngine — FastAPI application entry point.
|
||||
Mounts all API routers and serves the built React frontend as static files.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from .routers import separate, transcribe, abc_convert, pipeline, jobs, lrc, subsonic, projects, models, lyric_transcribe, system
|
||||
|
||||
app = FastAPI(title="StemEngine", version="1.0.0")
|
||||
|
||||
# Allow all origins during development; tighten in production if needed.
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# API routers
|
||||
app.include_router(jobs.router)
|
||||
app.include_router(separate.router)
|
||||
app.include_router(transcribe.router)
|
||||
app.include_router(abc_convert.router)
|
||||
app.include_router(pipeline.router)
|
||||
app.include_router(lrc.router)
|
||||
app.include_router(subsonic.router)
|
||||
app.include_router(projects.router)
|
||||
app.include_router(models.router)
|
||||
app.include_router(lyric_transcribe.router)
|
||||
app.include_router(system.router)
|
||||
|
||||
# Serve built React frontend from /app/backend/static
|
||||
# Mount order matters: API routes (above) always win over static assets.
|
||||
_static_dir = Path(__file__).parent / "static"
|
||||
_assets_dir = _static_dir / "assets"
|
||||
|
||||
if _static_dir.exists():
|
||||
# Serve Vite-generated hashed assets (JS, CSS, images)
|
||||
if _assets_dir.exists():
|
||||
app.mount("/assets", StaticFiles(directory=str(_assets_dir)), name="assets")
|
||||
|
||||
@app.get("/", include_in_schema=False)
|
||||
async def root():
|
||||
return FileResponse(str(_static_dir / "index.html"))
|
||||
|
||||
@app.get("/{full_path:path}", include_in_schema=False)
|
||||
async def spa_fallback(full_path: str):
|
||||
"""Serve index.html for all non-API routes to support SPA client-side routing."""
|
||||
candidate = _static_dir / full_path
|
||||
if candidate.is_file():
|
||||
return FileResponse(str(candidate))
|
||||
return FileResponse(str(_static_dir / "index.html"))
|
||||
Executable
+73
@@ -0,0 +1,73 @@
|
||||
# Web framework
|
||||
fastapi==0.115.6
|
||||
uvicorn[standard]==0.34.0
|
||||
python-multipart==0.0.20
|
||||
aiofiles==24.1.0
|
||||
|
||||
# Database
|
||||
sqlalchemy==2.0.36
|
||||
|
||||
# Audio utilities
|
||||
pydub==0.25.1
|
||||
soundfile==0.12.1
|
||||
librosa==0.10.2
|
||||
mutagen==1.47.0
|
||||
|
||||
# ABC notation
|
||||
music21==9.3.0
|
||||
|
||||
# ML — PyTorch (CUDA 12.1 build; installed via --index-url in Dockerfile)
|
||||
# torch and torchaudio are installed separately in the Dockerfile with the CUDA wheel index.
|
||||
numpy==1.26.4
|
||||
|
||||
# Demucs stem separation
|
||||
demucs==4.0.1
|
||||
|
||||
# BS-RoFormer model architecture dependencies (includes MelBandRoformer)
|
||||
bs-roformer>=1.0.0
|
||||
omegaconf==2.3.0
|
||||
pyyaml>=6.0
|
||||
einops==0.8.0
|
||||
rotary-embedding-torch==0.6.5
|
||||
beartype==0.18.5
|
||||
tqdm>=4.66
|
||||
|
||||
# MelBand RoFormer dependencies
|
||||
hyper-connections>=0.0.10
|
||||
|
||||
# MDX-Net dependencies (ONNX models from UVR)
|
||||
# onnxruntime-gpu provides CUDAExecutionProvider so both MDX-Net and basic-pitch
|
||||
# automatically run on the GPU when available (conflicts with onnxruntime CPU package).
|
||||
onnxruntime-gpu>=1.16.0
|
||||
onnx>=1.14.0
|
||||
|
||||
# HuggingFace CLI (for BS-RoFormer model download)
|
||||
huggingface-hub[cli]>=0.24
|
||||
|
||||
# Subsonic server integration
|
||||
requests>=2.31.0
|
||||
cryptography>=42.0
|
||||
|
||||
# MIDI transcription via basic-pitch
|
||||
basic-pitch>=0.3.0
|
||||
|
||||
# Syllable hyphenation (pure-Python, for per-syllable ELRC timing)
|
||||
pyphen>=0.14.0
|
||||
|
||||
# Lyrics-sync dependencies (for ELRC alignment)
|
||||
# Pin to <4.46 — newer releases change internal lazy-module wiring in ways that
|
||||
# break Wav2Vec2ForCTC import when audio-separator also pins transformers deps.
|
||||
transformers>=4.29.0,<4.46.0
|
||||
accelerate>=0.19.0
|
||||
|
||||
# audio-separator (required by SCNet, MedleyVox, Apollo, Aspiration, Bandit separators)
|
||||
audio-separator[gpu]>=0.28.0
|
||||
|
||||
# Lyric transcription via OpenAI Whisper (local, offline)
|
||||
# Provides word-level timestamps for lyrics transcription.
|
||||
# Models are downloaded automatically on first use.
|
||||
openai-whisper>=20231117
|
||||
|
||||
# Optional: WhisperX for faster transcription + phoneme-level word alignment
|
||||
# Uncomment to enable the WhisperX engine option in AI Transcribe:
|
||||
whisperx>=3.1.0
|
||||
Executable
Whitespace-only changes.
Executable
+64
@@ -0,0 +1,64 @@
|
||||
"""ABC notation conversion endpoint."""
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
from fastapi import APIRouter, File, Form, UploadFile, HTTPException
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import UPLOAD_DIR
|
||||
from ..services.abc_service import run_abc_conversion
|
||||
|
||||
router = APIRouter(prefix="/api/abc-convert", tags=["abc-convert"])
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def abc_convert(
|
||||
files: List[UploadFile] = File(...),
|
||||
song_title: str = Form("Untitled"),
|
||||
):
|
||||
if not files:
|
||||
raise HTTPException(status_code=400, detail="At least one MIDI file required")
|
||||
|
||||
job_id = jm.create_job("abc-convert")
|
||||
upload_dir = UPLOAD_DIR / job_id
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
midi_paths = []
|
||||
for upload in files:
|
||||
dest = upload_dir / upload.filename
|
||||
with open(dest, "wb") as f:
|
||||
shutil.copyfileobj(upload.file, f)
|
||||
midi_paths.append(str(dest))
|
||||
|
||||
jm.submit(run_abc_conversion, job_id, midi_paths, song_title)
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/from-job")
|
||||
async def abc_convert_from_job(
|
||||
source_job_id: str = Form(...),
|
||||
song_title: str = Form(""),
|
||||
):
|
||||
"""
|
||||
Convert MIDI files from a completed transcription (or pipeline) job to ABC notation.
|
||||
Reuses the already-generated MIDI files on disk — no re-upload needed.
|
||||
"""
|
||||
source_job = jm.get_job(source_job_id)
|
||||
if source_job is None:
|
||||
raise HTTPException(status_code=404, detail=f"Source job not found: {source_job_id}")
|
||||
if source_job["status"] != "done":
|
||||
raise HTTPException(status_code=400, detail="Source job has not completed successfully")
|
||||
|
||||
midi_paths = [
|
||||
f["path"] for f in source_job.get("files", [])
|
||||
if Path(f.get("path", "")).suffix.lower() == ".mid"
|
||||
and Path(f.get("path", "")).exists()
|
||||
]
|
||||
if not midi_paths:
|
||||
raise HTTPException(status_code=404, detail="No accessible MIDI files found in source job")
|
||||
|
||||
title = song_title.strip() or source_job_id[:8]
|
||||
job_id = jm.create_job("abc-convert")
|
||||
jm.submit(run_abc_conversion, job_id, midi_paths, title)
|
||||
return {"job_id": job_id}
|
||||
Executable
+55
@@ -0,0 +1,55 @@
|
||||
"""Shared job status and download endpoints used by all tools."""
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from .. import job_manager as jm
|
||||
|
||||
router = APIRouter(prefix="/api/jobs", tags=["jobs"])
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_jobs():
|
||||
return jm.list_jobs()
|
||||
|
||||
|
||||
@router.get("/{job_id}")
|
||||
def get_job(job_id: str):
|
||||
job = jm.get_job(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
return job
|
||||
|
||||
|
||||
@router.delete("/{job_id}")
|
||||
def cancel_job(job_id: str):
|
||||
job = jm.get_job(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
jm.cancel_job(job_id)
|
||||
return {"detail": "Cancellation requested"}
|
||||
|
||||
|
||||
@router.get("/{job_id}/download/{filename}")
|
||||
def download_file(job_id: str, filename: str):
|
||||
job = jm.get_job(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
if job["status"] != "done":
|
||||
raise HTTPException(status_code=400, detail="Job not complete")
|
||||
|
||||
# Find the file in the job's file list
|
||||
match = next((f for f in job["files"] if f["name"] == filename), None)
|
||||
if match is None:
|
||||
raise HTTPException(status_code=404, detail="File not found")
|
||||
|
||||
file_path = Path(match["path"])
|
||||
if not file_path.exists():
|
||||
raise HTTPException(status_code=404, detail="File missing from disk")
|
||||
|
||||
return FileResponse(
|
||||
path=str(file_path),
|
||||
filename=filename,
|
||||
media_type="application/octet-stream",
|
||||
)
|
||||
Executable
+238
@@ -0,0 +1,238 @@
|
||||
"""LRC alignment endpoint for creating enhanced ELRC files with word-level timestamps."""
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, File, Form, UploadFile, HTTPException
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import UPLOAD_DIR, LRC_SYNC_DIR, PROJECTS_DIR
|
||||
from ..services.lrc_service import run_lrc_alignment
|
||||
from ..services.metadata_util import extract_lyrics_from_file
|
||||
|
||||
router = APIRouter(prefix="/api/lrc", tags=["lrc"])
|
||||
|
||||
|
||||
@router.get("/sources")
|
||||
def get_lyrics_sources():
|
||||
"""Get available lyrics source options."""
|
||||
return [
|
||||
{"id": "auto", "label": "Auto-detect (tags → LRC file)"},
|
||||
{"id": "tag", "label": "Audio file metadata tags only"},
|
||||
{"id": "file", "label": "Upload LRC/TXT file"},
|
||||
{"id": "text", "label": "Type lyrics directly"},
|
||||
]
|
||||
|
||||
|
||||
@router.post("/extract")
|
||||
async def extract_lyrics(
|
||||
file: UploadFile = File(...),
|
||||
):
|
||||
"""
|
||||
Extract lyrics from an uploaded audio file.
|
||||
Returns lyrics source info and plain text.
|
||||
"""
|
||||
temp_path = UPLOAD_DIR / "temp" / file.filename
|
||||
temp_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
with open(temp_path, "wb") as f:
|
||||
shutil.copyfileobj(file.file, f)
|
||||
|
||||
result = extract_lyrics_from_file(str(temp_path))
|
||||
|
||||
return {
|
||||
"source": result["source"],
|
||||
"synced": result["synced"],
|
||||
"lyrics": result["plain"],
|
||||
"raw": result["raw"],
|
||||
}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Failed to extract lyrics: {e}")
|
||||
finally:
|
||||
# Cleanup
|
||||
try:
|
||||
if temp_path.exists():
|
||||
temp_path.unlink()
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
@router.post("/align")
|
||||
async def align_lyrics(
|
||||
audio: UploadFile = File(..., description="Audio file with vocals"),
|
||||
lyrics_source: str = Form("auto", description="Source of lyrics: auto, tag, file, text"),
|
||||
lyrics_text: Optional[str] = Form(None, description="Lyrics text when source='text'"),
|
||||
output_filename: Optional[str] = Form(None, description="Custom output filename"),
|
||||
style_options: Optional[str] = Form(None, description="JSON string of style options for lyrics display"),
|
||||
):
|
||||
"""
|
||||
Align lyrics with audio to create ELRC file with word-level timestamps.
|
||||
|
||||
- lyrics_source='auto': Try metadata tags, then external LRC file
|
||||
- lyrics_source='tag': Only use metadata tags from audio file
|
||||
- lyrics_source='text': Use provided lyrics_text
|
||||
"""
|
||||
if lyrics_source not in ("auto", "tag", "file", "text"):
|
||||
raise HTTPException(status_code=400, detail=f"Invalid lyrics_source: {lyrics_source}")
|
||||
|
||||
if lyrics_source == "text" and not lyrics_text:
|
||||
raise HTTPException(status_code=400, detail="lyrics_text required when source='text'")
|
||||
|
||||
# Create job
|
||||
job_id = jm.create_job("lrc_align")
|
||||
upload_dir = UPLOAD_DIR / job_id
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Save audio file
|
||||
audio_path = upload_dir / audio.filename
|
||||
with open(audio_path, "wb") as f:
|
||||
shutil.copyfileobj(audio.file, f)
|
||||
|
||||
# Parse style options
|
||||
parsed_style_options = None
|
||||
if style_options:
|
||||
try:
|
||||
parsed_style_options = json.loads(style_options)
|
||||
except json.JSONDecodeError:
|
||||
raise HTTPException(status_code=400, detail="Invalid style_options JSON")
|
||||
|
||||
# Submit job
|
||||
jm.submit_gpu(
|
||||
run_lrc_alignment,
|
||||
job_id,
|
||||
str(audio_path),
|
||||
lyrics_source=lyrics_source,
|
||||
lyrics_content=lyrics_text,
|
||||
output_filename=output_filename,
|
||||
style_options=parsed_style_options,
|
||||
)
|
||||
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/align-with-file")
|
||||
async def align_lyrics_with_file(
|
||||
audio: UploadFile = File(..., description="Audio file with vocals"),
|
||||
lyrics_file: UploadFile = File(..., description="LRC or TXT file with lyrics"),
|
||||
output_filename: Optional[str] = Form(None, description="Custom output filename"),
|
||||
style_options: Optional[str] = Form(None, description="JSON string of style options for lyrics display"),
|
||||
):
|
||||
"""
|
||||
Align lyrics from uploaded file with audio to create ELRC file.
|
||||
"""
|
||||
job_id = jm.create_job("lrc_align")
|
||||
upload_dir = UPLOAD_DIR / job_id
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Save audio file
|
||||
audio_path = upload_dir / audio.filename
|
||||
with open(audio_path, "wb") as f:
|
||||
shutil.copyfileobj(audio.file, f)
|
||||
|
||||
# Save lyrics file
|
||||
lyrics_path = upload_dir / lyrics_file.filename
|
||||
with open(lyrics_path, "wb") as f:
|
||||
shutil.copyfileobj(lyrics_file.file, f)
|
||||
|
||||
# Parse style options
|
||||
parsed_style_options = None
|
||||
if style_options:
|
||||
try:
|
||||
parsed_style_options = json.loads(style_options)
|
||||
except json.JSONDecodeError:
|
||||
raise HTTPException(status_code=400, detail="Invalid style_options JSON")
|
||||
|
||||
# Submit job
|
||||
jm.submit_gpu(
|
||||
run_lrc_alignment,
|
||||
job_id,
|
||||
str(audio_path),
|
||||
lyrics_source="file",
|
||||
lyrics_file_path=str(lyrics_path),
|
||||
output_filename=output_filename,
|
||||
style_options=parsed_style_options,
|
||||
)
|
||||
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/align-existing")
|
||||
async def align_lyrics_existing(
|
||||
project_name: str = Form(..., description="Project name (e.g., 'Artist - Title')"),
|
||||
lyrics_source: str = Form("auto", description="Source of lyrics: auto, tag, text"),
|
||||
lyrics_text: Optional[str] = Form(None, description="Lyrics text when source='text'"),
|
||||
output_filename: Optional[str] = Form(None, description="Custom output filename"),
|
||||
style_options: Optional[str] = Form(None, description="JSON string of style options for lyrics display"),
|
||||
):
|
||||
"""
|
||||
Align lyrics with an existing vocal file in a project.
|
||||
Looks for the vocal file in the project's Stems folder.
|
||||
"""
|
||||
project_dir = PROJECTS_DIR / project_name
|
||||
stems_dir = project_dir / "Stems"
|
||||
|
||||
if not stems_dir.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Stems folder not found for project: {project_name}")
|
||||
|
||||
# Find vocals file
|
||||
vocal_file = None
|
||||
for name in ["vocals", "vocals.mp3", "vocals.wav", "vocals.flac"]:
|
||||
for ext in ["", ".mp3", ".wav", ".flac"]:
|
||||
candidate = stems_dir / f"{name}{ext}"
|
||||
if candidate.exists():
|
||||
vocal_file = candidate
|
||||
break
|
||||
if vocal_file:
|
||||
break
|
||||
|
||||
if not vocal_file:
|
||||
# Try any file with "vocal" in the name
|
||||
for f in stems_dir.iterdir():
|
||||
if "vocal" in f.name.lower() and f.suffix in [".mp3", ".wav", ".flac"]:
|
||||
vocal_file = f
|
||||
break
|
||||
|
||||
if not vocal_file:
|
||||
raise HTTPException(status_code=404, detail="No vocals file found in Stems folder")
|
||||
|
||||
job_id = jm.create_job("lrc_align")
|
||||
|
||||
# Save ELRC/LRC directly into the project root folder so it sits alongside
|
||||
# the source audio (which is also named after the project).
|
||||
project_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Determine output filename — must match the source audio base name
|
||||
if not output_filename:
|
||||
# Check whether a source audio file exists and use its stem
|
||||
source_stem = project_name
|
||||
for ext in ['.mp3', '.flac', '.wav', '.m4a', '.ogg']:
|
||||
candidate = project_dir / f"{project_name}{ext}"
|
||||
if candidate.exists():
|
||||
source_stem = candidate.stem
|
||||
break
|
||||
output_filename = f"{source_stem}.elrc"
|
||||
|
||||
# Parse style options
|
||||
parsed_style_options = None
|
||||
if style_options:
|
||||
try:
|
||||
parsed_style_options = json.loads(style_options)
|
||||
except json.JSONDecodeError:
|
||||
raise HTTPException(status_code=400, detail="Invalid style_options JSON")
|
||||
|
||||
# Submit job — vocal file is already separated, so skip Demucs extraction
|
||||
jm.submit_gpu(
|
||||
run_lrc_alignment,
|
||||
job_id,
|
||||
str(vocal_file),
|
||||
lyrics_source=lyrics_source,
|
||||
lyrics_content=lyrics_text,
|
||||
output_filename=output_filename,
|
||||
style_options=parsed_style_options,
|
||||
is_vocals=True,
|
||||
output_dir=str(project_dir),
|
||||
)
|
||||
|
||||
return {"job_id": job_id}
|
||||
Executable
+542
@@ -0,0 +1,542 @@
|
||||
"""
|
||||
Lyric transcription endpoint - transcribe vocals to text with timestamps.
|
||||
Based on Scriberr's transcription approach using OpenAI Whisper.
|
||||
|
||||
This provides:
|
||||
- Word-level and line-level timestamps
|
||||
- Multiple output formats (LRC, ELRC, JSON, SRT, VTT, TXT)
|
||||
- Speaker diarization support (via WhisperX)
|
||||
- Multiple Whisper model sizes
|
||||
"""
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Optional, List
|
||||
|
||||
from fastapi import APIRouter, File, Form, UploadFile, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import UPLOAD_DIR, PROJECTS_DIR
|
||||
from ..services.lyric_transcriber_service import (
|
||||
run_lyric_transcription,
|
||||
run_lyric_transcription_with_diarization,
|
||||
WHISPER_MODELS,
|
||||
DEFAULT_WHISPER_MODEL,
|
||||
is_whisper_available,
|
||||
is_whisperx_available,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/lyric-transcribe", tags=["lyric-transcribe"])
|
||||
|
||||
|
||||
class LyricTranscribeRequest(BaseModel):
|
||||
"""Request body for project-based lyric transcription."""
|
||||
stem_name: str
|
||||
model: str = DEFAULT_WHISPER_MODEL
|
||||
language: Optional[str] = None
|
||||
task: str = "transcribe"
|
||||
output_formats: List[str] = ["elrc", "json", "txt"]
|
||||
initial_prompt: Optional[str] = None
|
||||
temperature: float = 0.0
|
||||
condition_on_previous_text: bool = True
|
||||
word_timestamps: bool = True
|
||||
beam_size: Optional[int] = None
|
||||
|
||||
|
||||
class LyricTranscribeWithDiarizationRequest(BaseModel):
|
||||
"""Request body for transcription with speaker diarization."""
|
||||
stem_name: str
|
||||
model: str = DEFAULT_WHISPER_MODEL
|
||||
language: Optional[str] = None
|
||||
num_speakers: Optional[int] = None
|
||||
min_speakers: Optional[int] = None
|
||||
max_speakers: Optional[int] = None
|
||||
output_formats: List[str] = ["json", "txt"]
|
||||
|
||||
|
||||
VALID_OUTPUT_FORMATS = {"lrc", "elrc", "json", "txt", "srt", "vtt"}
|
||||
VALID_TASKS = {"transcribe", "translate"}
|
||||
_AUDIO_EXTS = {".mp3", ".wav", ".flac", ".m4a", ".ogg", ".wma"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _find_vocal_file(project_dir: Path) -> Optional[Path]:
|
||||
"""Find the vocal file in a project's Stems folder."""
|
||||
stems_dir = project_dir / "Stems"
|
||||
if not stems_dir.exists():
|
||||
return None
|
||||
|
||||
# Look for common vocal file names
|
||||
vocal_patterns = ["vocals", "Vocals", "lead", "Lead", "vocal", "Vocal"]
|
||||
for pattern in vocal_patterns:
|
||||
for ext in _AUDIO_EXTS:
|
||||
candidate = stems_dir / f"{pattern}{ext}"
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
|
||||
# Fallback: any file with "vocal" in the name
|
||||
for f in stems_dir.iterdir():
|
||||
if f.is_file() and f.suffix in _AUDIO_EXTS:
|
||||
if "vocal" in f.name.lower():
|
||||
return f
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _find_stem_file(project_dir: Path, stem_name: str) -> Optional[Path]:
|
||||
"""Find a specific stem file in a project's Stems folder."""
|
||||
stems_dir = project_dir / "Stems"
|
||||
if not stems_dir.exists():
|
||||
return None
|
||||
|
||||
# Exact match
|
||||
for ext in _AUDIO_EXTS:
|
||||
candidate = stems_dir / f"{stem_name}{ext}"
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
|
||||
# Case-insensitive match
|
||||
for f in stems_dir.iterdir():
|
||||
if f.is_file() and f.suffix in _AUDIO_EXTS:
|
||||
if f.stem.lower() == stem_name.lower():
|
||||
return f
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/models")
|
||||
def get_available_models():
|
||||
"""Get list of available Whisper models."""
|
||||
return {
|
||||
"models": list(WHISPER_MODELS.values()),
|
||||
"default": DEFAULT_WHISPER_MODEL,
|
||||
"whisper_available": is_whisper_available(),
|
||||
"whisperx_available": is_whisperx_available(),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/languages")
|
||||
def get_supported_languages():
|
||||
"""Get list of languages supported by Whisper."""
|
||||
# Common languages for music/lyrics
|
||||
languages = [
|
||||
{"code": None, "name": "Auto-detect", "label": "Auto-detect language"},
|
||||
{"code": "en", "name": "English", "label": "English"},
|
||||
{"code": "zh", "name": "Chinese", "label": "Chinese (Mandarin)"},
|
||||
{"code": "de", "name": "German", "label": "German"},
|
||||
{"code": "es", "name": "Spanish", "label": "Spanish"},
|
||||
{"code": "ru", "name": "Russian", "label": "Russian"},
|
||||
{"code": "ko", "name": "Korean", "label": "Korean"},
|
||||
{"code": "fr", "name": "French", "label": "French"},
|
||||
{"code": "ja", "name": "Japanese", "label": "Japanese"},
|
||||
{"code": "pt", "name": "Portuguese", "label": "Portuguese"},
|
||||
{"code": "tr", "name": "Turkish", "label": "Turkish"},
|
||||
{"code": "pl", "name": "Polish", "label": "Polish"},
|
||||
{"code": "ca", "name": "Catalan", "label": "Catalan"},
|
||||
{"code": "nl", "name": "Dutch", "label": "Dutch"},
|
||||
{"code": "ar", "name": "Arabic", "label": "Arabic"},
|
||||
{"code": "sv", "name": "Swedish", "label": "Swedish"},
|
||||
{"code": "it", "name": "Italian", "label": "Italian"},
|
||||
{"code": "id", "name": "Indonesian", "label": "Indonesian"},
|
||||
{"code": "hi", "name": "Hindi", "label": "Hindi"},
|
||||
{"code": "fi", "name": "Finnish", "label": "Finnish"},
|
||||
{"code": "vi", "name": "Vietnamese", "label": "Vietnamese"},
|
||||
{"code": "he", "name": "Hebrew", "label": "Hebrew"},
|
||||
{"code": "uk", "name": "Ukrainian", "label": "Ukrainian"},
|
||||
{"code": "el", "name": "Greek", "label": "Greek"},
|
||||
{"code": "ms", "name": "Malay", "label": "Malay"},
|
||||
{"code": "cs", "name": "Czech", "label": "Czech"},
|
||||
{"code": "ro", "name": "Romanian", "label": "Romanian"},
|
||||
{"code": "da", "name": "Danish", "label": "Danish"},
|
||||
{"code": "hu", "name": "Hungarian", "label": "Hungarian"},
|
||||
{"code": "ta", "name": "Tamil", "label": "Tamil"},
|
||||
{"code": "no", "name": "Norwegian", "label": "Norwegian"},
|
||||
{"code": "th", "name": "Thai", "label": "Thai"},
|
||||
{"code": "ur", "name": "Urdu", "label": "Urdu"},
|
||||
{"code": "hr", "name": "Croatian", "label": "Croatian"},
|
||||
{"code": "bg", "name": "Bulgarian", "label": "Bulgarian"},
|
||||
{"code": "lt", "name": "Lithuanian", "label": "Lithuanian"},
|
||||
{"code": "la", "name": "Latin", "label": "Latin"},
|
||||
{"code": "mi", "name": "Maori", "label": "Maori"},
|
||||
{"code": "ml", "name": "Malayalam", "label": "Malayalam"},
|
||||
{"code": "cy", "name": "Welsh", "label": "Welsh"},
|
||||
{"code": "sk", "name": "Slovak", "label": "Slovak"},
|
||||
{"code": "te", "name": "Telugu", "label": "Telugu"},
|
||||
{"code": "fa", "name": "Persian", "label": "Persian"},
|
||||
{"code": "lv", "name": "Latvian", "label": "Latvian"},
|
||||
{"code": "bn", "name": "Bengali", "label": "Bengali"},
|
||||
{"code": "sr", "name": "Serbian", "label": "Serbian"},
|
||||
{"code": "az", "name": "Azerbaijani", "label": "Azerbaijani"},
|
||||
{"code": "sl", "name": "Slovenian", "label": "Slovenian"},
|
||||
{"code": "kn", "name": "Kannada", "label": "Kannada"},
|
||||
{"code": "et", "name": "Estonian", "label": "Estonian"},
|
||||
{"code": "mk", "name": "Macedonian", "label": "Macedonian"},
|
||||
{"code": "br", "name": "Breton", "label": "Breton"},
|
||||
{"code": "eu", "name": "Basque", "label": "Basque"},
|
||||
{"code": "is", "name": "Icelandic", "label": "Icelandic"},
|
||||
{"code": "hy", "name": "Armenian", "label": "Armenian"},
|
||||
{"code": "ne", "name": "Nepali", "label": "Nepali"},
|
||||
{"code": "mn", "name": "Mongolian", "label": "Mongolian"},
|
||||
{"code": "bs", "name": "Bosnian", "label": "Bosnian"},
|
||||
{"code": "kk", "name": "Kazakh", "label": "Kazakh"},
|
||||
{"code": "sq", "name": "Albanian", "label": "Albanian"},
|
||||
{"code": "sw", "name": "Swahili", "label": "Swahili"},
|
||||
{"code": "gl", "name": "Galician", "label": "Galician"},
|
||||
{"code": "mr", "name": "Marathi", "label": "Marathi"},
|
||||
{"code": "pa", "name": "Punjabi", "label": "Punjabi"},
|
||||
{"code": "si", "name": "Sinhala", "label": "Sinhala"},
|
||||
{"code": "km", "name": "Khmer", "label": "Khmer"},
|
||||
{"code": "sn", "name": "Shona", "label": "Shona"},
|
||||
{"code": "yo", "name": "Yoruba", "label": "Yoruba"},
|
||||
{"code": "so", "name": "Somali", "label": "Somali"},
|
||||
{"code": "af", "name": "Afrikaans", "label": "Afrikaans"},
|
||||
{"code": "oc", "name": "Occitan", "label": "Occitan"},
|
||||
{"code": "ka", "name": "Georgian", "label": "Georgian"},
|
||||
{"code": "be", "name": "Belarusian", "label": "Belarusian"},
|
||||
{"code": "tg", "name": "Tajik", "label": "Tajik"},
|
||||
{"code": "sd", "name": "Sindhi", "label": "Sindhi"},
|
||||
{"code": "gu", "name": "Gujarati", "label": "Gujarati"},
|
||||
{"code": "am", "name": "Amharic", "label": "Amharic"},
|
||||
{"code": "yi", "name": "Yiddish", "label": "Yiddish"},
|
||||
{"code": "lo", "name": "Lao", "label": "Lao"},
|
||||
{"code": "uz", "name": "Uzbek", "label": "Uzbek"},
|
||||
{"code": "fo", "name": "Faroese", "label": "Faroese"},
|
||||
{"code": "ht", "name": "Haitian Creole", "label": "Haitian Creole"},
|
||||
{"code": "ps", "name": "Pashto", "label": "Pashto"},
|
||||
{"code": "tk", "name": "Turkmen", "label": "Turkmen"},
|
||||
{"code": "nn", "name": "Nynorsk", "label": "Nynorsk"},
|
||||
{"code": "mt", "name": "Maltese", "label": "Maltese"},
|
||||
{"code": "sa", "name": "Sanskrit", "label": "Sanskrit"},
|
||||
{"code": "lb", "name": "Luxembourgish", "label": "Luxembourgish"},
|
||||
{"code": "my", "name": "Myanmar", "label": "Myanmar"},
|
||||
{"code": "bo", "name": "Tibetan", "label": "Tibetan"},
|
||||
{"code": "tl", "name": "Tagalog", "label": "Tagalog"},
|
||||
{"code": "mg", "name": "Malagasy", "label": "Malagasy"},
|
||||
{"code": "as", "name": "Assamese", "label": "Assamese"},
|
||||
{"code": "tt", "name": "Tatar", "label": "Tatar"},
|
||||
{"code": "haw", "name": "Hawaiian", "label": "Hawaiian"},
|
||||
{"code": "ln", "name": "Lingala", "label": "Lingala"},
|
||||
{"code": "ha", "name": "Hausa", "label": "Hausa"},
|
||||
{"code": "ba", "name": "Bashkir", "label": "Bashkir"},
|
||||
{"code": "jw", "name": "Javanese", "label": "Javanese"},
|
||||
{"code": "su", "name": "Sundanese", "label": "Sundanese"},
|
||||
]
|
||||
return {"languages": languages}
|
||||
|
||||
|
||||
@router.get("/formats")
|
||||
def get_output_formats():
|
||||
"""Get available output formats."""
|
||||
return {
|
||||
"formats": [
|
||||
{"id": "lrc", "label": "LRC (line-level timestamps)", "extension": ".lrc"},
|
||||
{"id": "elrc", "label": "ELRC (word-level timestamps)", "extension": ".elrc"},
|
||||
{"id": "json", "label": "JSON (full data)", "extension": ".json"},
|
||||
{"id": "txt", "label": "Plain Text", "extension": ".txt"},
|
||||
{"id": "srt", "label": "SRT Subtitles", "extension": ".srt"},
|
||||
{"id": "vtt", "label": "WebVTT Subtitles", "extension": ".vtt"},
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def transcribe_lyrics(
|
||||
file: UploadFile = File(...),
|
||||
model: str = Form(DEFAULT_WHISPER_MODEL),
|
||||
language: Optional[str] = Form(None),
|
||||
task: str = Form("transcribe"),
|
||||
output_formats: Optional[str] = Form("elrc,json,txt"),
|
||||
initial_prompt: Optional[str] = Form(None),
|
||||
temperature: float = Form(0.0),
|
||||
):
|
||||
"""
|
||||
Transcribe lyrics from an uploaded audio file.
|
||||
|
||||
- **file**: Audio file (vocals stem recommended for best results)
|
||||
- **model**: Whisper model to use (tiny, base, small, medium, large, large-v2, large-v3)
|
||||
- **language**: ISO-639-1 language code (auto-detect if not provided)
|
||||
- **task**: 'transcribe' for original language, 'translate' for English
|
||||
- **output_formats**: Comma-separated list: lrc, elrc, json, txt, srt, vtt
|
||||
- **initial_prompt**: Optional text to guide the model's style
|
||||
- **temperature**: Sampling temperature (0.0 = greedy, higher = more random)
|
||||
"""
|
||||
if not is_whisper_available():
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Whisper is not installed. Install with: pip install openai-whisper"
|
||||
)
|
||||
|
||||
if model not in WHISPER_MODELS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {model}")
|
||||
|
||||
if task not in VALID_TASKS:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid task: {task}")
|
||||
|
||||
# Parse output formats
|
||||
formats = [f.strip() for f in (output_formats or "json").split(",")]
|
||||
invalid_formats = set(formats) - VALID_OUTPUT_FORMATS
|
||||
if invalid_formats:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid output formats: {invalid_formats}"
|
||||
)
|
||||
|
||||
# Create job and save file
|
||||
job_id = jm.create_job("lyric-transcribe")
|
||||
upload_dir = UPLOAD_DIR / job_id
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
audio_path = upload_dir / file.filename
|
||||
with open(audio_path, "wb") as f:
|
||||
shutil.copyfileobj(file.file, f)
|
||||
|
||||
# Submit job
|
||||
jm.submit_gpu(
|
||||
run_lyric_transcription,
|
||||
job_id,
|
||||
str(audio_path),
|
||||
model=model,
|
||||
language=language,
|
||||
task=task,
|
||||
output_formats=formats,
|
||||
initial_prompt=initial_prompt,
|
||||
temperature=temperature,
|
||||
)
|
||||
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/from-project")
|
||||
async def transcribe_lyrics_from_project(
|
||||
project_name: str = Form(...),
|
||||
stem_name: Optional[str] = Form(None),
|
||||
model: str = Form(DEFAULT_WHISPER_MODEL),
|
||||
language: Optional[str] = Form(None),
|
||||
task: str = Form("transcribe"),
|
||||
output_formats: Optional[str] = Form("elrc,json,txt"),
|
||||
initial_prompt: Optional[str] = Form(None),
|
||||
temperature: float = Form(0.0),
|
||||
):
|
||||
"""
|
||||
Transcribe lyrics from a project's vocal stem.
|
||||
|
||||
- **project_name**: Name of the project
|
||||
- **stem_name**: Specific stem to use (default: auto-detect vocals)
|
||||
- Other parameters same as /api/lyric-transcribe
|
||||
"""
|
||||
if not is_whisper_available():
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Whisper is not installed. Install with: pip install openai-whisper"
|
||||
)
|
||||
|
||||
if model not in WHISPER_MODELS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {model}")
|
||||
|
||||
if task not in VALID_TASKS:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid task: {task}")
|
||||
|
||||
project_dir = PROJECTS_DIR / project_name
|
||||
if not project_dir.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Project not found: {project_name}")
|
||||
|
||||
# Find audio file
|
||||
if stem_name:
|
||||
audio_file = _find_stem_file(project_dir, stem_name)
|
||||
if not audio_file:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Stem not found: {stem_name}"
|
||||
)
|
||||
else:
|
||||
audio_file = _find_vocal_file(project_dir)
|
||||
if not audio_file:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="No vocal file found in project. Please specify stem_name."
|
||||
)
|
||||
|
||||
# Parse output formats
|
||||
formats = [f.strip() for f in (output_formats or "json").split(",")]
|
||||
invalid_formats = set(formats) - VALID_OUTPUT_FORMATS
|
||||
if invalid_formats:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid output formats: {invalid_formats}"
|
||||
)
|
||||
|
||||
# Create job
|
||||
job_id = jm.create_job("lyric-transcribe")
|
||||
|
||||
# Output to project's LRC folder
|
||||
output_dir = project_dir / "LRC"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Submit job
|
||||
jm.submit_gpu(
|
||||
run_lyric_transcription,
|
||||
job_id,
|
||||
str(audio_file),
|
||||
model=model,
|
||||
language=language,
|
||||
task=task,
|
||||
output_formats=formats,
|
||||
initial_prompt=initial_prompt,
|
||||
temperature=temperature,
|
||||
output_dir=str(output_dir),
|
||||
)
|
||||
|
||||
return {"job_id": job_id, "source": audio_file.name}
|
||||
|
||||
|
||||
@router.post("/with-diarization")
|
||||
async def transcribe_lyrics_with_diarization(
|
||||
file: UploadFile = File(...),
|
||||
model: str = Form(DEFAULT_WHISPER_MODEL),
|
||||
language: Optional[str] = Form(None),
|
||||
num_speakers: Optional[int] = Form(None),
|
||||
min_speakers: Optional[int] = Form(None),
|
||||
max_speakers: Optional[int] = Form(None),
|
||||
output_formats: Optional[str] = Form("json,txt"),
|
||||
):
|
||||
"""
|
||||
Transcribe lyrics with speaker diarization (who spoke when).
|
||||
|
||||
Requires WhisperX to be installed: pip install whisperx
|
||||
|
||||
- **file**: Audio file
|
||||
- **num_speakers**: Exact number of speakers (optional)
|
||||
- **min_speakers**: Minimum number of speakers (optional)
|
||||
- **max_speakers**: Maximum number of speakers (optional)
|
||||
"""
|
||||
if not is_whisperx_available():
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="WhisperX is not installed. Install with: pip install whisperx"
|
||||
)
|
||||
|
||||
if model not in WHISPER_MODELS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {model}")
|
||||
|
||||
# Parse output formats
|
||||
formats = [f.strip() for f in (output_formats or "json").split(",")]
|
||||
invalid_formats = set(formats) - VALID_OUTPUT_FORMATS
|
||||
if invalid_formats:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid output formats: {invalid_formats}"
|
||||
)
|
||||
|
||||
# Create job and save file
|
||||
job_id = jm.create_job("lyric-transcribe-diarize")
|
||||
upload_dir = UPLOAD_DIR / job_id
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
audio_path = upload_dir / file.filename
|
||||
with open(audio_path, "wb") as f:
|
||||
shutil.copyfileobj(file.file, f)
|
||||
|
||||
# Submit job
|
||||
jm.submit_gpu(
|
||||
run_lyric_transcription_with_diarization,
|
||||
job_id,
|
||||
str(audio_path),
|
||||
model=model,
|
||||
language=language,
|
||||
num_speakers=num_speakers,
|
||||
min_speakers=min_speakers,
|
||||
max_speakers=max_speakers,
|
||||
output_formats=formats,
|
||||
)
|
||||
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/project/{project_id}")
|
||||
def transcribe_project_lyrics(project_id: str, body: LyricTranscribeRequest):
|
||||
"""
|
||||
Transcribe lyrics from a specific stem in a project.
|
||||
Similar to transcribe_lyrics_from_project but uses project_id and JSON body.
|
||||
"""
|
||||
if not is_whisper_available():
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Whisper is not installed. Install with: pip install openai-whisper"
|
||||
)
|
||||
|
||||
if body.model not in WHISPER_MODELS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {body.model}")
|
||||
|
||||
if body.task not in VALID_TASKS:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid task: {body.task}")
|
||||
|
||||
# Get project
|
||||
project = jm.get_project(project_id)
|
||||
if not project:
|
||||
raise HTTPException(status_code=404, detail=f"Project not found: {project_id}")
|
||||
|
||||
# Get separation job to find stems
|
||||
sep_job = jm.get_job(project["sep_job_id"])
|
||||
if not sep_job:
|
||||
raise HTTPException(status_code=404, detail="Separation job not found")
|
||||
|
||||
# Find the stem file
|
||||
stem_files = {
|
||||
f["stem"]: f["path"]
|
||||
for f in sep_job.get("files", [])
|
||||
if f.get("type") == "stem" and f.get("stem")
|
||||
}
|
||||
|
||||
if body.stem_name not in stem_files:
|
||||
raise HTTPException(status_code=404, detail=f"Stem not found: {body.stem_name}")
|
||||
|
||||
audio_path = stem_files[body.stem_name]
|
||||
if not Path(audio_path).exists():
|
||||
raise HTTPException(status_code=404, detail=f"Stem file not found: {audio_path}")
|
||||
|
||||
# Validate output formats
|
||||
invalid_formats = set(body.output_formats) - VALID_OUTPUT_FORMATS
|
||||
if invalid_formats:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid output formats: {invalid_formats}"
|
||||
)
|
||||
|
||||
# Create job
|
||||
job_id = jm.create_job("lyric-transcribe")
|
||||
|
||||
# Output to project directory
|
||||
project_dir = PROJECTS_DIR / project["name"]
|
||||
output_dir = project_dir / "LRC"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Store job reference
|
||||
from ..job_manager import set_project_lrc_job
|
||||
set_project_lrc_job(project_id, body.stem_name, job_id)
|
||||
|
||||
# Submit job
|
||||
jm.submit_gpu(
|
||||
run_lyric_transcription,
|
||||
job_id,
|
||||
audio_path,
|
||||
model=body.model,
|
||||
language=body.language,
|
||||
task=body.task,
|
||||
output_formats=body.output_formats,
|
||||
initial_prompt=body.initial_prompt,
|
||||
temperature=body.temperature,
|
||||
condition_on_previous_text=body.condition_on_previous_text,
|
||||
word_timestamps=body.word_timestamps,
|
||||
beam_size=body.beam_size,
|
||||
output_dir=str(output_dir),
|
||||
)
|
||||
|
||||
return {"job_id": job_id, "project_id": project_id, "stem": body.stem_name}
|
||||
Executable
+744
@@ -0,0 +1,744 @@
|
||||
"""
|
||||
Model management router — list, download, pause, cancel model downloads.
|
||||
Models are downloaded from HuggingFace on-demand or pre-downloaded.
|
||||
"""
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException, BackgroundTasks
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ..config import (
|
||||
ALL_MODELS,
|
||||
BS_ROFORMER_MODEL_DIR,
|
||||
MEL_BAND_MODEL_DIR,
|
||||
MDXNET_MODEL_DIR,
|
||||
SCNET_MODEL_DIR,
|
||||
MEDLEYVOX_MODEL_DIR,
|
||||
APOLLO_MODEL_DIR,
|
||||
ASPIRATION_MODEL_DIR,
|
||||
BANDIT_MODEL_DIR,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/models", tags=["models"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model metadata with download info
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class ModelDownloadStatus(str, Enum):
|
||||
NOT_DOWNLOADED = "not_downloaded"
|
||||
DOWNLOADING = "downloading"
|
||||
PAUSED = "paused"
|
||||
COMPLETED = "downloaded" # matches the frontend's status === 'downloaded' checks
|
||||
ERROR = "error"
|
||||
CANCELLED = "cancelled"
|
||||
|
||||
|
||||
@dataclass
|
||||
class DownloadProgress:
|
||||
model_id: str
|
||||
status: ModelDownloadStatus
|
||||
progress_percent: float = 0.0
|
||||
bytes_downloaded: int = 0
|
||||
total_bytes: int = 0
|
||||
speed_mbps: float = 0.0
|
||||
error_message: Optional[str] = None
|
||||
|
||||
# Control flags (threading.Event so they work correctly in background threads)
|
||||
_pause_event: threading.Event = field(default_factory=threading.Event)
|
||||
_cancel_event: threading.Event = field(default_factory=threading.Event)
|
||||
_thread: Optional[threading.Thread] = None
|
||||
|
||||
def __post_init__(self):
|
||||
self._pause_event.set() # Not paused by default
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"model_id": self.model_id,
|
||||
"status": self.status.value,
|
||||
"progress_percent": round(self.progress_percent, 1),
|
||||
"bytes_downloaded": self.bytes_downloaded,
|
||||
"total_bytes": self.total_bytes,
|
||||
"speed_mbps": round(self.speed_mbps, 2),
|
||||
"error_message": self.error_message,
|
||||
}
|
||||
|
||||
|
||||
# Global download tracking
|
||||
download_progress: Dict[str, DownloadProgress] = {}
|
||||
|
||||
|
||||
# Model download metadata (approximate sizes from HuggingFace)
|
||||
MODEL_DOWNLOAD_INFO = {
|
||||
# Demucs models (downloaded automatically by the demucs library on first use via torch.hub)
|
||||
"htdemucs": {
|
||||
"size_mb": 80,
|
||||
"description": "Demucs v4 hybrid transformer model. 4-stem separation (vocals, drums, bass, other). State-of-the-art quality; downloaded automatically on first use.",
|
||||
"hf_repo": None,
|
||||
"files": [],
|
||||
},
|
||||
"htdemucs_ft": {
|
||||
"size_mb": 400,
|
||||
"description": "Fine-tuned Demucs v4 with separate per-stem models. Improved vocal and drum quality over htdemucs; downloaded automatically on first use.",
|
||||
"hf_repo": None,
|
||||
"files": [],
|
||||
},
|
||||
"mdx_extra_q": {
|
||||
"size_mb": 500,
|
||||
"description": "Demucs v3 MDX Extra Quantized — quantized for faster inference and lower memory usage. Legacy model, superseded by htdemucs; downloaded automatically on first use.",
|
||||
"hf_repo": None,
|
||||
"files": [],
|
||||
},
|
||||
"mdx": {
|
||||
"size_mb": 280,
|
||||
"description": "Demucs v3 baseline 4-stem model (vocals, drums, bass, other). Legacy predecessor to htdemucs; downloaded automatically on first use.",
|
||||
"hf_repo": None,
|
||||
"files": [],
|
||||
},
|
||||
|
||||
# BS-RoFormer standard models — all share the same SW-6 weights from jarredou/BS-ROFO-SW-Fixed.
|
||||
# The 4/5/6-stem "modes" are produced by merging the 6-stem outputs in bs_roformer_separator.py.
|
||||
"bs_roformer_4stem": {
|
||||
"size_mb": 204,
|
||||
"description": "BS-RoFormer 4-stem separation (vocals, drums, bass, other). Good balance of quality and speed.",
|
||||
"hf_repo": "jarredou/BS-ROFO-SW-Fixed",
|
||||
"files": ["BS-Rofo-SW-Fixed.ckpt", "BS-Rofo-SW-Fixed.yaml"],
|
||||
},
|
||||
"bs_roformer_5stem_guitar": {
|
||||
"size_mb": 204,
|
||||
"description": "BS-RoFormer 5-stem with guitar separation (vocals, drums, bass, guitar, other).",
|
||||
"hf_repo": "jarredou/BS-ROFO-SW-Fixed",
|
||||
"files": ["BS-Rofo-SW-Fixed.ckpt", "BS-Rofo-SW-Fixed.yaml"],
|
||||
},
|
||||
"bs_roformer_5stem_piano": {
|
||||
"size_mb": 204,
|
||||
"description": "BS-RoFormer 5-stem with piano separation (vocals, drums, bass, piano, other).",
|
||||
"hf_repo": "jarredou/BS-ROFO-SW-Fixed",
|
||||
"files": ["BS-Rofo-SW-Fixed.ckpt", "BS-Rofo-SW-Fixed.yaml"],
|
||||
},
|
||||
"bs_roformer_6stem": {
|
||||
"size_mb": 204,
|
||||
"description": "BS-RoFormer 6-stem full separation (vocals, drums, bass, guitar, piano, other).",
|
||||
"hf_repo": "jarredou/BS-ROFO-SW-Fixed",
|
||||
"files": ["BS-Rofo-SW-Fixed.ckpt", "BS-Rofo-SW-Fixed.yaml"],
|
||||
},
|
||||
"bs_roformer_chorus": {
|
||||
"size_mb": 204,
|
||||
"description": "BS-RoFormer specialized for male/female chorus separation.",
|
||||
"hf_repo": None, # No public download URL — install manually
|
||||
"files": ["model.ckpt", "config.yaml"],
|
||||
},
|
||||
"bs_roformer_anvuew": {
|
||||
"size_mb": 195,
|
||||
"description": "BS-RoFormer Anvuew variant optimized for vocal separation (SDR 12.45).",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["BS-RoFormer-Anvuew/bs_roformer_anvuew_sdr_12.45.ckpt", "BS-RoFormer-Anvuew/config.yaml"],
|
||||
},
|
||||
"bs_roformer_karaoke_anvuew": {
|
||||
"size_mb": 195,
|
||||
"description": "BS-RoFormer Karaoke Anvuew for lead/backing separation.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Karaoke-BS-RoFormer-Anvuew/karaoke_bs_roformer_anvuew.ckpt", "Karaoke-BS-RoFormer-Anvuew/karaoke_bs_roformer_anvuew.yaml"],
|
||||
},
|
||||
"bs_roformer_sw_6stem": {
|
||||
"size_mb": 180,
|
||||
"description": "BS-RoFormer SW (Swin Transformer) 6-stem variant.",
|
||||
"hf_repo": None, # No public download URL — install manually
|
||||
"files": ["model.ckpt"],
|
||||
},
|
||||
|
||||
# MelBand RoFormer models
|
||||
"mel_band_roformer_4stem": {
|
||||
"size_mb": 170,
|
||||
"description": "MelBand RoFormer 4-stem separation. Efficient alternative to BS-RoFormer.",
|
||||
"hf_repo": None, # No public download URL — install manually
|
||||
"files": ["model.ckpt", "config.yaml"],
|
||||
},
|
||||
"mel_band_vocals": {
|
||||
"size_mb": 170,
|
||||
"description": "MelBand RoFormer optimized for 2-stem vocals/instrumental separation.",
|
||||
"hf_repo": "becruily/mel-band-roformer-vocals",
|
||||
"files": ["model.ckpt", "config.yaml"],
|
||||
},
|
||||
"mel_band_vocals_anvuew": {
|
||||
"size_mb": 195,
|
||||
"description": "MelBand Vocals Anvuew variant with enhanced vocal extraction.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["BS-RoFormer-Anvuew/bs_roformer_anvuew_sdr_12.45.ckpt", "BS-RoFormer-Anvuew/config.yaml"],
|
||||
},
|
||||
"mel_band_dereverb": {
|
||||
"size_mb": 170,
|
||||
"description": "MelBand RoFormer for reverb removal - separates dry vocals from reverb.",
|
||||
"hf_repo": "anvuew/dereverb_mel_band_roformer",
|
||||
"files": ["dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt", "config.yaml"],
|
||||
},
|
||||
"mel_band_dereverb_anvuew": {
|
||||
"size_mb": 195,
|
||||
"description": "MelBand Dereverb Anvuew variant (SDR 19.17) - aggressive reverb removal.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Dereverb-MelBand-Anvuew/dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt", "Dereverb-MelBand-Anvuew/dereverb_mel_band_roformer_anvuew.yaml"],
|
||||
},
|
||||
"mel_band_dereverb_room": {
|
||||
"size_mb": 195,
|
||||
"description": "MelBand Room Dereverb for room/ambient reverb removal (SDR 13.74).",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Dereverb-Room-Anvuew/dereverb_room_anvuew_sdr_13.7432.ckpt", "Dereverb-Room-Anvuew/dereverb_room_anvuew.yaml"],
|
||||
},
|
||||
"mel_band_karaoke": {
|
||||
"size_mb": 170,
|
||||
"description": "MelBand RoFormer for karaoke - separates lead vocals from backing+instrumental.",
|
||||
"hf_repo": "becruily/mel-band-roformer-karaoke",
|
||||
"files": ["model.ckpt", "config.yaml"],
|
||||
},
|
||||
"mel_band_karaoke_gabox": {
|
||||
"size_mb": 195,
|
||||
"description": "MelBand Karaoke Gabox variant optimized for backing vocal separation.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["MelBandRoformers/bsroformers/bs_karaoke_gabox_IS.ckpt", "MelBandRoformers/bsroformers/karaoke_bs_roformer.yaml"],
|
||||
},
|
||||
"mel_band_deux_becruily": {
|
||||
"size_mb": 180,
|
||||
"description": "MelBand RoFormer Deux Becruily - specialized 2-stem vocals model.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["MelBand-Roformer-Deux-Becruily/becruily_deux.ckpt", "MelBand-Roformer-Deux-Becruily/config_deux_becruily.yaml"],
|
||||
},
|
||||
|
||||
# MDX-Net models
|
||||
"mdxnet_voc_ft": {
|
||||
"size_mb": 45,
|
||||
"description": "MDX-Net Vocals Fine-tuned - lightweight vocals model (ONNX).",
|
||||
"hf_repo": "Politrees/UVR_resources",
|
||||
"files": ["models/MDXNet/UVR-MDX-NET-Voc_FT.onnx"],
|
||||
},
|
||||
"mdxnet_inst_hq_1": {
|
||||
"size_mb": 45,
|
||||
"description": "MDX-Net Instrumental HQ 1 - focused on instrumental extraction.",
|
||||
"hf_repo": "Politrees/UVR_resources",
|
||||
"files": ["models/MDXNet/UVR-MDX-NET-Inst_HQ_1.onnx"],
|
||||
},
|
||||
"mdxnet_inst_hq_2": {
|
||||
"size_mb": 45,
|
||||
"description": "MDX-Net Instrumental HQ 2 - alternative instrumental model.",
|
||||
"hf_repo": "Politrees/UVR_resources",
|
||||
"files": ["models/MDXNet/UVR-MDX-NET-Inst_HQ_2.onnx"],
|
||||
},
|
||||
"mdxnet_karaoke": {
|
||||
"size_mb": 45,
|
||||
"description": "MDX-Net Karaoke - separates lead from backing vocals.",
|
||||
"hf_repo": "Politrees/UVR_resources",
|
||||
"files": ["models/MDXNet/UVR_MDXNET_KARA.onnx"],
|
||||
},
|
||||
"mdxnet_dereverb": {
|
||||
"size_mb": 195,
|
||||
"description": "MDX-Net Dereverb via MelBand Anvuew architecture.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Dereverb-MelBand-Anvuew/dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt"],
|
||||
},
|
||||
"mdxnet_drumsep": {
|
||||
"size_mb": 448,
|
||||
"description": "MDX23C DrumSep - specialized drum separation model.",
|
||||
"hf_repo": "Politrees/UVR_resources",
|
||||
"files": ["models/MDX23C/MDX23C-DrumSep-aufr33-jarredou.ckpt"],
|
||||
},
|
||||
|
||||
# SCNet XL models (2024-2025 state-of-the-art)
|
||||
"scnet_xl": {
|
||||
"size_mb": 350,
|
||||
"description": "SCNet XL - State-of-the-art 2-stem separation (SDR 11.93 vocals, 18.23 instrumental).",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Politrees/UVR_resources/models/SCnet/scnet_checkpoint_musdb18.ckpt", "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_xl.yaml"],
|
||||
},
|
||||
"scnet_xl_ihf": {
|
||||
"size_mb": 350,
|
||||
"description": "SCNet XL IHF variant - optimized for instrumental extraction.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Politrees/UVR_resources/models/SCnet/scnet_checkpoint_musdb18.ckpt", "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_xl.yaml"],
|
||||
},
|
||||
"scnet_large": {
|
||||
"size_mb": 200,
|
||||
"description": "SCNet Large - 4-stem variant with excellent quality.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Politrees/UVR_resources/models/SCnet/SCNet-large_starrytong_fixed.ckpt", "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_large_starrytong.yaml"],
|
||||
},
|
||||
"scnet_large_starrytong": {
|
||||
"size_mb": 200,
|
||||
"description": "SCNet Large Starrytong - fixed variant with improved stability.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Politrees/UVR_resources/models/SCnet/SCNet-large_starrytong_fixed.ckpt", "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_large_starrytong.yaml"],
|
||||
},
|
||||
|
||||
# MedleyVox models
|
||||
"medleyvox": {
|
||||
"size_mb": 170,
|
||||
"description": "MedleyVox - Multi-singer vocal separation for ensemble vocals.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["MedleyVox-MultiSinger/vocals 100.pth"],
|
||||
},
|
||||
"medleyvox_55ep": {
|
||||
"size_mb": 170,
|
||||
"description": "MedleyVox 55 Epochs - earlier training checkpoint.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["MedleyVox-MultiSinger/vocals 55 epochs.pth"],
|
||||
},
|
||||
"medleyvox_66ep": {
|
||||
"size_mb": 170,
|
||||
"description": "MedleyVox 66 Epochs - intermediate checkpoint.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["MedleyVox-MultiSinger/vocals 66.pth"],
|
||||
},
|
||||
"medleyvox_88ep": {
|
||||
"size_mb": 170,
|
||||
"description": "MedleyVox 88 Epochs - near-final checkpoint.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["MedleyVox-MultiSinger/vocals 88.pth"],
|
||||
},
|
||||
"medleyvox_100ep": {
|
||||
"size_mb": 170,
|
||||
"description": "MedleyVox 100 Epochs - final training checkpoint.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["MedleyVox-MultiSinger/vocals 100.pth"],
|
||||
},
|
||||
|
||||
# Apollo models
|
||||
"apollo_vocals": {
|
||||
"size_mb": 400,
|
||||
"description": "Apollo Vocals - Vocal enhancement and reverb removal (MSST).",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Apollo-Vocal-MSST/epoch=54-val_loss=-17.6221.ckpt", "Apollo-Vocal-MSST/config_apollo_vocals_ep_54.yaml"],
|
||||
},
|
||||
"apollo_edm": {
|
||||
"size_mb": 200,
|
||||
"description": "Apollo EDM - Optimized for EDM vocal enhancement.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Politrees/UVR_resources/models/Apollo/apollo_edm_by_essid.ckpt", "Politrees/UVR_resources/models/Apollo/apollo_edm_by_essid.yaml"],
|
||||
},
|
||||
"apollo_edm_big": {
|
||||
"size_mb": 300,
|
||||
"description": "Apollo EDM Big - Larger EDM vocal enhancement model.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Politrees/UVR_resources/models/Apollo/apollo_edm_big_by_essid.ckpt", "Politrees/UVR_resources/models/Apollo/apollo_edm_big_by_essid.yaml"],
|
||||
},
|
||||
|
||||
# Aspiration models
|
||||
"aspiration_full": {
|
||||
"size_mb": 200,
|
||||
"description": "Aspiration Removal (Full) - Aggressive breath/aspiration removal (SDR 18.98).",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Aspiration-MelBand-Sucial/aspiration_mel_band_roformer_sdr_18.9845.ckpt", "Aspiration-MelBand-Sucial/config_aspiration_mel_band_roformer.yaml"],
|
||||
},
|
||||
"aspiration_light": {
|
||||
"size_mb": 200,
|
||||
"description": "Aspiration Removal (Light) - Gentler breath removal (SDR 18.12).",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Aspiration-MelBand-Sucial/aspiration_mel_band_roformer_less_aggr_sdr_18.1201.ckpt", "Aspiration-MelBand-Sucial/config_aspiration_mel_band_roformer.yaml"],
|
||||
},
|
||||
|
||||
# Bandit models
|
||||
"bandit_v2_multi": {
|
||||
"size_mb": 250,
|
||||
"description": "Bandit v2 Multi-Language - Multi-language vocal separation.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Bandit-v2/checkpoint-multi_state_dict.ckpt", "Bandit-v2/config_dnr_bandit_v2_mus64.yaml"],
|
||||
},
|
||||
"bandit_v2_eng": {
|
||||
"size_mb": 250,
|
||||
"description": "Bandit v2 English - Optimized for English vocals.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Bandit-v2/checkpoint-eng_state_dict.ckpt", "Bandit-v2/config_dnr_bandit_v2_mus64.yaml"],
|
||||
},
|
||||
|
||||
# Ensemble modes (no model files — combine outputs from multiple other models)
|
||||
"ensemble_vocal_best": {
|
||||
"size_mb": 0,
|
||||
"description": "Ensemble combining multiple top vocal separators (BS-RoFormer + SCNet) for best possible vocal quality. Requires constituent models to be downloaded.",
|
||||
"hf_repo": None,
|
||||
"files": [],
|
||||
},
|
||||
"ensemble_4stem": {
|
||||
"size_mb": 0,
|
||||
"description": "4-stem ensemble combining BS-RoFormer and SCNet Large outputs for vocals, drums, bass, and other. Higher quality than any single model.",
|
||||
"hf_repo": None,
|
||||
"files": [],
|
||||
},
|
||||
"ensemble_6stem": {
|
||||
"size_mb": 0,
|
||||
"description": "Full 6-stem ensemble combining multiple models for vocals, drums, bass, guitar, piano, and other. Highest quality multi-instrument separation.",
|
||||
"hf_repo": None,
|
||||
"files": [],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# Explicit mapping from model_id to the exact directory its separator service uses.
|
||||
# Must stay in sync with the DEFAULT_MODEL_DIR / model_dir values in each *_separator.py.
|
||||
MODEL_EXACT_DIRS: Dict[str, Path] = {
|
||||
# ── BS-RoFormer standard (all share the same weights flat in the family dir) ──
|
||||
"bs_roformer_4stem": BS_ROFORMER_MODEL_DIR,
|
||||
"bs_roformer_5stem_guitar": BS_ROFORMER_MODEL_DIR,
|
||||
"bs_roformer_5stem_piano": BS_ROFORMER_MODEL_DIR,
|
||||
"bs_roformer_6stem": BS_ROFORMER_MODEL_DIR,
|
||||
# ── BS-RoFormer specialized ──
|
||||
"bs_roformer_chorus": BS_ROFORMER_MODEL_DIR / "chorus",
|
||||
"bs_roformer_sw_6stem": BS_ROFORMER_MODEL_DIR, # flat; hf_repo=None
|
||||
"bs_roformer_anvuew": MEL_BAND_MODEL_DIR / "vocals_anvuew",
|
||||
"bs_roformer_karaoke_anvuew": MEL_BAND_MODEL_DIR / "karaoke_gabox",
|
||||
# ── MelBand RoFormer ──
|
||||
"mel_band_roformer_4stem": MEL_BAND_MODEL_DIR, # flat; hf_repo=None
|
||||
"mel_band_vocals": MEL_BAND_MODEL_DIR / "vocals",
|
||||
"mel_band_vocals_anvuew": MEL_BAND_MODEL_DIR / "vocals_anvuew",
|
||||
"mel_band_dereverb": MEL_BAND_MODEL_DIR / "dereverb",
|
||||
"mel_band_dereverb_anvuew": MEL_BAND_MODEL_DIR / "dereverb_anvuew",
|
||||
"mel_band_dereverb_room": MEL_BAND_MODEL_DIR / "dereverb_room",
|
||||
"mel_band_karaoke": MEL_BAND_MODEL_DIR / "karaoke",
|
||||
"mel_band_karaoke_gabox": MEL_BAND_MODEL_DIR / "karaoke_gabox",
|
||||
"mel_band_deux_becruily": MEL_BAND_MODEL_DIR / "deux_becruily",
|
||||
# ── MDX-Net (flat family dir — separator uses basename only) ──
|
||||
"mdxnet_voc_ft": MDXNET_MODEL_DIR,
|
||||
"mdxnet_inst_hq_1": MDXNET_MODEL_DIR,
|
||||
"mdxnet_inst_hq_2": MDXNET_MODEL_DIR,
|
||||
"mdxnet_karaoke": MDXNET_MODEL_DIR,
|
||||
"mdxnet_dereverb": MEL_BAND_MODEL_DIR / "dereverb", # routed to MelBand
|
||||
"mdxnet_drumsep": MDXNET_MODEL_DIR,
|
||||
# ── SCNet (per-model subdir) ──
|
||||
"scnet_xl": SCNET_MODEL_DIR / "scnet_xl",
|
||||
"scnet_xl_ihf": SCNET_MODEL_DIR / "scnet_xl_ihf",
|
||||
"scnet_large": SCNET_MODEL_DIR / "scnet_large",
|
||||
"scnet_large_starrytong": SCNET_MODEL_DIR / "scnet_large_starrytong",
|
||||
# ── MedleyVox (per-model subdir) ──
|
||||
"medleyvox": MEDLEYVOX_MODEL_DIR / "medleyvox",
|
||||
"medleyvox_55ep": MEDLEYVOX_MODEL_DIR / "medleyvox_55ep",
|
||||
"medleyvox_66ep": MEDLEYVOX_MODEL_DIR / "medleyvox_66ep",
|
||||
"medleyvox_88ep": MEDLEYVOX_MODEL_DIR / "medleyvox_88ep",
|
||||
"medleyvox_100ep": MEDLEYVOX_MODEL_DIR / "medleyvox_100ep",
|
||||
# ── Apollo (per-model subdir) ──
|
||||
"apollo_vocals": APOLLO_MODEL_DIR / "apollo_vocals",
|
||||
"apollo_edm": APOLLO_MODEL_DIR / "apollo_edm",
|
||||
"apollo_edm_big": APOLLO_MODEL_DIR / "apollo_edm_big",
|
||||
# ── Aspiration (per-model subdir) ──
|
||||
"aspiration_full": ASPIRATION_MODEL_DIR / "aspiration_full",
|
||||
"aspiration_light": ASPIRATION_MODEL_DIR / "aspiration_light",
|
||||
# ── Bandit (per-model subdir) ──
|
||||
"bandit_v2_multi": BANDIT_MODEL_DIR / "bandit_v2_multi",
|
||||
"bandit_v2_eng": BANDIT_MODEL_DIR / "bandit_v2_eng",
|
||||
}
|
||||
|
||||
|
||||
def get_model_directory(model_id: str) -> Path:
|
||||
"""Get the exact directory where a model's files are stored on disk."""
|
||||
return MODEL_EXACT_DIRS.get(model_id, BS_ROFORMER_MODEL_DIR / model_id)
|
||||
|
||||
|
||||
def is_model_downloaded(model_id: str) -> bool:
|
||||
"""Check if a model is already downloaded."""
|
||||
if model_id not in MODEL_DOWNLOAD_INFO:
|
||||
return False
|
||||
|
||||
model_dir = get_model_directory(model_id)
|
||||
info = MODEL_DOWNLOAD_INFO[model_id]
|
||||
|
||||
for file_path in info["files"]:
|
||||
flat = model_dir / Path(file_path).name # flat download (our convention)
|
||||
deep = model_dir / file_path # subdirectory structure
|
||||
if not flat.exists() and not deep.exists():
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class ModelInfo(BaseModel):
|
||||
id: str
|
||||
label: str
|
||||
group: str
|
||||
description: str
|
||||
size_mb: int
|
||||
status: str # not_downloaded, downloaded, downloading, paused, error
|
||||
progress_percent: float = 0.0
|
||||
|
||||
|
||||
class ModelDownloadRequest(BaseModel):
|
||||
model_id: str
|
||||
|
||||
|
||||
class ModelDownloadResponse(BaseModel):
|
||||
success: bool
|
||||
message: str
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/list", response_model=List[ModelInfo])
|
||||
def list_models():
|
||||
"""List all available models with their download status."""
|
||||
result = []
|
||||
|
||||
for model in ALL_MODELS:
|
||||
model_id = model["id"]
|
||||
info = MODEL_DOWNLOAD_INFO.get(model_id, {})
|
||||
|
||||
# Check current download status
|
||||
if model_id in download_progress:
|
||||
prog = download_progress[model_id]
|
||||
status = prog.status.value
|
||||
progress = prog.progress_percent
|
||||
elif is_model_downloaded(model_id):
|
||||
status = ModelDownloadStatus.COMPLETED.value
|
||||
progress = 100.0
|
||||
else:
|
||||
status = ModelDownloadStatus.NOT_DOWNLOADED.value
|
||||
progress = 0.0
|
||||
|
||||
result.append(ModelInfo(
|
||||
id=model_id,
|
||||
label=model["label"],
|
||||
group=model.get("group", "Other"),
|
||||
description=info.get("description", "No description available."),
|
||||
size_mb=info.get("size_mb", 0),
|
||||
status=status,
|
||||
progress_percent=progress,
|
||||
))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/download", response_model=ModelDownloadResponse)
|
||||
def download_model(body: ModelDownloadRequest, background_tasks: BackgroundTasks):
|
||||
"""Start downloading a model in the background."""
|
||||
model_id = body.model_id
|
||||
|
||||
if model_id not in MODEL_DOWNLOAD_INFO:
|
||||
raise HTTPException(status_code=404, detail=f"Unknown model: {model_id}")
|
||||
|
||||
# Check if already downloaded
|
||||
if is_model_downloaded(model_id):
|
||||
return ModelDownloadResponse(success=True, message=f"Model {model_id} is already downloaded.")
|
||||
|
||||
# Check if already downloading
|
||||
if model_id in download_progress:
|
||||
prog = download_progress[model_id]
|
||||
if prog.status == ModelDownloadStatus.DOWNLOADING:
|
||||
return ModelDownloadResponse(success=True, message=f"Download already in progress for {model_id}.")
|
||||
elif prog.status == ModelDownloadStatus.PAUSED:
|
||||
# Resume
|
||||
prog._pause_event.set()
|
||||
prog.status = ModelDownloadStatus.DOWNLOADING
|
||||
return ModelDownloadResponse(success=True, message=f"Resuming download for {model_id}.")
|
||||
|
||||
# Start new download
|
||||
progress = DownloadProgress(
|
||||
model_id=model_id,
|
||||
status=ModelDownloadStatus.DOWNLOADING,
|
||||
total_bytes=MODEL_DOWNLOAD_INFO[model_id]["size_mb"] * 1024 * 1024,
|
||||
)
|
||||
download_progress[model_id] = progress
|
||||
|
||||
# Start background download
|
||||
background_tasks.add_task(_download_model_task, model_id, progress)
|
||||
|
||||
return ModelDownloadResponse(success=True, message=f"Started downloading {model_id}.")
|
||||
|
||||
|
||||
@router.post("/pause")
|
||||
def pause_download(body: ModelDownloadRequest):
|
||||
"""Pause an ongoing download."""
|
||||
model_id = body.model_id
|
||||
|
||||
if model_id not in download_progress:
|
||||
raise HTTPException(status_code=404, detail=f"No active download for {model_id}")
|
||||
|
||||
prog = download_progress[model_id]
|
||||
if prog.status != ModelDownloadStatus.DOWNLOADING:
|
||||
raise HTTPException(status_code=400, detail=f"Download is not in progress (status: {prog.status.value})")
|
||||
|
||||
prog._pause_event.clear()
|
||||
prog.status = ModelDownloadStatus.PAUSED
|
||||
|
||||
return {"success": True, "message": f"Paused download for {model_id}"}
|
||||
|
||||
|
||||
@router.post("/resume")
|
||||
def resume_download(body: ModelDownloadRequest):
|
||||
"""Resume a paused download."""
|
||||
model_id = body.model_id
|
||||
|
||||
if model_id not in download_progress:
|
||||
raise HTTPException(status_code=404, detail=f"No download to resume for {model_id}")
|
||||
|
||||
prog = download_progress[model_id]
|
||||
if prog.status != ModelDownloadStatus.PAUSED:
|
||||
raise HTTPException(status_code=400, detail=f"Download is not paused (status: {prog.status.value})")
|
||||
|
||||
prog._pause_event.set()
|
||||
prog.status = ModelDownloadStatus.DOWNLOADING
|
||||
|
||||
return {"success": True, "message": f"Resumed download for {model_id}"}
|
||||
|
||||
|
||||
@router.post("/cancel")
|
||||
def cancel_download(body: ModelDownloadRequest):
|
||||
"""Cancel an ongoing or paused download."""
|
||||
model_id = body.model_id
|
||||
|
||||
if model_id not in download_progress:
|
||||
raise HTTPException(status_code=404, detail=f"No active download for {model_id}")
|
||||
|
||||
prog = download_progress[model_id]
|
||||
|
||||
# Signal cancellation
|
||||
prog._cancel_event.set()
|
||||
prog.status = ModelDownloadStatus.CANCELLED
|
||||
|
||||
# Clean up partial downloads
|
||||
_cleanup_partial_download(model_id)
|
||||
|
||||
return {"success": True, "message": f"Cancelled download for {model_id}"}
|
||||
|
||||
|
||||
@router.get("/progress/{model_id}")
|
||||
def get_progress(model_id: str):
|
||||
"""Get download progress for a specific model."""
|
||||
if model_id not in download_progress:
|
||||
# Check if downloaded
|
||||
if is_model_downloaded(model_id):
|
||||
return {
|
||||
"model_id": model_id,
|
||||
"status": ModelDownloadStatus.COMPLETED.value,
|
||||
"progress_percent": 100.0,
|
||||
"bytes_downloaded": MODEL_DOWNLOAD_INFO.get(model_id, {}).get("size_mb", 0) * 1024 * 1024,
|
||||
"total_bytes": MODEL_DOWNLOAD_INFO.get(model_id, {}).get("size_mb", 0) * 1024 * 1024,
|
||||
"speed_mbps": 0.0,
|
||||
}
|
||||
raise HTTPException(status_code=404, detail=f"No download found for {model_id}")
|
||||
|
||||
return download_progress[model_id].to_dict()
|
||||
|
||||
|
||||
@router.get("/progress")
|
||||
def get_all_progress():
|
||||
"""Get download progress for all models."""
|
||||
return {model_id: prog.to_dict() for model_id, prog in download_progress.items()}
|
||||
|
||||
|
||||
@router.delete("/{model_id}")
|
||||
def delete_model(model_id: str):
|
||||
"""Delete a downloaded model to free up space."""
|
||||
if model_id not in MODEL_DOWNLOAD_INFO:
|
||||
raise HTTPException(status_code=404, detail=f"Unknown model: {model_id}")
|
||||
|
||||
if not is_model_downloaded(model_id):
|
||||
raise HTTPException(status_code=400, detail=f"Model {model_id} is not downloaded")
|
||||
|
||||
# Remove files
|
||||
model_dir = get_model_directory(model_id)
|
||||
info = MODEL_DOWNLOAD_INFO[model_id]
|
||||
|
||||
deleted = 0
|
||||
for file_path in info["files"]:
|
||||
# Try flat first (our download convention), then deep path
|
||||
for candidate in (model_dir / Path(file_path).name, model_dir / file_path):
|
||||
if candidate.exists():
|
||||
candidate.unlink()
|
||||
deleted += 1
|
||||
break
|
||||
|
||||
# Clean up progress tracking
|
||||
if model_id in download_progress:
|
||||
del download_progress[model_id]
|
||||
|
||||
return {"success": True, "message": f"Deleted {deleted} files for {model_id}"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Background download task
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _download_model_task(model_id: str, progress: DownloadProgress):
|
||||
"""Background task to download a model from HuggingFace."""
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
info = MODEL_DOWNLOAD_INFO[model_id]
|
||||
hf_repo = info.get("hf_repo")
|
||||
|
||||
# Reject placeholder / unconfigured repos immediately
|
||||
if not hf_repo:
|
||||
progress.status = ModelDownloadStatus.ERROR
|
||||
progress.error_message = (
|
||||
"No download URL configured for this model. "
|
||||
"Install the weights manually into the model directory."
|
||||
)
|
||||
return
|
||||
|
||||
model_dir = get_model_directory(model_id)
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
total_files = len(info["files"])
|
||||
|
||||
for idx, file_path in enumerate(info["files"]):
|
||||
# Check for cancellation
|
||||
if progress._cancel_event.is_set():
|
||||
progress.status = ModelDownloadStatus.CANCELLED
|
||||
return
|
||||
|
||||
# Block here until resumed (threading.Event.wait() works correctly in threads)
|
||||
progress._pause_event.wait()
|
||||
|
||||
if progress._cancel_event.is_set():
|
||||
progress.status = ModelDownloadStatus.CANCELLED
|
||||
return
|
||||
|
||||
dest = model_dir / Path(file_path).name
|
||||
if dest.exists():
|
||||
# Already present — count it and skip
|
||||
progress.bytes_downloaded += dest.stat().st_size
|
||||
progress.progress_percent = (idx + 1) / total_files * 100
|
||||
continue
|
||||
|
||||
try:
|
||||
# Download to a temp directory first, then move flat into model_dir.
|
||||
# This ensures is_model_downloaded() finds files at the expected path
|
||||
# regardless of the subdirectory structure inside the HF repo.
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp_path = hf_hub_download(
|
||||
repo_id=hf_repo,
|
||||
filename=file_path,
|
||||
local_dir=tmp,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
shutil.copy2(tmp_path, str(dest))
|
||||
|
||||
progress.bytes_downloaded += dest.stat().st_size
|
||||
progress.progress_percent = (idx + 1) / total_files * 100
|
||||
|
||||
except Exception as e:
|
||||
progress.status = ModelDownloadStatus.ERROR
|
||||
progress.error_message = str(e)
|
||||
return
|
||||
|
||||
progress.progress_percent = 100.0
|
||||
progress.status = ModelDownloadStatus.COMPLETED
|
||||
|
||||
except Exception as e:
|
||||
progress.status = ModelDownloadStatus.ERROR
|
||||
progress.error_message = str(e)
|
||||
|
||||
|
||||
def _cleanup_partial_download(model_id: str):
|
||||
"""Remove partially downloaded files."""
|
||||
model_dir = get_model_directory(model_id)
|
||||
if model_dir.exists():
|
||||
import shutil
|
||||
shutil.rmtree(model_dir, ignore_errors=True)
|
||||
Executable
+53
@@ -0,0 +1,53 @@
|
||||
"""Full pipeline endpoint: separate → transcribe → ABC."""
|
||||
import shutil
|
||||
|
||||
from fastapi import APIRouter, File, Form, UploadFile, HTTPException
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import (
|
||||
UPLOAD_DIR, ALL_MODEL_IDS, OUTPUT_FORMATS, TRANSCRIBE_MODES,
|
||||
DEFAULT_MODEL, DEFAULT_OUTPUT_FORMAT, DEFAULT_TRANSCRIBE_MODE,
|
||||
)
|
||||
from ..services.pipeline_service import run_pipeline
|
||||
|
||||
router = APIRouter(prefix="/api/pipeline", tags=["pipeline"])
|
||||
|
||||
VALID_FORMATS = {f["id"] for f in OUTPUT_FORMATS}
|
||||
VALID_MODES = {m["id"] for m in TRANSCRIBE_MODES}
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def pipeline(
|
||||
file: UploadFile = File(...),
|
||||
sep_model: str = Form(DEFAULT_MODEL),
|
||||
sep_format: str = Form(DEFAULT_OUTPUT_FORMAT),
|
||||
transcribe_mode: str = Form(DEFAULT_TRANSCRIBE_MODE),
|
||||
song_title: str = Form(""),
|
||||
):
|
||||
if sep_model not in ALL_MODEL_IDS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {sep_model}")
|
||||
if sep_format not in VALID_FORMATS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown format: {sep_format}")
|
||||
if transcribe_mode not in VALID_MODES:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown mode: {transcribe_mode}")
|
||||
|
||||
# Fall back to filename as song title
|
||||
effective_title = song_title.strip() or file.filename.rsplit(".", 1)[0]
|
||||
|
||||
job_id = jm.create_job("pipeline")
|
||||
upload_path = UPLOAD_DIR / job_id / file.filename
|
||||
upload_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(upload_path, "wb") as f:
|
||||
shutil.copyfileobj(file.file, f)
|
||||
|
||||
jm.submit_gpu(
|
||||
run_pipeline,
|
||||
job_id,
|
||||
str(upload_path),
|
||||
sep_model,
|
||||
sep_format,
|
||||
transcribe_mode,
|
||||
effective_title,
|
||||
)
|
||||
return {"job_id": job_id}
|
||||
Executable
+751
@@ -0,0 +1,751 @@
|
||||
"""
|
||||
Project management endpoints.
|
||||
|
||||
A Project is created automatically when a separation job completes.
|
||||
It links the separation job to downstream jobs (transcription, ABC, mix).
|
||||
"""
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import PROJECTS_DIR, UPLOAD_DIR, TRANSCRIBE_MODES, DEFAULT_TRANSCRIBE_MODE, STATIC_OPTIONS
|
||||
from ..services.project_metadata import write_project_metadata, read_project_metadata
|
||||
from ..services.transcriber_service import run_transcription
|
||||
from ..services.abc_service import run_abc_conversion
|
||||
from ..services.mix_service import run_mix
|
||||
from ..services.lrc_service import run_lrc_alignment, clean_lrc_content
|
||||
from ..services.lyric_transcriber_service import (
|
||||
run_lyric_transcription,
|
||||
WHISPER_MODELS,
|
||||
DEFAULT_WHISPER_MODEL,
|
||||
)
|
||||
from ..job_manager import set_project_trans_job, set_project_abc_job, set_project_lrc_job
|
||||
|
||||
router = APIRouter(prefix="/api/projects", tags=["projects"])
|
||||
|
||||
VALID_TRANSCRIBE_MODES = {m["id"] for m in TRANSCRIBE_MODES}
|
||||
VALID_STATIC_OPTIONS = {o["id"] for o in STATIC_OPTIONS}
|
||||
_AUDIO_EXTS = {".mp3", ".wav", ".flac"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request bodies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TranscribeRequest(BaseModel):
|
||||
stem_name: str # which stem to transcribe (key from sep job)
|
||||
mode: str = DEFAULT_TRANSCRIBE_MODE
|
||||
|
||||
|
||||
class AbcRequest(BaseModel):
|
||||
stem_name: str # which stem's MIDI to convert
|
||||
song_title: Optional[str] = None
|
||||
|
||||
|
||||
class LrcRequest(BaseModel):
|
||||
stem_name: str # which stem to align (usually vocals)
|
||||
lyrics_source: str = "text" # text | auto | tag
|
||||
lyrics_text: Optional[str] = None # required when lyrics_source='text'
|
||||
output_filename: Optional[str] = None
|
||||
style_options: Optional[dict] = None
|
||||
timing_precision: str = "word" # 'word' | 'syllable'
|
||||
lrc_power_preset: str = "standard" # 'eco' | 'standard' | 'performance'
|
||||
|
||||
|
||||
class MixRequest(BaseModel):
|
||||
preset_type: str # 'static' | 'dynamic'
|
||||
static_option: Optional[str] = None
|
||||
dynamic_levels: Optional[dict] = None # stem_name → float volume
|
||||
output_format: Optional[str] = None # inferred from stems if absent
|
||||
|
||||
|
||||
class LyricTranscribeRequest(BaseModel):
|
||||
"""Request body for AI-based lyric transcription using Whisper."""
|
||||
stem_name: str # which stem to transcribe (usually vocals)
|
||||
model: str = "large-v2" # whisper model: tiny, base, small, medium, large, large-v2, large-v3
|
||||
language: Optional[str] = None # ISO-639-1 code (auto-detect if None)
|
||||
task: str = "transcribe" # 'transcribe' or 'translate' (to English)
|
||||
output_formats: list = ["elrc", "json", "txt"] # lrc, elrc, json, txt, srt, vtt
|
||||
initial_prompt: Optional[str] = None # optional prompt to guide the model
|
||||
temperature: float = 0.0 # sampling temperature
|
||||
condition_on_previous_text: bool = False # if True, pass previous output as prompt; can cause hallucination loops
|
||||
engine: str = "whisperx" # 'whisperx' (only supported engine)
|
||||
diarize: bool = False # run speaker diarization (WhisperX only)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _get_project_or_404(project_id: str) -> dict:
|
||||
project = jm.get_project(project_id)
|
||||
if project is None:
|
||||
raise HTTPException(status_code=404, detail=f"Project not found: {project_id}")
|
||||
return project
|
||||
|
||||
|
||||
def _get_job_or_none(job_id: Optional[str]) -> Optional[dict]:
|
||||
if not job_id:
|
||||
return None
|
||||
return jm.get_job(job_id)
|
||||
|
||||
|
||||
def _stem_files_from_job(sep_job: dict) -> dict:
|
||||
"""Return {stem_name: abs_path} for all stem files in the separation job."""
|
||||
return {
|
||||
f["stem"]: f["path"]
|
||||
for f in sep_job.get("files", [])
|
||||
if f.get("type") == "stem" and f.get("stem") and Path(f.get("path", "")).exists()
|
||||
}
|
||||
|
||||
|
||||
def _infer_output_format(stem_files: dict) -> str:
|
||||
"""Infer output format from stem file extensions."""
|
||||
for path in stem_files.values():
|
||||
ext = Path(path).suffix.lower().lstrip(".")
|
||||
if ext in ("mp3", "flac", "wav"):
|
||||
return ext
|
||||
return "mp3"
|
||||
|
||||
|
||||
def _project_output_dir(project: dict) -> Path:
|
||||
"""Return the project's root directory under MEDIA_DIR."""
|
||||
sep_job = jm.get_job(project["sep_job_id"]) or {}
|
||||
result = sep_job.get("result") or {}
|
||||
project_path = result.get("project_path")
|
||||
if project_path and Path(project_path).is_dir():
|
||||
return Path(project_path)
|
||||
# Fallback: PROJECTS_DIR / project.name
|
||||
return PROJECTS_DIR / project["name"]
|
||||
|
||||
|
||||
def _find_stem_path(stem_files: dict, stem_name: str) -> Optional[str]:
|
||||
"""Return path for the named stem, matching by key or by file basename/stem."""
|
||||
if stem_name in stem_files:
|
||||
return stem_files[stem_name]
|
||||
for path in stem_files.values():
|
||||
p = Path(path)
|
||||
if p.name == stem_name or p.stem == stem_name:
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("")
|
||||
def list_projects():
|
||||
"""Return all projects, newest first. Migrates orphaned separation jobs."""
|
||||
return jm.list_projects()
|
||||
|
||||
|
||||
@router.get("/{project_id}")
|
||||
def get_project_detail(project_id: str):
|
||||
"""Return project with all linked jobs expanded."""
|
||||
project = _get_project_or_404(project_id)
|
||||
trans_jobs_expanded = {
|
||||
stem: jm.get_job(jid)
|
||||
for stem, jid in (project.get("trans_jobs") or {}).items()
|
||||
}
|
||||
abc_jobs_expanded = {
|
||||
stem: jm.get_job(jid)
|
||||
for stem, jid in (project.get("abc_jobs") or {}).items()
|
||||
}
|
||||
lrc_jobs_expanded = {
|
||||
stem: jm.get_job(jid)
|
||||
for stem, jid in (project.get("lrc_jobs") or {}).items()
|
||||
}
|
||||
return {
|
||||
**project,
|
||||
"sep_job": jm.get_job(project["sep_job_id"]),
|
||||
"trans_jobs": trans_jobs_expanded,
|
||||
"abc_jobs": abc_jobs_expanded,
|
||||
"lrc_jobs": lrc_jobs_expanded,
|
||||
"mix_job": _get_job_or_none(project["mix_job_id"]),
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/{project_id}")
|
||||
def delete_project(project_id: str):
|
||||
"""Remove the project record and all associated files from disk."""
|
||||
project = _get_project_or_404(project_id)
|
||||
|
||||
# Delete project directory from disk
|
||||
project_dir = _project_output_dir(project)
|
||||
if project_dir.exists():
|
||||
shutil.rmtree(project_dir, ignore_errors=True)
|
||||
|
||||
# Delete the upload directory for the originating separation job
|
||||
upload_dir = UPLOAD_DIR / project["sep_job_id"]
|
||||
if upload_dir.exists():
|
||||
shutil.rmtree(upload_dir, ignore_errors=True)
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
from ..job_manager import engine, ProjectRecord
|
||||
with Session(engine) as session:
|
||||
record = session.get(ProjectRecord, project_id)
|
||||
if record:
|
||||
session.delete(record)
|
||||
session.commit()
|
||||
return {"detail": "Project deleted"}
|
||||
|
||||
|
||||
@router.post("/{project_id}/transcribe")
|
||||
def transcribe_project(project_id: str, body: TranscribeRequest):
|
||||
"""Transcribe a specific stem from this project to MIDI."""
|
||||
if body.mode not in VALID_TRANSCRIBE_MODES:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown transcribe mode: {body.mode}")
|
||||
|
||||
project = _get_project_or_404(project_id)
|
||||
sep_job = jm.get_job(project["sep_job_id"])
|
||||
if sep_job is None:
|
||||
raise HTTPException(status_code=404, detail="Separation job not found")
|
||||
|
||||
stem_files = _stem_files_from_job(sep_job)
|
||||
if not stem_files:
|
||||
raise HTTPException(status_code=404, detail="No stem files found in separation job")
|
||||
|
||||
stem_path = _find_stem_path(stem_files, body.stem_name)
|
||||
if stem_path is None:
|
||||
raise HTTPException(status_code=404, detail=f"Stem not found: {body.stem_name}")
|
||||
|
||||
job_id = jm.create_job("transcribe")
|
||||
set_project_trans_job(project_id, body.stem_name, job_id)
|
||||
jm.submit_gpu(run_transcription, job_id, stem_path, body.mode)
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/{project_id}/abc")
|
||||
def abc_project(project_id: str, body: AbcRequest):
|
||||
"""Convert MIDI from a specific stem's transcription job to ABC notation."""
|
||||
project = _get_project_or_404(project_id)
|
||||
|
||||
trans_jobs_dict = project.get("trans_jobs") or {}
|
||||
trans_job_id = trans_jobs_dict.get(body.stem_name)
|
||||
if not trans_job_id:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"No transcription job for stem '{body.stem_name}'",
|
||||
)
|
||||
|
||||
trans_job = jm.get_job(trans_job_id)
|
||||
if trans_job is None:
|
||||
raise HTTPException(status_code=404, detail="Transcription job not found")
|
||||
if trans_job["status"] != "done":
|
||||
raise HTTPException(status_code=400, detail="Transcription job has not completed successfully")
|
||||
|
||||
midi_paths = [
|
||||
f["path"]
|
||||
for f in trans_job.get("files", [])
|
||||
if Path(f.get("path", "")).suffix.lower() == ".mid"
|
||||
and Path(f.get("path", "")).exists()
|
||||
]
|
||||
if not midi_paths:
|
||||
raise HTTPException(status_code=404, detail="No MIDI files found in transcription job")
|
||||
|
||||
title = (body.song_title or "").strip() or project["name"]
|
||||
job_id = jm.create_job("abc-convert")
|
||||
set_project_abc_job(project_id, body.stem_name, job_id)
|
||||
jm.submit(run_abc_conversion, job_id, midi_paths, title)
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/{project_id}/mix")
|
||||
def mix_project(project_id: str, body: MixRequest):
|
||||
"""Create a static or dynamic mix from this project's stems."""
|
||||
if body.preset_type not in ("static", "dynamic"):
|
||||
raise HTTPException(status_code=400, detail="preset_type must be 'static' or 'dynamic'")
|
||||
if body.preset_type == "static" and body.static_option and body.static_option not in VALID_STATIC_OPTIONS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown static_option: {body.static_option}")
|
||||
|
||||
project = _get_project_or_404(project_id)
|
||||
sep_job = jm.get_job(project["sep_job_id"])
|
||||
if sep_job is None:
|
||||
raise HTTPException(status_code=404, detail="Separation job not found")
|
||||
|
||||
stem_files = _stem_files_from_job(sep_job)
|
||||
if not stem_files:
|
||||
raise HTTPException(status_code=404, detail="No stem files found in separation job")
|
||||
|
||||
output_format = body.output_format or _infer_output_format(stem_files)
|
||||
ext = "mp3" if output_format.startswith("mp3") else output_format
|
||||
|
||||
# Determine output directory and filename
|
||||
project_dir = _project_output_dir(project)
|
||||
if body.preset_type == "static":
|
||||
sub = "Static"
|
||||
suffix = _static_label(body.static_option)
|
||||
else:
|
||||
sub = "Dynamic"
|
||||
suffix = "Dynamic Mix"
|
||||
|
||||
mix_dir = project_dir / sub
|
||||
mix_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_path = str(mix_dir / f"{project['name']} - {suffix}.{ext}")
|
||||
|
||||
job_id = jm.create_job("mix")
|
||||
jm.update_project(project_id, mix_job_id=job_id)
|
||||
jm.submit(
|
||||
run_mix,
|
||||
job_id,
|
||||
stem_files,
|
||||
body.preset_type,
|
||||
output_path,
|
||||
output_format,
|
||||
body.static_option,
|
||||
{str(k): float(v) for k, v in (body.dynamic_levels or {}).items()},
|
||||
)
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/{project_id}/lrc")
|
||||
def lrc_project(project_id: str, body: LrcRequest):
|
||||
"""Align lyrics with a specific stem from this project to produce an ELRC file."""
|
||||
if body.lyrics_source not in ("auto", "tag", "text"):
|
||||
raise HTTPException(status_code=400, detail=f"Invalid lyrics_source: {body.lyrics_source}")
|
||||
if body.lyrics_source == "text" and not (body.lyrics_text or "").strip():
|
||||
raise HTTPException(status_code=400, detail="lyrics_text required when lyrics_source='text'")
|
||||
|
||||
project = _get_project_or_404(project_id)
|
||||
sep_job = jm.get_job(project["sep_job_id"])
|
||||
if sep_job is None:
|
||||
raise HTTPException(status_code=404, detail="Separation job not found")
|
||||
|
||||
project_dir = _project_output_dir(project)
|
||||
|
||||
# Find the source audio file (for output filename derivation)
|
||||
source_file = None
|
||||
for f in sep_job.get("files", []):
|
||||
if f.get("type") == "source":
|
||||
source_file = f
|
||||
break
|
||||
|
||||
if source_file is None:
|
||||
raise HTTPException(status_code=404, detail="Source audio file not found in project")
|
||||
|
||||
base_name = Path(source_file["name"]).stem
|
||||
output_filename = (body.output_filename or "").strip() or f"{base_name}.elrc"
|
||||
|
||||
# Prefer the already-separated stem file to avoid re-running Demucs.
|
||||
# If the stem file exists on disk, pass it directly with is_vocals=True.
|
||||
stem_files = _stem_files_from_job(sep_job)
|
||||
stem_path = _find_stem_path(stem_files, body.stem_name)
|
||||
if stem_path and Path(stem_path).exists():
|
||||
audio_path = stem_path
|
||||
is_vocals = True
|
||||
else:
|
||||
# Stem file missing — fall back to the original mix and let the
|
||||
# alignment service extract vocals via Demucs.
|
||||
audio_path = source_file["path"]
|
||||
is_vocals = False
|
||||
|
||||
job_id = jm.create_job("lrc_align")
|
||||
set_project_lrc_job(project_id, body.stem_name, job_id)
|
||||
jm.submit_gpu(
|
||||
run_lrc_alignment,
|
||||
job_id,
|
||||
audio_path,
|
||||
lyrics_source=body.lyrics_source,
|
||||
lyrics_content=body.lyrics_text,
|
||||
output_filename=output_filename,
|
||||
style_options=body.style_options,
|
||||
is_vocals=is_vocals,
|
||||
output_dir=str(project_dir),
|
||||
timing_precision=body.timing_precision,
|
||||
lrc_power_preset=body.lrc_power_preset,
|
||||
)
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/{project_id}/lyric-transcribe")
|
||||
def lyric_transcribe_project(project_id: str, body: LyricTranscribeRequest):
|
||||
"""
|
||||
Transcribe lyrics from a vocal stem using AI (Whisper).
|
||||
|
||||
Unlike /lrc which aligns existing lyrics, this endpoint generates
|
||||
lyrics from audio using speech recognition. It produces:
|
||||
- ELRC with word-level timestamps
|
||||
- LRC with line-level timestamps
|
||||
- JSON with full transcription data
|
||||
- Plain text lyrics
|
||||
"""
|
||||
# Validate model
|
||||
if body.model not in WHISPER_MODELS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {body.model}")
|
||||
|
||||
if body.task not in ("transcribe", "translate"):
|
||||
raise HTTPException(status_code=400, detail=f"Invalid task: {body.task}")
|
||||
|
||||
project = _get_project_or_404(project_id)
|
||||
sep_job = jm.get_job(project["sep_job_id"])
|
||||
if sep_job is None:
|
||||
raise HTTPException(status_code=404, detail="Separation job not found")
|
||||
|
||||
# Find the stem file
|
||||
stem_files = _stem_files_from_job(sep_job)
|
||||
if body.stem_name not in stem_files:
|
||||
raise HTTPException(status_code=404, detail=f"Stem not found: {body.stem_name}")
|
||||
|
||||
stem_path = stem_files[body.stem_name]
|
||||
|
||||
# Create output directory
|
||||
project_dir = _project_output_dir(project)
|
||||
output_dir = project_dir / "Transcriptions"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
job_id = jm.create_job("lyric-transcribe")
|
||||
|
||||
# Store job reference keyed by stem name; same key for both engines so the
|
||||
# frontend always shows the most-recent AI transcription for that stem.
|
||||
set_project_lrc_job(project_id, f"{body.stem_name}_whisper", job_id)
|
||||
|
||||
jm.submit_gpu(
|
||||
run_lyric_transcription,
|
||||
job_id,
|
||||
stem_path,
|
||||
model=body.model,
|
||||
language=body.language,
|
||||
task=body.task,
|
||||
output_formats=body.output_formats,
|
||||
initial_prompt=body.initial_prompt,
|
||||
temperature=body.temperature,
|
||||
condition_on_previous_text=body.condition_on_previous_text,
|
||||
output_dir=str(output_dir),
|
||||
engine=body.engine,
|
||||
diarize=body.diarize,
|
||||
)
|
||||
return {"job_id": job_id, "output_dir": str(output_dir)}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stem Reprocessing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class ReprocessRequest(BaseModel):
|
||||
stem_name: str
|
||||
model: str
|
||||
output_format: str = "mp3"
|
||||
|
||||
|
||||
@router.post("/{project_id}/reprocess")
|
||||
def reprocess_stem_endpoint(project_id: str, body: ReprocessRequest):
|
||||
"""
|
||||
Reprocess an individual stem using a specialized model.
|
||||
Useful for applying per-instrument models like MDX-Net or MelBand.
|
||||
"""
|
||||
from ..config import ALL_MODEL_IDS
|
||||
from ..services.separator_service import run_separation
|
||||
|
||||
if body.model not in ALL_MODEL_IDS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {body.model}")
|
||||
|
||||
project = _get_project_or_404(project_id)
|
||||
sep_job = jm.get_job(project["sep_job_id"])
|
||||
if sep_job is None:
|
||||
raise HTTPException(status_code=404, detail="Separation job not found")
|
||||
|
||||
stem_files = _stem_files_from_job(sep_job)
|
||||
stem_path = _find_stem_path(stem_files, body.stem_name)
|
||||
if stem_path is None:
|
||||
raise HTTPException(status_code=404, detail=f"Stem not found: {body.stem_name}")
|
||||
|
||||
# Determine the original project's Stems directory from the stem file path.
|
||||
# stem_path is e.g. /app/media/projects/Artist - Title/Stems/vocals.mp3
|
||||
stems_dir = Path(stem_path).parent
|
||||
|
||||
job_id = jm.create_job("separate")
|
||||
jm.submit_gpu(
|
||||
run_separation,
|
||||
job_id,
|
||||
stem_path,
|
||||
body.model,
|
||||
body.output_format,
|
||||
preset_type="stems_only",
|
||||
output_dir_override=stems_dir,
|
||||
parent_job_id=project["sep_job_id"],
|
||||
)
|
||||
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stem Renaming
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class RenameStemRequest(BaseModel):
|
||||
stem_name: str
|
||||
new_name: str
|
||||
|
||||
|
||||
@router.post("/{project_id}/rename-stem")
|
||||
def rename_stem_endpoint(project_id: str, body: RenameStemRequest):
|
||||
"""Rename a stem file and update the job record."""
|
||||
import os
|
||||
|
||||
project = _get_project_or_404(project_id)
|
||||
sep_job = jm.get_job(project["sep_job_id"])
|
||||
if sep_job is None:
|
||||
raise HTTPException(status_code=404, detail="Separation job not found")
|
||||
|
||||
# Find the stem file
|
||||
stem_files = sep_job.get("files", [])
|
||||
stem_file = None
|
||||
stem_idx = None
|
||||
|
||||
for idx, f in enumerate(stem_files):
|
||||
if f.get("stem") == body.stem_name or f.get("name") == body.stem_name:
|
||||
stem_file = f
|
||||
stem_idx = idx
|
||||
break
|
||||
|
||||
if stem_file is None:
|
||||
raise HTTPException(status_code=404, detail=f"Stem not found: {body.stem_name}")
|
||||
|
||||
old_path = Path(stem_file["path"])
|
||||
if not old_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Stem file not found on disk: {old_path}")
|
||||
|
||||
# Generate new filename
|
||||
new_name = body.new_name.strip()
|
||||
if not new_name:
|
||||
raise HTTPException(status_code=400, detail="New name cannot be empty")
|
||||
|
||||
# Keep the same extension
|
||||
new_filename = f"{new_name}{old_path.suffix}"
|
||||
new_path = old_path.parent / new_filename
|
||||
|
||||
# Avoid overwriting existing files
|
||||
if new_path.exists() and new_path != old_path:
|
||||
raise HTTPException(status_code=409, detail=f"A file with name '{new_filename}' already exists")
|
||||
|
||||
try:
|
||||
# Rename the file on disk
|
||||
os.rename(str(old_path), str(new_path))
|
||||
|
||||
# Update the job record
|
||||
stem_files[stem_idx]["name"] = new_filename
|
||||
stem_files[stem_idx]["stem"] = new_name
|
||||
stem_files[stem_idx]["path"] = str(new_path)
|
||||
|
||||
jm.update_job(project["sep_job_id"], files=stem_files)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"old_name": body.stem_name,
|
||||
"new_name": new_name,
|
||||
"new_path": str(new_path),
|
||||
}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Failed to rename stem: {e}")
|
||||
|
||||
|
||||
def _static_label(static_option: Optional[str]) -> str:
|
||||
labels = {
|
||||
"no_vocals": "No Vocals",
|
||||
"no_drums": "No Drums",
|
||||
"no_bass": "No Bass",
|
||||
"no_vocals_no_bass": "No Vocals No Bass",
|
||||
"no_drums_no_bass": "No Drums No Bass",
|
||||
}
|
||||
return labels.get(static_option or "", "Static Mix")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cleanup / maintenance endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/cleanup/verify")
|
||||
def cleanup_verify():
|
||||
"""
|
||||
Check every project's stem files are actually on disk.
|
||||
Returns a list of projects with missing files — no mutations performed.
|
||||
"""
|
||||
projects = jm.list_projects()
|
||||
results = []
|
||||
for proj in projects:
|
||||
sep_job = jm.get_job(proj["sep_job_id"]) or {}
|
||||
files = sep_job.get("files", [])
|
||||
missing = [
|
||||
f["path"]
|
||||
for f in files
|
||||
if f.get("type") == "stem" and not Path(f.get("path", "")).exists()
|
||||
]
|
||||
if missing:
|
||||
results.append({
|
||||
"project_id": proj["id"],
|
||||
"name": proj["name"],
|
||||
"missing_files": missing,
|
||||
})
|
||||
return {"projects_with_missing_files": results, "total": len(results)}
|
||||
|
||||
|
||||
@router.post("/cleanup/write-metadata")
|
||||
def cleanup_write_metadata():
|
||||
"""
|
||||
Write/refresh project.json for every project in the database.
|
||||
Uses current DB state so trans_jobs, lrc_jobs etc. are included.
|
||||
"""
|
||||
projects = jm.list_projects()
|
||||
written = 0
|
||||
errors = []
|
||||
for proj in projects:
|
||||
try:
|
||||
project_dir = _project_output_dir(proj)
|
||||
if not project_dir.exists():
|
||||
continue
|
||||
|
||||
sep_job = jm.get_job(proj["sep_job_id"]) or {}
|
||||
files = sep_job.get("files", [])
|
||||
stem_files = {
|
||||
f["stem"]: f["path"]
|
||||
for f in files
|
||||
if f.get("type") == "stem" and f.get("stem")
|
||||
}
|
||||
source_file = next(
|
||||
(f["path"] for f in files if f.get("type") == "source"), ""
|
||||
)
|
||||
result = sep_job.get("result") or {}
|
||||
|
||||
write_project_metadata(
|
||||
project_dir,
|
||||
project_id=proj["id"],
|
||||
name=proj["name"],
|
||||
model=result.get("model_label", ""),
|
||||
model_label=result.get("model_label", ""),
|
||||
format=result.get("format_label", ""),
|
||||
format_label=result.get("format_label", ""),
|
||||
source_file=source_file,
|
||||
stems=stem_files,
|
||||
sep_job_id=proj["sep_job_id"],
|
||||
trans_jobs=proj.get("trans_jobs") or {},
|
||||
abc_jobs=proj.get("abc_jobs") or {},
|
||||
lrc_jobs=proj.get("lrc_jobs") or {},
|
||||
mix_job_id=proj.get("mix_job_id"),
|
||||
created_at=proj.get("created_at"),
|
||||
)
|
||||
written += 1
|
||||
except Exception as e:
|
||||
errors.append({"project": proj.get("name", proj["id"]), "error": str(e)})
|
||||
|
||||
return {"written": written, "errors": errors}
|
||||
|
||||
|
||||
@router.post("/cleanup/rename-stems")
|
||||
def cleanup_rename_stems():
|
||||
"""
|
||||
Rename all stem files that still use the old 'filename - Vocals.ext' format
|
||||
to the new 'Artist - Title (vocals).ext' format.
|
||||
Updates the DB job file records after each rename.
|
||||
"""
|
||||
from ..services.separator_service import _stem_filename
|
||||
|
||||
projects = jm.list_projects()
|
||||
renamed_total = 0
|
||||
errors = []
|
||||
|
||||
for proj in projects:
|
||||
sep_job = jm.get_job(proj["sep_job_id"])
|
||||
if not sep_job:
|
||||
continue
|
||||
|
||||
files = list(sep_job.get("files", []))
|
||||
proj_name = proj["name"]
|
||||
changed = False
|
||||
|
||||
for i, f in enumerate(files):
|
||||
if f.get("type") != "stem" or not f.get("stem"):
|
||||
continue
|
||||
|
||||
old_path = Path(f["path"])
|
||||
if not old_path.exists():
|
||||
continue
|
||||
|
||||
# Build the expected new name and skip if already correct
|
||||
new_filename = _stem_filename(proj_name, f["stem"], old_path.suffix)
|
||||
new_path = old_path.parent / new_filename
|
||||
if old_path == new_path:
|
||||
continue
|
||||
|
||||
try:
|
||||
old_path.rename(new_path)
|
||||
files[i] = {**f, "name": new_filename, "path": str(new_path)}
|
||||
changed = True
|
||||
renamed_total += 1
|
||||
except Exception as e:
|
||||
errors.append({
|
||||
"project": proj_name,
|
||||
"file": str(old_path),
|
||||
"error": str(e),
|
||||
})
|
||||
|
||||
if changed:
|
||||
jm.update_job(proj["sep_job_id"], files=files)
|
||||
|
||||
return {"renamed": renamed_total, "errors": errors}
|
||||
|
||||
|
||||
@router.post("/cleanup/purge-orphaned")
|
||||
def cleanup_purge_orphaned():
|
||||
"""
|
||||
Delete separation job rows (job_type='separate') that have no matching
|
||||
ProjectRecord. Also deletes jobs of any type whose parent project is gone.
|
||||
"""
|
||||
project_job_ids = {
|
||||
proj["sep_job_id"]
|
||||
for proj in jm.list_projects()
|
||||
if proj.get("sep_job_id")
|
||||
}
|
||||
all_jobs = jm.list_all_jobs()
|
||||
purged = 0
|
||||
for job in all_jobs:
|
||||
if job.get("job_type") == "separate" and job["id"] not in project_job_ids:
|
||||
jm.delete_job(job["id"])
|
||||
purged += 1
|
||||
|
||||
return {"purged": purged}
|
||||
|
||||
|
||||
@router.post("/cleanup/clean-lrc-files")
|
||||
def cleanup_clean_lrc_files():
|
||||
"""
|
||||
Scan every LRC/ELRC output file in all lrc jobs and strip section-marker
|
||||
lines (e.g. [01:30.00][Chorus]). Returns counts of files and lines affected.
|
||||
"""
|
||||
all_jobs = jm.list_all_jobs()
|
||||
files_modified = 0
|
||||
lines_removed = 0
|
||||
errors = []
|
||||
|
||||
for job in all_jobs:
|
||||
if job.get("job_type") != "lrc":
|
||||
continue
|
||||
for file_entry in (job.get("files") or []):
|
||||
ftype = file_entry.get("type", "")
|
||||
if ftype not in ("elrc", "lrc"):
|
||||
continue
|
||||
fpath = file_entry.get("path")
|
||||
if not fpath:
|
||||
continue
|
||||
p = Path(fpath)
|
||||
if not p.exists():
|
||||
continue
|
||||
try:
|
||||
content = p.read_text(encoding="utf-8")
|
||||
cleaned, removed = clean_lrc_content(content)
|
||||
if removed > 0:
|
||||
p.write_text(cleaned, encoding="utf-8")
|
||||
files_modified += 1
|
||||
lines_removed += removed
|
||||
except Exception as exc:
|
||||
errors.append({"file": fpath, "error": str(exc)})
|
||||
|
||||
return {
|
||||
"files_modified": files_modified,
|
||||
"lines_removed": lines_removed,
|
||||
"errors": errors,
|
||||
}
|
||||
Executable
+264
@@ -0,0 +1,264 @@
|
||||
"""Stem separation endpoint with preset support."""
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, File, Form, UploadFile, HTTPException
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import (
|
||||
UPLOAD_DIR, ALL_MODELS, ALL_MODEL_IDS, OUTPUT_FORMATS,
|
||||
DEFAULT_MODEL, DEFAULT_OUTPUT_FORMAT, PRESET_TYPES, STATIC_OPTIONS,
|
||||
IMPORTS_DIR,
|
||||
)
|
||||
from ..services.separator_service import run_separation
|
||||
|
||||
router = APIRouter(prefix="/api/separate", tags=["separate"])
|
||||
|
||||
VALID_FORMATS = {f["id"] for f in OUTPUT_FORMATS}
|
||||
VALID_PRESETS = {p["id"] for p in PRESET_TYPES}
|
||||
VALID_STATIC_OPTIONS = {o["id"] for o in STATIC_OPTIONS}
|
||||
|
||||
|
||||
@router.get("/models")
|
||||
def get_models():
|
||||
return ALL_MODELS
|
||||
|
||||
|
||||
@router.get("/formats")
|
||||
def get_formats():
|
||||
return OUTPUT_FORMATS
|
||||
|
||||
|
||||
@router.get("/presets")
|
||||
def get_presets():
|
||||
"""Get available preset types and options."""
|
||||
return {
|
||||
"preset_types": PRESET_TYPES,
|
||||
"static_options": STATIC_OPTIONS,
|
||||
}
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def separate(
|
||||
file: UploadFile = File(...),
|
||||
model: str = Form(DEFAULT_MODEL),
|
||||
output_format: str = Form(DEFAULT_OUTPUT_FORMAT),
|
||||
# Preset options
|
||||
preset_type: str = Form("stems_only"),
|
||||
static_option: Optional[str] = Form(None),
|
||||
dynamic_levels: Optional[str] = Form(None), # JSON string: {"vocals": 1.0, ...}
|
||||
# Project naming
|
||||
project_name: Optional[str] = Form(None),
|
||||
# Power preset
|
||||
sep_power_preset: str = Form("standard"),
|
||||
):
|
||||
"""
|
||||
Separate audio into stems with optional preset mixing.
|
||||
|
||||
Args:
|
||||
file: Audio file to process
|
||||
model: Separation model ID
|
||||
output_format: Output audio format
|
||||
preset_type: Type of preset mix to create (stems_only, instrumental, static, dynamic)
|
||||
static_option: For static preset, which stems to remove
|
||||
dynamic_levels: For dynamic preset, JSON dict of stem name to volume multiplier (0.0-2.0)
|
||||
project_name: Override auto-generated project name
|
||||
"""
|
||||
# Validate inputs
|
||||
if model not in ALL_MODEL_IDS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {model}")
|
||||
if output_format not in VALID_FORMATS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown format: {output_format}")
|
||||
if preset_type not in VALID_PRESETS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown preset type: {preset_type}")
|
||||
|
||||
if preset_type == 'static' and static_option and static_option not in VALID_STATIC_OPTIONS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown static option: {static_option}")
|
||||
|
||||
# Parse dynamic levels JSON
|
||||
parsed_dynamic_levels = None
|
||||
if dynamic_levels:
|
||||
try:
|
||||
parsed_dynamic_levels = json.loads(dynamic_levels)
|
||||
except json.JSONDecodeError:
|
||||
raise HTTPException(status_code=400, detail="Invalid dynamic_levels JSON")
|
||||
|
||||
# Create job
|
||||
job_id = jm.create_job("separate")
|
||||
upload_dir = UPLOAD_DIR / job_id
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Save uploaded file
|
||||
upload_path = upload_dir / file.filename
|
||||
with open(upload_path, "wb") as f:
|
||||
shutil.copyfileobj(file.file, f)
|
||||
|
||||
# Submit job with all options
|
||||
jm.submit_gpu(
|
||||
run_separation,
|
||||
job_id,
|
||||
str(upload_path),
|
||||
model,
|
||||
output_format,
|
||||
preset_type=preset_type,
|
||||
static_option=static_option,
|
||||
dynamic_levels=parsed_dynamic_levels,
|
||||
project_name_override=project_name,
|
||||
sep_power_preset=sep_power_preset,
|
||||
)
|
||||
|
||||
return {
|
||||
"job_id": job_id,
|
||||
"preset_type": preset_type,
|
||||
"project_name": project_name,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/existing")
|
||||
async def separate_existing(
|
||||
project_name: str = Form(..., description="Project name to process"),
|
||||
model: str = Form(DEFAULT_MODEL),
|
||||
output_format: str = Form(DEFAULT_OUTPUT_FORMAT),
|
||||
preset_type: str = Form("stems_only"),
|
||||
static_option: Optional[str] = Form(None),
|
||||
dynamic_levels: Optional[str] = Form(None),
|
||||
sep_power_preset: str = Form("standard"),
|
||||
):
|
||||
"""
|
||||
Re-run separation on an existing project (if source audio exists).
|
||||
"""
|
||||
from ..config import PROJECTS_DIR
|
||||
|
||||
if model not in ALL_MODEL_IDS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {model}")
|
||||
if output_format not in VALID_FORMATS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown format: {output_format}")
|
||||
if preset_type not in VALID_PRESETS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown preset type: {preset_type}")
|
||||
|
||||
project_dir = PROJECTS_DIR / project_name
|
||||
if not project_dir.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Project not found: {project_name}")
|
||||
|
||||
# Look for source audio file
|
||||
source_file = None
|
||||
for ext in ['.mp3', '.flac', '.wav', '.m4a', '.ogg']:
|
||||
candidate = project_dir / f"{project_name}{ext}"
|
||||
if candidate.exists():
|
||||
source_file = candidate
|
||||
break
|
||||
# Check for any file with source-like name
|
||||
for f in project_dir.iterdir():
|
||||
if f.suffix.lower() == ext and 'source' in f.name.lower():
|
||||
source_file = f
|
||||
break
|
||||
if source_file:
|
||||
break
|
||||
|
||||
if not source_file:
|
||||
# Check uploads for the original file
|
||||
upload_dirs = list(UPLOAD_DIR.iterdir()) if UPLOAD_DIR.exists() else []
|
||||
for ud in sorted(upload_dirs, key=lambda x: x.stat().st_mtime, reverse=True):
|
||||
for f in ud.iterdir():
|
||||
if f.suffix.lower() in ['.mp3', '.flac', '.wav', '.m4a', '.ogg']:
|
||||
source_file = f
|
||||
break
|
||||
if source_file:
|
||||
break
|
||||
|
||||
if not source_file:
|
||||
raise HTTPException(status_code=404, detail="Source audio file not found")
|
||||
|
||||
# Parse dynamic levels
|
||||
parsed_dynamic_levels = None
|
||||
if dynamic_levels:
|
||||
try:
|
||||
parsed_dynamic_levels = json.loads(dynamic_levels)
|
||||
except json.JSONDecodeError:
|
||||
raise HTTPException(status_code=400, detail="Invalid dynamic_levels JSON")
|
||||
|
||||
# Create job
|
||||
job_id = jm.create_job("separate")
|
||||
|
||||
jm.submit(
|
||||
run_separation,
|
||||
job_id,
|
||||
str(source_file),
|
||||
model,
|
||||
output_format,
|
||||
preset_type=preset_type,
|
||||
static_option=static_option,
|
||||
dynamic_levels=parsed_dynamic_levels,
|
||||
project_name_override=project_name,
|
||||
sep_power_preset=sep_power_preset,
|
||||
)
|
||||
|
||||
return {
|
||||
"job_id": job_id,
|
||||
"preset_type": preset_type,
|
||||
"project_name": project_name,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/from-path")
|
||||
async def separate_from_path(
|
||||
file_path: str = Form(...),
|
||||
model: str = Form(DEFAULT_MODEL),
|
||||
output_format: str = Form(DEFAULT_OUTPUT_FORMAT),
|
||||
preset_type: str = Form("stems_only"),
|
||||
static_option: Optional[str] = Form(None),
|
||||
dynamic_levels: Optional[str] = Form(None),
|
||||
project_name: Optional[str] = Form(None),
|
||||
sep_power_preset: str = Form("standard"),
|
||||
):
|
||||
"""
|
||||
Separate a server-side audio file (e.g., imported from Subsonic).
|
||||
Only files inside IMPORTS_DIR are accepted for security.
|
||||
"""
|
||||
if model not in ALL_MODEL_IDS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {model}")
|
||||
if output_format not in VALID_FORMATS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown format: {output_format}")
|
||||
if preset_type not in VALID_PRESETS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown preset type: {preset_type}")
|
||||
|
||||
source = Path(file_path)
|
||||
if not source.exists() or not source.is_file():
|
||||
raise HTTPException(status_code=404, detail="File not found")
|
||||
|
||||
# Security: reject paths that escape IMPORTS_DIR
|
||||
try:
|
||||
source.resolve().relative_to(IMPORTS_DIR.resolve())
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=403, detail="File is not in the imports directory")
|
||||
|
||||
parsed_dynamic_levels = None
|
||||
if dynamic_levels:
|
||||
try:
|
||||
parsed_dynamic_levels = json.loads(dynamic_levels)
|
||||
except json.JSONDecodeError:
|
||||
raise HTTPException(status_code=400, detail="Invalid dynamic_levels JSON")
|
||||
|
||||
job_id = jm.create_job("separate")
|
||||
jm.submit_gpu(
|
||||
run_separation,
|
||||
job_id,
|
||||
str(source),
|
||||
model,
|
||||
output_format,
|
||||
preset_type=preset_type,
|
||||
static_option=static_option,
|
||||
dynamic_levels=parsed_dynamic_levels,
|
||||
project_name_override=project_name or source.stem,
|
||||
sep_power_preset=sep_power_preset,
|
||||
)
|
||||
|
||||
return {"job_id": job_id, "preset_type": preset_type}
|
||||
|
||||
|
||||
@router.get("/per-instrument-models")
|
||||
def get_per_instrument_models():
|
||||
"""Get all separation models available for per-stem reprocessing."""
|
||||
return ALL_MODELS
|
||||
Executable
+111
@@ -0,0 +1,111 @@
|
||||
"""Subsonic server integration — search, import, and config management."""
|
||||
import json
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ..config import SUBSONIC_CONFIG_PATH
|
||||
from ..services import subsonic_service
|
||||
|
||||
router = APIRouter(prefix="/api/subsonic", tags=["subsonic"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class SubsonicConfigIn(BaseModel):
|
||||
url: str
|
||||
username: str
|
||||
password: str
|
||||
|
||||
|
||||
@router.get("/config")
|
||||
def get_config():
|
||||
"""Return connection status (never exposes the password)."""
|
||||
if not SUBSONIC_CONFIG_PATH.exists():
|
||||
return {"configured": False}
|
||||
try:
|
||||
cfg = json.loads(SUBSONIC_CONFIG_PATH.read_text())
|
||||
return {
|
||||
"configured": True,
|
||||
"url": cfg.get("url", ""),
|
||||
"username": cfg.get("username", ""),
|
||||
}
|
||||
except Exception:
|
||||
return {"configured": False}
|
||||
|
||||
|
||||
@router.post("/config")
|
||||
def save_config(body: SubsonicConfigIn):
|
||||
"""Validate credentials against the server, then persist them."""
|
||||
cfg = {"url": body.url, "username": body.username, "password": body.password}
|
||||
if not subsonic_service.test_connection(cfg):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Could not connect to Subsonic server — check URL and credentials.",
|
||||
)
|
||||
subsonic_service.save_config(cfg)
|
||||
return {"configured": True, "url": body.url, "username": body.username}
|
||||
|
||||
|
||||
@router.delete("/config")
|
||||
def remove_config():
|
||||
"""Remove stored Subsonic credentials."""
|
||||
subsonic_service.delete_config()
|
||||
return {"configured": False}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Search & Import
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/search")
|
||||
def search(q: str = "", count: int = 20):
|
||||
"""Search songs on the configured Subsonic server."""
|
||||
if not q.strip():
|
||||
return []
|
||||
try:
|
||||
return subsonic_service.search_songs(q.strip(), count)
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=502, detail=f"Subsonic search failed: {exc}")
|
||||
|
||||
|
||||
class ImportRequest(BaseModel):
|
||||
id: str
|
||||
title: str
|
||||
artist: str = ""
|
||||
suffix: str = "mp3"
|
||||
|
||||
|
||||
@router.post("/import")
|
||||
def import_song(req: ImportRequest):
|
||||
"""Download a Subsonic song to the local imports directory."""
|
||||
try:
|
||||
return subsonic_service.import_song(req.id, req.title, req.artist, req.suffix)
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=502, detail=f"Import failed: {exc}")
|
||||
|
||||
|
||||
@router.get("/imports")
|
||||
def list_imports():
|
||||
"""List all locally imported audio files."""
|
||||
return subsonic_service.list_imports()
|
||||
|
||||
|
||||
@router.delete("/imports/{filename}")
|
||||
def delete_import(filename: str):
|
||||
"""Delete an imported file from the imports directory."""
|
||||
from ..config import IMPORTS_DIR
|
||||
# Security: reject any path traversal
|
||||
if ".." in filename or "/" in filename or "\\" in filename:
|
||||
raise HTTPException(status_code=400, detail="Invalid filename")
|
||||
target = IMPORTS_DIR / filename
|
||||
if not target.exists():
|
||||
raise HTTPException(status_code=404, detail="File not found")
|
||||
target.unlink()
|
||||
return {"detail": f"Deleted {filename}"}
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
"""System information endpoints (GPU hardware)."""
|
||||
from fastapi import APIRouter
|
||||
|
||||
router = APIRouter(prefix="/api/system", tags=["system"])
|
||||
|
||||
|
||||
@router.post("/gpu-reset")
|
||||
def gpu_reset():
|
||||
"""
|
||||
Re-enable GPU acceleration for all services without restarting the container.
|
||||
|
||||
After a CUDA OOM or 'unknown error', the lyric transcriber marks the GPU
|
||||
context as failed and falls back to CPU. The separator may also fail with
|
||||
a CUDA context error. Calling this endpoint clears the CUDA allocation
|
||||
cache and resets the Whisper device flag, so both services can retry on GPU.
|
||||
|
||||
NOTE: if the CUDA context is truly corrupted (persistent 'unknown error'),
|
||||
a container restart is still required.
|
||||
"""
|
||||
from ..services.lyric_transcriber_service import reset_cuda
|
||||
return reset_cuda()
|
||||
|
||||
|
||||
@router.get("/gpu-info")
|
||||
def gpu_info():
|
||||
"""Return GPU name, VRAM figures, and cuda_ok health flag."""
|
||||
try:
|
||||
from ..services.lyric_transcriber_service import _cuda_ok as whisper_ok
|
||||
except Exception:
|
||||
whisper_ok = True
|
||||
|
||||
try:
|
||||
import torch # type: ignore
|
||||
if not torch.cuda.is_available():
|
||||
return {"cuda_available": False, "cuda_ok": whisper_ok, "name": None, "total_mb": 0, "free_mb": 0}
|
||||
props = torch.cuda.get_device_properties(0)
|
||||
free_bytes, total_bytes = torch.cuda.mem_get_info(0)
|
||||
return {
|
||||
"cuda_available": True,
|
||||
"cuda_ok": whisper_ok,
|
||||
"name": props.name,
|
||||
"total_mb": total_bytes // (1024 * 1024),
|
||||
"free_mb": free_bytes // (1024 * 1024),
|
||||
}
|
||||
except Exception as exc: # pragma: no cover
|
||||
return {"cuda_available": False, "cuda_ok": False, "name": None, "total_mb": 0, "free_mb": 0, "error": str(exc)}
|
||||
Executable
+88
@@ -0,0 +1,88 @@
|
||||
"""MIDI transcription endpoint."""
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, File, Form, UploadFile, HTTPException
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import UPLOAD_DIR, PROJECTS_DIR, TRANSCRIBE_MODES, DEFAULT_TRANSCRIBE_MODE
|
||||
from ..services.transcriber_service import run_transcription
|
||||
|
||||
router = APIRouter(prefix="/api/transcribe", tags=["transcribe"])
|
||||
|
||||
VALID_MODES = {m["id"] for m in TRANSCRIBE_MODES}
|
||||
|
||||
# Preferred stem order for transcription (drums are skipped; vocals last since
|
||||
# Omnizart's "music" mode is most useful on pitched instruments).
|
||||
_STEM_PREFERENCE = ["other", "piano", "guitar", "bass", "vocals"]
|
||||
_AUDIO_EXTS = {".mp3", ".wav", ".flac"}
|
||||
|
||||
|
||||
@router.get("/modes")
|
||||
def get_modes():
|
||||
return TRANSCRIBE_MODES
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def transcribe(
|
||||
file: UploadFile = File(...),
|
||||
mode: str = Form(DEFAULT_TRANSCRIBE_MODE),
|
||||
):
|
||||
if mode not in VALID_MODES:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown mode: {mode}")
|
||||
|
||||
job_id = jm.create_job("transcribe")
|
||||
upload_path = UPLOAD_DIR / job_id / file.filename
|
||||
upload_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(upload_path, "wb") as f:
|
||||
shutil.copyfileobj(file.file, f)
|
||||
|
||||
jm.submit_gpu(run_transcription, job_id, str(upload_path), mode)
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/from-project")
|
||||
async def transcribe_from_project(
|
||||
project_name: str = Form(...),
|
||||
mode: str = Form(DEFAULT_TRANSCRIBE_MODE),
|
||||
):
|
||||
"""
|
||||
Transcribe a stem from an existing separation project.
|
||||
Picks the best non-drum stem (other > piano > guitar > bass > vocals).
|
||||
"""
|
||||
if mode not in VALID_MODES:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown mode: {mode}")
|
||||
|
||||
stems_dir = PROJECTS_DIR / project_name / "Stems"
|
||||
if not stems_dir.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Stems folder not found for project: {project_name}")
|
||||
|
||||
# Find the best non-drum stem in order of preference
|
||||
candidates: dict[str, Path] = {}
|
||||
for f in stems_dir.iterdir():
|
||||
if f.is_file() and f.suffix.lower() in _AUDIO_EXTS and "drum" not in f.name.lower():
|
||||
name_lower = f.stem.lower()
|
||||
for pref in _STEM_PREFERENCE:
|
||||
if pref in name_lower:
|
||||
candidates.setdefault(pref, f)
|
||||
|
||||
stem_file: Path | None = None
|
||||
for pref in _STEM_PREFERENCE:
|
||||
if pref in candidates:
|
||||
stem_file = candidates[pref]
|
||||
break
|
||||
|
||||
# Fallback: any non-drum audio file
|
||||
if stem_file is None:
|
||||
for f in stems_dir.iterdir():
|
||||
if f.is_file() and f.suffix.lower() in _AUDIO_EXTS and "drum" not in f.name.lower():
|
||||
stem_file = f
|
||||
break
|
||||
|
||||
if stem_file is None:
|
||||
raise HTTPException(status_code=404, detail="No suitable (non-drum) stem found in project")
|
||||
|
||||
job_id = jm.create_job("transcribe")
|
||||
jm.submit_gpu(run_transcription, job_id, str(stem_file), mode)
|
||||
return {"job_id": job_id, "stem": stem_file.name}
|
||||
Executable
Whitespace-only changes.
Executable
+64
@@ -0,0 +1,64 @@
|
||||
"""ABC notation conversion endpoint."""
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
from fastapi import APIRouter, File, Form, UploadFile, HTTPException
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import UPLOAD_DIR
|
||||
from ..services.abc_service import run_abc_conversion
|
||||
|
||||
router = APIRouter(prefix="/api/abc-convert", tags=["abc-convert"])
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def abc_convert(
|
||||
files: List[UploadFile] = File(...),
|
||||
song_title: str = Form("Untitled"),
|
||||
):
|
||||
if not files:
|
||||
raise HTTPException(status_code=400, detail="At least one MIDI file required")
|
||||
|
||||
job_id = jm.create_job("abc-convert")
|
||||
upload_dir = UPLOAD_DIR / job_id
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
midi_paths = []
|
||||
for upload in files:
|
||||
dest = upload_dir / upload.filename
|
||||
with open(dest, "wb") as f:
|
||||
shutil.copyfileobj(upload.file, f)
|
||||
midi_paths.append(str(dest))
|
||||
|
||||
jm.submit(run_abc_conversion, job_id, midi_paths, song_title)
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/from-job")
|
||||
async def abc_convert_from_job(
|
||||
source_job_id: str = Form(...),
|
||||
song_title: str = Form(""),
|
||||
):
|
||||
"""
|
||||
Convert MIDI files from a completed transcription (or pipeline) job to ABC notation.
|
||||
Reuses the already-generated MIDI files on disk — no re-upload needed.
|
||||
"""
|
||||
source_job = jm.get_job(source_job_id)
|
||||
if source_job is None:
|
||||
raise HTTPException(status_code=404, detail=f"Source job not found: {source_job_id}")
|
||||
if source_job["status"] != "done":
|
||||
raise HTTPException(status_code=400, detail="Source job has not completed successfully")
|
||||
|
||||
midi_paths = [
|
||||
f["path"] for f in source_job.get("files", [])
|
||||
if Path(f.get("path", "")).suffix.lower() == ".mid"
|
||||
and Path(f.get("path", "")).exists()
|
||||
]
|
||||
if not midi_paths:
|
||||
raise HTTPException(status_code=404, detail="No accessible MIDI files found in source job")
|
||||
|
||||
title = song_title.strip() or source_job_id[:8]
|
||||
job_id = jm.create_job("abc-convert")
|
||||
jm.submit(run_abc_conversion, job_id, midi_paths, title)
|
||||
return {"job_id": job_id}
|
||||
Executable
+55
@@ -0,0 +1,55 @@
|
||||
"""Shared job status and download endpoints used by all tools."""
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from .. import job_manager as jm
|
||||
|
||||
router = APIRouter(prefix="/api/jobs", tags=["jobs"])
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_jobs():
|
||||
return jm.list_jobs()
|
||||
|
||||
|
||||
@router.get("/{job_id}")
|
||||
def get_job(job_id: str):
|
||||
job = jm.get_job(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
return job
|
||||
|
||||
|
||||
@router.delete("/{job_id}")
|
||||
def cancel_job(job_id: str):
|
||||
job = jm.get_job(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
jm.cancel_job(job_id)
|
||||
return {"detail": "Cancellation requested"}
|
||||
|
||||
|
||||
@router.get("/{job_id}/download/{filename}")
|
||||
def download_file(job_id: str, filename: str):
|
||||
job = jm.get_job(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
if job["status"] != "done":
|
||||
raise HTTPException(status_code=400, detail="Job not complete")
|
||||
|
||||
# Find the file in the job's file list
|
||||
match = next((f for f in job["files"] if f["name"] == filename), None)
|
||||
if match is None:
|
||||
raise HTTPException(status_code=404, detail="File not found")
|
||||
|
||||
file_path = Path(match["path"])
|
||||
if not file_path.exists():
|
||||
raise HTTPException(status_code=404, detail="File missing from disk")
|
||||
|
||||
return FileResponse(
|
||||
path=str(file_path),
|
||||
filename=filename,
|
||||
media_type="application/octet-stream",
|
||||
)
|
||||
Executable
+238
@@ -0,0 +1,238 @@
|
||||
"""LRC alignment endpoint for creating enhanced ELRC files with word-level timestamps."""
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, File, Form, UploadFile, HTTPException
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import UPLOAD_DIR, LRC_SYNC_DIR, PROJECTS_DIR
|
||||
from ..services.lrc_service import run_lrc_alignment
|
||||
from ..services.metadata_util import extract_lyrics_from_file
|
||||
|
||||
router = APIRouter(prefix="/api/lrc", tags=["lrc"])
|
||||
|
||||
|
||||
@router.get("/sources")
|
||||
def get_lyrics_sources():
|
||||
"""Get available lyrics source options."""
|
||||
return [
|
||||
{"id": "auto", "label": "Auto-detect (tags → LRC file)"},
|
||||
{"id": "tag", "label": "Audio file metadata tags only"},
|
||||
{"id": "file", "label": "Upload LRC/TXT file"},
|
||||
{"id": "text", "label": "Type lyrics directly"},
|
||||
]
|
||||
|
||||
|
||||
@router.post("/extract")
|
||||
async def extract_lyrics(
|
||||
file: UploadFile = File(...),
|
||||
):
|
||||
"""
|
||||
Extract lyrics from an uploaded audio file.
|
||||
Returns lyrics source info and plain text.
|
||||
"""
|
||||
temp_path = UPLOAD_DIR / "temp" / file.filename
|
||||
temp_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
with open(temp_path, "wb") as f:
|
||||
shutil.copyfileobj(file.file, f)
|
||||
|
||||
result = extract_lyrics_from_file(str(temp_path))
|
||||
|
||||
return {
|
||||
"source": result["source"],
|
||||
"synced": result["synced"],
|
||||
"lyrics": result["plain"],
|
||||
"raw": result["raw"],
|
||||
}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Failed to extract lyrics: {e}")
|
||||
finally:
|
||||
# Cleanup
|
||||
try:
|
||||
if temp_path.exists():
|
||||
temp_path.unlink()
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
@router.post("/align")
|
||||
async def align_lyrics(
|
||||
audio: UploadFile = File(..., description="Audio file with vocals"),
|
||||
lyrics_source: str = Form("auto", description="Source of lyrics: auto, tag, file, text"),
|
||||
lyrics_text: Optional[str] = Form(None, description="Lyrics text when source='text'"),
|
||||
output_filename: Optional[str] = Form(None, description="Custom output filename"),
|
||||
style_options: Optional[str] = Form(None, description="JSON string of style options for lyrics display"),
|
||||
):
|
||||
"""
|
||||
Align lyrics with audio to create ELRC file with word-level timestamps.
|
||||
|
||||
- lyrics_source='auto': Try metadata tags, then external LRC file
|
||||
- lyrics_source='tag': Only use metadata tags from audio file
|
||||
- lyrics_source='text': Use provided lyrics_text
|
||||
"""
|
||||
if lyrics_source not in ("auto", "tag", "file", "text"):
|
||||
raise HTTPException(status_code=400, detail=f"Invalid lyrics_source: {lyrics_source}")
|
||||
|
||||
if lyrics_source == "text" and not lyrics_text:
|
||||
raise HTTPException(status_code=400, detail="lyrics_text required when source='text'")
|
||||
|
||||
# Create job
|
||||
job_id = jm.create_job("lrc_align")
|
||||
upload_dir = UPLOAD_DIR / job_id
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Save audio file
|
||||
audio_path = upload_dir / audio.filename
|
||||
with open(audio_path, "wb") as f:
|
||||
shutil.copyfileobj(audio.file, f)
|
||||
|
||||
# Parse style options
|
||||
parsed_style_options = None
|
||||
if style_options:
|
||||
try:
|
||||
parsed_style_options = json.loads(style_options)
|
||||
except json.JSONDecodeError:
|
||||
raise HTTPException(status_code=400, detail="Invalid style_options JSON")
|
||||
|
||||
# Submit job
|
||||
jm.submit_gpu(
|
||||
run_lrc_alignment,
|
||||
job_id,
|
||||
str(audio_path),
|
||||
lyrics_source=lyrics_source,
|
||||
lyrics_content=lyrics_text,
|
||||
output_filename=output_filename,
|
||||
style_options=parsed_style_options,
|
||||
)
|
||||
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/align-with-file")
|
||||
async def align_lyrics_with_file(
|
||||
audio: UploadFile = File(..., description="Audio file with vocals"),
|
||||
lyrics_file: UploadFile = File(..., description="LRC or TXT file with lyrics"),
|
||||
output_filename: Optional[str] = Form(None, description="Custom output filename"),
|
||||
style_options: Optional[str] = Form(None, description="JSON string of style options for lyrics display"),
|
||||
):
|
||||
"""
|
||||
Align lyrics from uploaded file with audio to create ELRC file.
|
||||
"""
|
||||
job_id = jm.create_job("lrc_align")
|
||||
upload_dir = UPLOAD_DIR / job_id
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Save audio file
|
||||
audio_path = upload_dir / audio.filename
|
||||
with open(audio_path, "wb") as f:
|
||||
shutil.copyfileobj(audio.file, f)
|
||||
|
||||
# Save lyrics file
|
||||
lyrics_path = upload_dir / lyrics_file.filename
|
||||
with open(lyrics_path, "wb") as f:
|
||||
shutil.copyfileobj(lyrics_file.file, f)
|
||||
|
||||
# Parse style options
|
||||
parsed_style_options = None
|
||||
if style_options:
|
||||
try:
|
||||
parsed_style_options = json.loads(style_options)
|
||||
except json.JSONDecodeError:
|
||||
raise HTTPException(status_code=400, detail="Invalid style_options JSON")
|
||||
|
||||
# Submit job
|
||||
jm.submit_gpu(
|
||||
run_lrc_alignment,
|
||||
job_id,
|
||||
str(audio_path),
|
||||
lyrics_source="file",
|
||||
lyrics_file_path=str(lyrics_path),
|
||||
output_filename=output_filename,
|
||||
style_options=parsed_style_options,
|
||||
)
|
||||
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/align-existing")
|
||||
async def align_lyrics_existing(
|
||||
project_name: str = Form(..., description="Project name (e.g., 'Artist - Title')"),
|
||||
lyrics_source: str = Form("auto", description="Source of lyrics: auto, tag, text"),
|
||||
lyrics_text: Optional[str] = Form(None, description="Lyrics text when source='text'"),
|
||||
output_filename: Optional[str] = Form(None, description="Custom output filename"),
|
||||
style_options: Optional[str] = Form(None, description="JSON string of style options for lyrics display"),
|
||||
):
|
||||
"""
|
||||
Align lyrics with an existing vocal file in a project.
|
||||
Looks for the vocal file in the project's Stems folder.
|
||||
"""
|
||||
project_dir = PROJECTS_DIR / project_name
|
||||
stems_dir = project_dir / "Stems"
|
||||
|
||||
if not stems_dir.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Stems folder not found for project: {project_name}")
|
||||
|
||||
# Find vocals file
|
||||
vocal_file = None
|
||||
for name in ["vocals", "vocals.mp3", "vocals.wav", "vocals.flac"]:
|
||||
for ext in ["", ".mp3", ".wav", ".flac"]:
|
||||
candidate = stems_dir / f"{name}{ext}"
|
||||
if candidate.exists():
|
||||
vocal_file = candidate
|
||||
break
|
||||
if vocal_file:
|
||||
break
|
||||
|
||||
if not vocal_file:
|
||||
# Try any file with "vocal" in the name
|
||||
for f in stems_dir.iterdir():
|
||||
if "vocal" in f.name.lower() and f.suffix in [".mp3", ".wav", ".flac"]:
|
||||
vocal_file = f
|
||||
break
|
||||
|
||||
if not vocal_file:
|
||||
raise HTTPException(status_code=404, detail="No vocals file found in Stems folder")
|
||||
|
||||
job_id = jm.create_job("lrc_align")
|
||||
|
||||
# Save ELRC/LRC directly into the project root folder so it sits alongside
|
||||
# the source audio (which is also named after the project).
|
||||
project_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Determine output filename — must match the source audio base name
|
||||
if not output_filename:
|
||||
# Check whether a source audio file exists and use its stem
|
||||
source_stem = project_name
|
||||
for ext in ['.mp3', '.flac', '.wav', '.m4a', '.ogg']:
|
||||
candidate = project_dir / f"{project_name}{ext}"
|
||||
if candidate.exists():
|
||||
source_stem = candidate.stem
|
||||
break
|
||||
output_filename = f"{source_stem}.elrc"
|
||||
|
||||
# Parse style options
|
||||
parsed_style_options = None
|
||||
if style_options:
|
||||
try:
|
||||
parsed_style_options = json.loads(style_options)
|
||||
except json.JSONDecodeError:
|
||||
raise HTTPException(status_code=400, detail="Invalid style_options JSON")
|
||||
|
||||
# Submit job — vocal file is already separated, so skip Demucs extraction
|
||||
jm.submit_gpu(
|
||||
run_lrc_alignment,
|
||||
job_id,
|
||||
str(vocal_file),
|
||||
lyrics_source=lyrics_source,
|
||||
lyrics_content=lyrics_text,
|
||||
output_filename=output_filename,
|
||||
style_options=parsed_style_options,
|
||||
is_vocals=True,
|
||||
output_dir=str(project_dir),
|
||||
)
|
||||
|
||||
return {"job_id": job_id}
|
||||
+542
@@ -0,0 +1,542 @@
|
||||
"""
|
||||
Lyric transcription endpoint - transcribe vocals to text with timestamps.
|
||||
Based on Scriberr's transcription approach using OpenAI Whisper.
|
||||
|
||||
This provides:
|
||||
- Word-level and line-level timestamps
|
||||
- Multiple output formats (LRC, ELRC, JSON, SRT, VTT, TXT)
|
||||
- Speaker diarization support (via WhisperX)
|
||||
- Multiple Whisper model sizes
|
||||
"""
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Optional, List
|
||||
|
||||
from fastapi import APIRouter, File, Form, UploadFile, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import UPLOAD_DIR, PROJECTS_DIR
|
||||
from ..services.lyric_transcriber_service import (
|
||||
run_lyric_transcription,
|
||||
run_lyric_transcription_with_diarization,
|
||||
WHISPER_MODELS,
|
||||
DEFAULT_WHISPER_MODEL,
|
||||
is_whisper_available,
|
||||
is_whisperx_available,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/lyric-transcribe", tags=["lyric-transcribe"])
|
||||
|
||||
|
||||
class LyricTranscribeRequest(BaseModel):
|
||||
"""Request body for project-based lyric transcription."""
|
||||
stem_name: str
|
||||
model: str = DEFAULT_WHISPER_MODEL
|
||||
language: Optional[str] = None
|
||||
task: str = "transcribe"
|
||||
output_formats: List[str] = ["elrc", "json", "txt"]
|
||||
initial_prompt: Optional[str] = None
|
||||
temperature: float = 0.0
|
||||
condition_on_previous_text: bool = True
|
||||
word_timestamps: bool = True
|
||||
beam_size: Optional[int] = None
|
||||
|
||||
|
||||
class LyricTranscribeWithDiarizationRequest(BaseModel):
|
||||
"""Request body for transcription with speaker diarization."""
|
||||
stem_name: str
|
||||
model: str = DEFAULT_WHISPER_MODEL
|
||||
language: Optional[str] = None
|
||||
num_speakers: Optional[int] = None
|
||||
min_speakers: Optional[int] = None
|
||||
max_speakers: Optional[int] = None
|
||||
output_formats: List[str] = ["json", "txt"]
|
||||
|
||||
|
||||
VALID_OUTPUT_FORMATS = {"lrc", "elrc", "json", "txt", "srt", "vtt"}
|
||||
VALID_TASKS = {"transcribe", "translate"}
|
||||
_AUDIO_EXTS = {".mp3", ".wav", ".flac", ".m4a", ".ogg", ".wma"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _find_vocal_file(project_dir: Path) -> Optional[Path]:
|
||||
"""Find the vocal file in a project's Stems folder."""
|
||||
stems_dir = project_dir / "Stems"
|
||||
if not stems_dir.exists():
|
||||
return None
|
||||
|
||||
# Look for common vocal file names
|
||||
vocal_patterns = ["vocals", "Vocals", "lead", "Lead", "vocal", "Vocal"]
|
||||
for pattern in vocal_patterns:
|
||||
for ext in _AUDIO_EXTS:
|
||||
candidate = stems_dir / f"{pattern}{ext}"
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
|
||||
# Fallback: any file with "vocal" in the name
|
||||
for f in stems_dir.iterdir():
|
||||
if f.is_file() and f.suffix in _AUDIO_EXTS:
|
||||
if "vocal" in f.name.lower():
|
||||
return f
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _find_stem_file(project_dir: Path, stem_name: str) -> Optional[Path]:
|
||||
"""Find a specific stem file in a project's Stems folder."""
|
||||
stems_dir = project_dir / "Stems"
|
||||
if not stems_dir.exists():
|
||||
return None
|
||||
|
||||
# Exact match
|
||||
for ext in _AUDIO_EXTS:
|
||||
candidate = stems_dir / f"{stem_name}{ext}"
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
|
||||
# Case-insensitive match
|
||||
for f in stems_dir.iterdir():
|
||||
if f.is_file() and f.suffix in _AUDIO_EXTS:
|
||||
if f.stem.lower() == stem_name.lower():
|
||||
return f
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/models")
|
||||
def get_available_models():
|
||||
"""Get list of available Whisper models."""
|
||||
return {
|
||||
"models": list(WHISPER_MODELS.values()),
|
||||
"default": DEFAULT_WHISPER_MODEL,
|
||||
"whisper_available": is_whisper_available(),
|
||||
"whisperx_available": is_whisperx_available(),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/languages")
|
||||
def get_supported_languages():
|
||||
"""Get list of languages supported by Whisper."""
|
||||
# Common languages for music/lyrics
|
||||
languages = [
|
||||
{"code": None, "name": "Auto-detect", "label": "Auto-detect language"},
|
||||
{"code": "en", "name": "English", "label": "English"},
|
||||
{"code": "zh", "name": "Chinese", "label": "Chinese (Mandarin)"},
|
||||
{"code": "de", "name": "German", "label": "German"},
|
||||
{"code": "es", "name": "Spanish", "label": "Spanish"},
|
||||
{"code": "ru", "name": "Russian", "label": "Russian"},
|
||||
{"code": "ko", "name": "Korean", "label": "Korean"},
|
||||
{"code": "fr", "name": "French", "label": "French"},
|
||||
{"code": "ja", "name": "Japanese", "label": "Japanese"},
|
||||
{"code": "pt", "name": "Portuguese", "label": "Portuguese"},
|
||||
{"code": "tr", "name": "Turkish", "label": "Turkish"},
|
||||
{"code": "pl", "name": "Polish", "label": "Polish"},
|
||||
{"code": "ca", "name": "Catalan", "label": "Catalan"},
|
||||
{"code": "nl", "name": "Dutch", "label": "Dutch"},
|
||||
{"code": "ar", "name": "Arabic", "label": "Arabic"},
|
||||
{"code": "sv", "name": "Swedish", "label": "Swedish"},
|
||||
{"code": "it", "name": "Italian", "label": "Italian"},
|
||||
{"code": "id", "name": "Indonesian", "label": "Indonesian"},
|
||||
{"code": "hi", "name": "Hindi", "label": "Hindi"},
|
||||
{"code": "fi", "name": "Finnish", "label": "Finnish"},
|
||||
{"code": "vi", "name": "Vietnamese", "label": "Vietnamese"},
|
||||
{"code": "he", "name": "Hebrew", "label": "Hebrew"},
|
||||
{"code": "uk", "name": "Ukrainian", "label": "Ukrainian"},
|
||||
{"code": "el", "name": "Greek", "label": "Greek"},
|
||||
{"code": "ms", "name": "Malay", "label": "Malay"},
|
||||
{"code": "cs", "name": "Czech", "label": "Czech"},
|
||||
{"code": "ro", "name": "Romanian", "label": "Romanian"},
|
||||
{"code": "da", "name": "Danish", "label": "Danish"},
|
||||
{"code": "hu", "name": "Hungarian", "label": "Hungarian"},
|
||||
{"code": "ta", "name": "Tamil", "label": "Tamil"},
|
||||
{"code": "no", "name": "Norwegian", "label": "Norwegian"},
|
||||
{"code": "th", "name": "Thai", "label": "Thai"},
|
||||
{"code": "ur", "name": "Urdu", "label": "Urdu"},
|
||||
{"code": "hr", "name": "Croatian", "label": "Croatian"},
|
||||
{"code": "bg", "name": "Bulgarian", "label": "Bulgarian"},
|
||||
{"code": "lt", "name": "Lithuanian", "label": "Lithuanian"},
|
||||
{"code": "la", "name": "Latin", "label": "Latin"},
|
||||
{"code": "mi", "name": "Maori", "label": "Maori"},
|
||||
{"code": "ml", "name": "Malayalam", "label": "Malayalam"},
|
||||
{"code": "cy", "name": "Welsh", "label": "Welsh"},
|
||||
{"code": "sk", "name": "Slovak", "label": "Slovak"},
|
||||
{"code": "te", "name": "Telugu", "label": "Telugu"},
|
||||
{"code": "fa", "name": "Persian", "label": "Persian"},
|
||||
{"code": "lv", "name": "Latvian", "label": "Latvian"},
|
||||
{"code": "bn", "name": "Bengali", "label": "Bengali"},
|
||||
{"code": "sr", "name": "Serbian", "label": "Serbian"},
|
||||
{"code": "az", "name": "Azerbaijani", "label": "Azerbaijani"},
|
||||
{"code": "sl", "name": "Slovenian", "label": "Slovenian"},
|
||||
{"code": "kn", "name": "Kannada", "label": "Kannada"},
|
||||
{"code": "et", "name": "Estonian", "label": "Estonian"},
|
||||
{"code": "mk", "name": "Macedonian", "label": "Macedonian"},
|
||||
{"code": "br", "name": "Breton", "label": "Breton"},
|
||||
{"code": "eu", "name": "Basque", "label": "Basque"},
|
||||
{"code": "is", "name": "Icelandic", "label": "Icelandic"},
|
||||
{"code": "hy", "name": "Armenian", "label": "Armenian"},
|
||||
{"code": "ne", "name": "Nepali", "label": "Nepali"},
|
||||
{"code": "mn", "name": "Mongolian", "label": "Mongolian"},
|
||||
{"code": "bs", "name": "Bosnian", "label": "Bosnian"},
|
||||
{"code": "kk", "name": "Kazakh", "label": "Kazakh"},
|
||||
{"code": "sq", "name": "Albanian", "label": "Albanian"},
|
||||
{"code": "sw", "name": "Swahili", "label": "Swahili"},
|
||||
{"code": "gl", "name": "Galician", "label": "Galician"},
|
||||
{"code": "mr", "name": "Marathi", "label": "Marathi"},
|
||||
{"code": "pa", "name": "Punjabi", "label": "Punjabi"},
|
||||
{"code": "si", "name": "Sinhala", "label": "Sinhala"},
|
||||
{"code": "km", "name": "Khmer", "label": "Khmer"},
|
||||
{"code": "sn", "name": "Shona", "label": "Shona"},
|
||||
{"code": "yo", "name": "Yoruba", "label": "Yoruba"},
|
||||
{"code": "so", "name": "Somali", "label": "Somali"},
|
||||
{"code": "af", "name": "Afrikaans", "label": "Afrikaans"},
|
||||
{"code": "oc", "name": "Occitan", "label": "Occitan"},
|
||||
{"code": "ka", "name": "Georgian", "label": "Georgian"},
|
||||
{"code": "be", "name": "Belarusian", "label": "Belarusian"},
|
||||
{"code": "tg", "name": "Tajik", "label": "Tajik"},
|
||||
{"code": "sd", "name": "Sindhi", "label": "Sindhi"},
|
||||
{"code": "gu", "name": "Gujarati", "label": "Gujarati"},
|
||||
{"code": "am", "name": "Amharic", "label": "Amharic"},
|
||||
{"code": "yi", "name": "Yiddish", "label": "Yiddish"},
|
||||
{"code": "lo", "name": "Lao", "label": "Lao"},
|
||||
{"code": "uz", "name": "Uzbek", "label": "Uzbek"},
|
||||
{"code": "fo", "name": "Faroese", "label": "Faroese"},
|
||||
{"code": "ht", "name": "Haitian Creole", "label": "Haitian Creole"},
|
||||
{"code": "ps", "name": "Pashto", "label": "Pashto"},
|
||||
{"code": "tk", "name": "Turkmen", "label": "Turkmen"},
|
||||
{"code": "nn", "name": "Nynorsk", "label": "Nynorsk"},
|
||||
{"code": "mt", "name": "Maltese", "label": "Maltese"},
|
||||
{"code": "sa", "name": "Sanskrit", "label": "Sanskrit"},
|
||||
{"code": "lb", "name": "Luxembourgish", "label": "Luxembourgish"},
|
||||
{"code": "my", "name": "Myanmar", "label": "Myanmar"},
|
||||
{"code": "bo", "name": "Tibetan", "label": "Tibetan"},
|
||||
{"code": "tl", "name": "Tagalog", "label": "Tagalog"},
|
||||
{"code": "mg", "name": "Malagasy", "label": "Malagasy"},
|
||||
{"code": "as", "name": "Assamese", "label": "Assamese"},
|
||||
{"code": "tt", "name": "Tatar", "label": "Tatar"},
|
||||
{"code": "haw", "name": "Hawaiian", "label": "Hawaiian"},
|
||||
{"code": "ln", "name": "Lingala", "label": "Lingala"},
|
||||
{"code": "ha", "name": "Hausa", "label": "Hausa"},
|
||||
{"code": "ba", "name": "Bashkir", "label": "Bashkir"},
|
||||
{"code": "jw", "name": "Javanese", "label": "Javanese"},
|
||||
{"code": "su", "name": "Sundanese", "label": "Sundanese"},
|
||||
]
|
||||
return {"languages": languages}
|
||||
|
||||
|
||||
@router.get("/formats")
|
||||
def get_output_formats():
|
||||
"""Get available output formats."""
|
||||
return {
|
||||
"formats": [
|
||||
{"id": "lrc", "label": "LRC (line-level timestamps)", "extension": ".lrc"},
|
||||
{"id": "elrc", "label": "ELRC (word-level timestamps)", "extension": ".elrc"},
|
||||
{"id": "json", "label": "JSON (full data)", "extension": ".json"},
|
||||
{"id": "txt", "label": "Plain Text", "extension": ".txt"},
|
||||
{"id": "srt", "label": "SRT Subtitles", "extension": ".srt"},
|
||||
{"id": "vtt", "label": "WebVTT Subtitles", "extension": ".vtt"},
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def transcribe_lyrics(
|
||||
file: UploadFile = File(...),
|
||||
model: str = Form(DEFAULT_WHISPER_MODEL),
|
||||
language: Optional[str] = Form(None),
|
||||
task: str = Form("transcribe"),
|
||||
output_formats: Optional[str] = Form("elrc,json,txt"),
|
||||
initial_prompt: Optional[str] = Form(None),
|
||||
temperature: float = Form(0.0),
|
||||
):
|
||||
"""
|
||||
Transcribe lyrics from an uploaded audio file.
|
||||
|
||||
- **file**: Audio file (vocals stem recommended for best results)
|
||||
- **model**: Whisper model to use (tiny, base, small, medium, large, large-v2, large-v3)
|
||||
- **language**: ISO-639-1 language code (auto-detect if not provided)
|
||||
- **task**: 'transcribe' for original language, 'translate' for English
|
||||
- **output_formats**: Comma-separated list: lrc, elrc, json, txt, srt, vtt
|
||||
- **initial_prompt**: Optional text to guide the model's style
|
||||
- **temperature**: Sampling temperature (0.0 = greedy, higher = more random)
|
||||
"""
|
||||
if not is_whisper_available():
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Whisper is not installed. Install with: pip install openai-whisper"
|
||||
)
|
||||
|
||||
if model not in WHISPER_MODELS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {model}")
|
||||
|
||||
if task not in VALID_TASKS:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid task: {task}")
|
||||
|
||||
# Parse output formats
|
||||
formats = [f.strip() for f in (output_formats or "json").split(",")]
|
||||
invalid_formats = set(formats) - VALID_OUTPUT_FORMATS
|
||||
if invalid_formats:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid output formats: {invalid_formats}"
|
||||
)
|
||||
|
||||
# Create job and save file
|
||||
job_id = jm.create_job("lyric-transcribe")
|
||||
upload_dir = UPLOAD_DIR / job_id
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
audio_path = upload_dir / file.filename
|
||||
with open(audio_path, "wb") as f:
|
||||
shutil.copyfileobj(file.file, f)
|
||||
|
||||
# Submit job
|
||||
jm.submit_gpu(
|
||||
run_lyric_transcription,
|
||||
job_id,
|
||||
str(audio_path),
|
||||
model=model,
|
||||
language=language,
|
||||
task=task,
|
||||
output_formats=formats,
|
||||
initial_prompt=initial_prompt,
|
||||
temperature=temperature,
|
||||
)
|
||||
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/from-project")
|
||||
async def transcribe_lyrics_from_project(
|
||||
project_name: str = Form(...),
|
||||
stem_name: Optional[str] = Form(None),
|
||||
model: str = Form(DEFAULT_WHISPER_MODEL),
|
||||
language: Optional[str] = Form(None),
|
||||
task: str = Form("transcribe"),
|
||||
output_formats: Optional[str] = Form("elrc,json,txt"),
|
||||
initial_prompt: Optional[str] = Form(None),
|
||||
temperature: float = Form(0.0),
|
||||
):
|
||||
"""
|
||||
Transcribe lyrics from a project's vocal stem.
|
||||
|
||||
- **project_name**: Name of the project
|
||||
- **stem_name**: Specific stem to use (default: auto-detect vocals)
|
||||
- Other parameters same as /api/lyric-transcribe
|
||||
"""
|
||||
if not is_whisper_available():
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Whisper is not installed. Install with: pip install openai-whisper"
|
||||
)
|
||||
|
||||
if model not in WHISPER_MODELS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {model}")
|
||||
|
||||
if task not in VALID_TASKS:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid task: {task}")
|
||||
|
||||
project_dir = PROJECTS_DIR / project_name
|
||||
if not project_dir.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Project not found: {project_name}")
|
||||
|
||||
# Find audio file
|
||||
if stem_name:
|
||||
audio_file = _find_stem_file(project_dir, stem_name)
|
||||
if not audio_file:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Stem not found: {stem_name}"
|
||||
)
|
||||
else:
|
||||
audio_file = _find_vocal_file(project_dir)
|
||||
if not audio_file:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="No vocal file found in project. Please specify stem_name."
|
||||
)
|
||||
|
||||
# Parse output formats
|
||||
formats = [f.strip() for f in (output_formats or "json").split(",")]
|
||||
invalid_formats = set(formats) - VALID_OUTPUT_FORMATS
|
||||
if invalid_formats:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid output formats: {invalid_formats}"
|
||||
)
|
||||
|
||||
# Create job
|
||||
job_id = jm.create_job("lyric-transcribe")
|
||||
|
||||
# Output to project's LRC folder
|
||||
output_dir = project_dir / "LRC"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Submit job
|
||||
jm.submit_gpu(
|
||||
run_lyric_transcription,
|
||||
job_id,
|
||||
str(audio_file),
|
||||
model=model,
|
||||
language=language,
|
||||
task=task,
|
||||
output_formats=formats,
|
||||
initial_prompt=initial_prompt,
|
||||
temperature=temperature,
|
||||
output_dir=str(output_dir),
|
||||
)
|
||||
|
||||
return {"job_id": job_id, "source": audio_file.name}
|
||||
|
||||
|
||||
@router.post("/with-diarization")
|
||||
async def transcribe_lyrics_with_diarization(
|
||||
file: UploadFile = File(...),
|
||||
model: str = Form(DEFAULT_WHISPER_MODEL),
|
||||
language: Optional[str] = Form(None),
|
||||
num_speakers: Optional[int] = Form(None),
|
||||
min_speakers: Optional[int] = Form(None),
|
||||
max_speakers: Optional[int] = Form(None),
|
||||
output_formats: Optional[str] = Form("json,txt"),
|
||||
):
|
||||
"""
|
||||
Transcribe lyrics with speaker diarization (who spoke when).
|
||||
|
||||
Requires WhisperX to be installed: pip install whisperx
|
||||
|
||||
- **file**: Audio file
|
||||
- **num_speakers**: Exact number of speakers (optional)
|
||||
- **min_speakers**: Minimum number of speakers (optional)
|
||||
- **max_speakers**: Maximum number of speakers (optional)
|
||||
"""
|
||||
if not is_whisperx_available():
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="WhisperX is not installed. Install with: pip install whisperx"
|
||||
)
|
||||
|
||||
if model not in WHISPER_MODELS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {model}")
|
||||
|
||||
# Parse output formats
|
||||
formats = [f.strip() for f in (output_formats or "json").split(",")]
|
||||
invalid_formats = set(formats) - VALID_OUTPUT_FORMATS
|
||||
if invalid_formats:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid output formats: {invalid_formats}"
|
||||
)
|
||||
|
||||
# Create job and save file
|
||||
job_id = jm.create_job("lyric-transcribe-diarize")
|
||||
upload_dir = UPLOAD_DIR / job_id
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
audio_path = upload_dir / file.filename
|
||||
with open(audio_path, "wb") as f:
|
||||
shutil.copyfileobj(file.file, f)
|
||||
|
||||
# Submit job
|
||||
jm.submit_gpu(
|
||||
run_lyric_transcription_with_diarization,
|
||||
job_id,
|
||||
str(audio_path),
|
||||
model=model,
|
||||
language=language,
|
||||
num_speakers=num_speakers,
|
||||
min_speakers=min_speakers,
|
||||
max_speakers=max_speakers,
|
||||
output_formats=formats,
|
||||
)
|
||||
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/project/{project_id}")
|
||||
def transcribe_project_lyrics(project_id: str, body: LyricTranscribeRequest):
|
||||
"""
|
||||
Transcribe lyrics from a specific stem in a project.
|
||||
Similar to transcribe_lyrics_from_project but uses project_id and JSON body.
|
||||
"""
|
||||
if not is_whisper_available():
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Whisper is not installed. Install with: pip install openai-whisper"
|
||||
)
|
||||
|
||||
if body.model not in WHISPER_MODELS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {body.model}")
|
||||
|
||||
if body.task not in VALID_TASKS:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid task: {body.task}")
|
||||
|
||||
# Get project
|
||||
project = jm.get_project(project_id)
|
||||
if not project:
|
||||
raise HTTPException(status_code=404, detail=f"Project not found: {project_id}")
|
||||
|
||||
# Get separation job to find stems
|
||||
sep_job = jm.get_job(project["sep_job_id"])
|
||||
if not sep_job:
|
||||
raise HTTPException(status_code=404, detail="Separation job not found")
|
||||
|
||||
# Find the stem file
|
||||
stem_files = {
|
||||
f["stem"]: f["path"]
|
||||
for f in sep_job.get("files", [])
|
||||
if f.get("type") == "stem" and f.get("stem")
|
||||
}
|
||||
|
||||
if body.stem_name not in stem_files:
|
||||
raise HTTPException(status_code=404, detail=f"Stem not found: {body.stem_name}")
|
||||
|
||||
audio_path = stem_files[body.stem_name]
|
||||
if not Path(audio_path).exists():
|
||||
raise HTTPException(status_code=404, detail=f"Stem file not found: {audio_path}")
|
||||
|
||||
# Validate output formats
|
||||
invalid_formats = set(body.output_formats) - VALID_OUTPUT_FORMATS
|
||||
if invalid_formats:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid output formats: {invalid_formats}"
|
||||
)
|
||||
|
||||
# Create job
|
||||
job_id = jm.create_job("lyric-transcribe")
|
||||
|
||||
# Output to project directory
|
||||
project_dir = PROJECTS_DIR / project["name"]
|
||||
output_dir = project_dir / "LRC"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Store job reference
|
||||
from ..job_manager import set_project_lrc_job
|
||||
set_project_lrc_job(project_id, body.stem_name, job_id)
|
||||
|
||||
# Submit job
|
||||
jm.submit_gpu(
|
||||
run_lyric_transcription,
|
||||
job_id,
|
||||
audio_path,
|
||||
model=body.model,
|
||||
language=body.language,
|
||||
task=body.task,
|
||||
output_formats=body.output_formats,
|
||||
initial_prompt=body.initial_prompt,
|
||||
temperature=body.temperature,
|
||||
condition_on_previous_text=body.condition_on_previous_text,
|
||||
word_timestamps=body.word_timestamps,
|
||||
beam_size=body.beam_size,
|
||||
output_dir=str(output_dir),
|
||||
)
|
||||
|
||||
return {"job_id": job_id, "project_id": project_id, "stem": body.stem_name}
|
||||
Executable
+744
@@ -0,0 +1,744 @@
|
||||
"""
|
||||
Model management router — list, download, pause, cancel model downloads.
|
||||
Models are downloaded from HuggingFace on-demand or pre-downloaded.
|
||||
"""
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException, BackgroundTasks
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ..config import (
|
||||
ALL_MODELS,
|
||||
BS_ROFORMER_MODEL_DIR,
|
||||
MEL_BAND_MODEL_DIR,
|
||||
MDXNET_MODEL_DIR,
|
||||
SCNET_MODEL_DIR,
|
||||
MEDLEYVOX_MODEL_DIR,
|
||||
APOLLO_MODEL_DIR,
|
||||
ASPIRATION_MODEL_DIR,
|
||||
BANDIT_MODEL_DIR,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/models", tags=["models"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model metadata with download info
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class ModelDownloadStatus(str, Enum):
|
||||
NOT_DOWNLOADED = "not_downloaded"
|
||||
DOWNLOADING = "downloading"
|
||||
PAUSED = "paused"
|
||||
COMPLETED = "downloaded" # matches the frontend's status === 'downloaded' checks
|
||||
ERROR = "error"
|
||||
CANCELLED = "cancelled"
|
||||
|
||||
|
||||
@dataclass
|
||||
class DownloadProgress:
|
||||
model_id: str
|
||||
status: ModelDownloadStatus
|
||||
progress_percent: float = 0.0
|
||||
bytes_downloaded: int = 0
|
||||
total_bytes: int = 0
|
||||
speed_mbps: float = 0.0
|
||||
error_message: Optional[str] = None
|
||||
|
||||
# Control flags (threading.Event so they work correctly in background threads)
|
||||
_pause_event: threading.Event = field(default_factory=threading.Event)
|
||||
_cancel_event: threading.Event = field(default_factory=threading.Event)
|
||||
_thread: Optional[threading.Thread] = None
|
||||
|
||||
def __post_init__(self):
|
||||
self._pause_event.set() # Not paused by default
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"model_id": self.model_id,
|
||||
"status": self.status.value,
|
||||
"progress_percent": round(self.progress_percent, 1),
|
||||
"bytes_downloaded": self.bytes_downloaded,
|
||||
"total_bytes": self.total_bytes,
|
||||
"speed_mbps": round(self.speed_mbps, 2),
|
||||
"error_message": self.error_message,
|
||||
}
|
||||
|
||||
|
||||
# Global download tracking
|
||||
download_progress: Dict[str, DownloadProgress] = {}
|
||||
|
||||
|
||||
# Model download metadata (approximate sizes from HuggingFace)
|
||||
MODEL_DOWNLOAD_INFO = {
|
||||
# Demucs models (downloaded automatically by the demucs library on first use via torch.hub)
|
||||
"htdemucs": {
|
||||
"size_mb": 80,
|
||||
"description": "Demucs v4 hybrid transformer model. 4-stem separation (vocals, drums, bass, other). State-of-the-art quality; downloaded automatically on first use.",
|
||||
"hf_repo": None,
|
||||
"files": [],
|
||||
},
|
||||
"htdemucs_ft": {
|
||||
"size_mb": 400,
|
||||
"description": "Fine-tuned Demucs v4 with separate per-stem models. Improved vocal and drum quality over htdemucs; downloaded automatically on first use.",
|
||||
"hf_repo": None,
|
||||
"files": [],
|
||||
},
|
||||
"mdx_extra_q": {
|
||||
"size_mb": 500,
|
||||
"description": "Demucs v3 MDX Extra Quantized — quantized for faster inference and lower memory usage. Legacy model, superseded by htdemucs; downloaded automatically on first use.",
|
||||
"hf_repo": None,
|
||||
"files": [],
|
||||
},
|
||||
"mdx": {
|
||||
"size_mb": 280,
|
||||
"description": "Demucs v3 baseline 4-stem model (vocals, drums, bass, other). Legacy predecessor to htdemucs; downloaded automatically on first use.",
|
||||
"hf_repo": None,
|
||||
"files": [],
|
||||
},
|
||||
|
||||
# BS-RoFormer standard models — all share the same SW-6 weights from jarredou/BS-ROFO-SW-Fixed.
|
||||
# The 4/5/6-stem "modes" are produced by merging the 6-stem outputs in bs_roformer_separator.py.
|
||||
"bs_roformer_4stem": {
|
||||
"size_mb": 204,
|
||||
"description": "BS-RoFormer 4-stem separation (vocals, drums, bass, other). Good balance of quality and speed.",
|
||||
"hf_repo": "jarredou/BS-ROFO-SW-Fixed",
|
||||
"files": ["BS-Rofo-SW-Fixed.ckpt", "BS-Rofo-SW-Fixed.yaml"],
|
||||
},
|
||||
"bs_roformer_5stem_guitar": {
|
||||
"size_mb": 204,
|
||||
"description": "BS-RoFormer 5-stem with guitar separation (vocals, drums, bass, guitar, other).",
|
||||
"hf_repo": "jarredou/BS-ROFO-SW-Fixed",
|
||||
"files": ["BS-Rofo-SW-Fixed.ckpt", "BS-Rofo-SW-Fixed.yaml"],
|
||||
},
|
||||
"bs_roformer_5stem_piano": {
|
||||
"size_mb": 204,
|
||||
"description": "BS-RoFormer 5-stem with piano separation (vocals, drums, bass, piano, other).",
|
||||
"hf_repo": "jarredou/BS-ROFO-SW-Fixed",
|
||||
"files": ["BS-Rofo-SW-Fixed.ckpt", "BS-Rofo-SW-Fixed.yaml"],
|
||||
},
|
||||
"bs_roformer_6stem": {
|
||||
"size_mb": 204,
|
||||
"description": "BS-RoFormer 6-stem full separation (vocals, drums, bass, guitar, piano, other).",
|
||||
"hf_repo": "jarredou/BS-ROFO-SW-Fixed",
|
||||
"files": ["BS-Rofo-SW-Fixed.ckpt", "BS-Rofo-SW-Fixed.yaml"],
|
||||
},
|
||||
"bs_roformer_chorus": {
|
||||
"size_mb": 204,
|
||||
"description": "BS-RoFormer specialized for male/female chorus separation.",
|
||||
"hf_repo": None, # No public download URL — install manually
|
||||
"files": ["model.ckpt", "config.yaml"],
|
||||
},
|
||||
"bs_roformer_anvuew": {
|
||||
"size_mb": 195,
|
||||
"description": "BS-RoFormer Anvuew variant optimized for vocal separation (SDR 12.45).",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["BS-RoFormer-Anvuew/bs_roformer_anvuew_sdr_12.45.ckpt", "BS-RoFormer-Anvuew/config.yaml"],
|
||||
},
|
||||
"bs_roformer_karaoke_anvuew": {
|
||||
"size_mb": 195,
|
||||
"description": "BS-RoFormer Karaoke Anvuew for lead/backing separation.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Karaoke-BS-RoFormer-Anvuew/karaoke_bs_roformer_anvuew.ckpt", "Karaoke-BS-RoFormer-Anvuew/karaoke_bs_roformer_anvuew.yaml"],
|
||||
},
|
||||
"bs_roformer_sw_6stem": {
|
||||
"size_mb": 180,
|
||||
"description": "BS-RoFormer SW (Swin Transformer) 6-stem variant.",
|
||||
"hf_repo": None, # No public download URL — install manually
|
||||
"files": ["model.ckpt"],
|
||||
},
|
||||
|
||||
# MelBand RoFormer models
|
||||
"mel_band_roformer_4stem": {
|
||||
"size_mb": 170,
|
||||
"description": "MelBand RoFormer 4-stem separation. Efficient alternative to BS-RoFormer.",
|
||||
"hf_repo": None, # No public download URL — install manually
|
||||
"files": ["model.ckpt", "config.yaml"],
|
||||
},
|
||||
"mel_band_vocals": {
|
||||
"size_mb": 170,
|
||||
"description": "MelBand RoFormer optimized for 2-stem vocals/instrumental separation.",
|
||||
"hf_repo": "becruily/mel-band-roformer-vocals",
|
||||
"files": ["model.ckpt", "config.yaml"],
|
||||
},
|
||||
"mel_band_vocals_anvuew": {
|
||||
"size_mb": 195,
|
||||
"description": "MelBand Vocals Anvuew variant with enhanced vocal extraction.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["BS-RoFormer-Anvuew/bs_roformer_anvuew_sdr_12.45.ckpt", "BS-RoFormer-Anvuew/config.yaml"],
|
||||
},
|
||||
"mel_band_dereverb": {
|
||||
"size_mb": 170,
|
||||
"description": "MelBand RoFormer for reverb removal - separates dry vocals from reverb.",
|
||||
"hf_repo": "anvuew/dereverb_mel_band_roformer",
|
||||
"files": ["dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt", "config.yaml"],
|
||||
},
|
||||
"mel_band_dereverb_anvuew": {
|
||||
"size_mb": 195,
|
||||
"description": "MelBand Dereverb Anvuew variant (SDR 19.17) - aggressive reverb removal.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Dereverb-MelBand-Anvuew/dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt", "Dereverb-MelBand-Anvuew/dereverb_mel_band_roformer_anvuew.yaml"],
|
||||
},
|
||||
"mel_band_dereverb_room": {
|
||||
"size_mb": 195,
|
||||
"description": "MelBand Room Dereverb for room/ambient reverb removal (SDR 13.74).",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Dereverb-Room-Anvuew/dereverb_room_anvuew_sdr_13.7432.ckpt", "Dereverb-Room-Anvuew/dereverb_room_anvuew.yaml"],
|
||||
},
|
||||
"mel_band_karaoke": {
|
||||
"size_mb": 170,
|
||||
"description": "MelBand RoFormer for karaoke - separates lead vocals from backing+instrumental.",
|
||||
"hf_repo": "becruily/mel-band-roformer-karaoke",
|
||||
"files": ["model.ckpt", "config.yaml"],
|
||||
},
|
||||
"mel_band_karaoke_gabox": {
|
||||
"size_mb": 195,
|
||||
"description": "MelBand Karaoke Gabox variant optimized for backing vocal separation.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["MelBandRoformers/bsroformers/bs_karaoke_gabox_IS.ckpt", "MelBandRoformers/bsroformers/karaoke_bs_roformer.yaml"],
|
||||
},
|
||||
"mel_band_deux_becruily": {
|
||||
"size_mb": 180,
|
||||
"description": "MelBand RoFormer Deux Becruily - specialized 2-stem vocals model.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["MelBand-Roformer-Deux-Becruily/becruily_deux.ckpt", "MelBand-Roformer-Deux-Becruily/config_deux_becruily.yaml"],
|
||||
},
|
||||
|
||||
# MDX-Net models
|
||||
"mdxnet_voc_ft": {
|
||||
"size_mb": 45,
|
||||
"description": "MDX-Net Vocals Fine-tuned - lightweight vocals model (ONNX).",
|
||||
"hf_repo": "Politrees/UVR_resources",
|
||||
"files": ["models/MDXNet/UVR-MDX-NET-Voc_FT.onnx"],
|
||||
},
|
||||
"mdxnet_inst_hq_1": {
|
||||
"size_mb": 45,
|
||||
"description": "MDX-Net Instrumental HQ 1 - focused on instrumental extraction.",
|
||||
"hf_repo": "Politrees/UVR_resources",
|
||||
"files": ["models/MDXNet/UVR-MDX-NET-Inst_HQ_1.onnx"],
|
||||
},
|
||||
"mdxnet_inst_hq_2": {
|
||||
"size_mb": 45,
|
||||
"description": "MDX-Net Instrumental HQ 2 - alternative instrumental model.",
|
||||
"hf_repo": "Politrees/UVR_resources",
|
||||
"files": ["models/MDXNet/UVR-MDX-NET-Inst_HQ_2.onnx"],
|
||||
},
|
||||
"mdxnet_karaoke": {
|
||||
"size_mb": 45,
|
||||
"description": "MDX-Net Karaoke - separates lead from backing vocals.",
|
||||
"hf_repo": "Politrees/UVR_resources",
|
||||
"files": ["models/MDXNet/UVR_MDXNET_KARA.onnx"],
|
||||
},
|
||||
"mdxnet_dereverb": {
|
||||
"size_mb": 195,
|
||||
"description": "MDX-Net Dereverb via MelBand Anvuew architecture.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Dereverb-MelBand-Anvuew/dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt"],
|
||||
},
|
||||
"mdxnet_drumsep": {
|
||||
"size_mb": 448,
|
||||
"description": "MDX23C DrumSep - specialized drum separation model.",
|
||||
"hf_repo": "Politrees/UVR_resources",
|
||||
"files": ["models/MDX23C/MDX23C-DrumSep-aufr33-jarredou.ckpt"],
|
||||
},
|
||||
|
||||
# SCNet XL models (2024-2025 state-of-the-art)
|
||||
"scnet_xl": {
|
||||
"size_mb": 350,
|
||||
"description": "SCNet XL - State-of-the-art 2-stem separation (SDR 11.93 vocals, 18.23 instrumental).",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Politrees/UVR_resources/models/SCnet/scnet_checkpoint_musdb18.ckpt", "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_xl.yaml"],
|
||||
},
|
||||
"scnet_xl_ihf": {
|
||||
"size_mb": 350,
|
||||
"description": "SCNet XL IHF variant - optimized for instrumental extraction.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Politrees/UVR_resources/models/SCnet/scnet_checkpoint_musdb18.ckpt", "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_xl.yaml"],
|
||||
},
|
||||
"scnet_large": {
|
||||
"size_mb": 200,
|
||||
"description": "SCNet Large - 4-stem variant with excellent quality.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Politrees/UVR_resources/models/SCnet/SCNet-large_starrytong_fixed.ckpt", "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_large_starrytong.yaml"],
|
||||
},
|
||||
"scnet_large_starrytong": {
|
||||
"size_mb": 200,
|
||||
"description": "SCNet Large Starrytong - fixed variant with improved stability.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Politrees/UVR_resources/models/SCnet/SCNet-large_starrytong_fixed.ckpt", "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_large_starrytong.yaml"],
|
||||
},
|
||||
|
||||
# MedleyVox models
|
||||
"medleyvox": {
|
||||
"size_mb": 170,
|
||||
"description": "MedleyVox - Multi-singer vocal separation for ensemble vocals.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["MedleyVox-MultiSinger/vocals 100.pth"],
|
||||
},
|
||||
"medleyvox_55ep": {
|
||||
"size_mb": 170,
|
||||
"description": "MedleyVox 55 Epochs - earlier training checkpoint.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["MedleyVox-MultiSinger/vocals 55 epochs.pth"],
|
||||
},
|
||||
"medleyvox_66ep": {
|
||||
"size_mb": 170,
|
||||
"description": "MedleyVox 66 Epochs - intermediate checkpoint.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["MedleyVox-MultiSinger/vocals 66.pth"],
|
||||
},
|
||||
"medleyvox_88ep": {
|
||||
"size_mb": 170,
|
||||
"description": "MedleyVox 88 Epochs - near-final checkpoint.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["MedleyVox-MultiSinger/vocals 88.pth"],
|
||||
},
|
||||
"medleyvox_100ep": {
|
||||
"size_mb": 170,
|
||||
"description": "MedleyVox 100 Epochs - final training checkpoint.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["MedleyVox-MultiSinger/vocals 100.pth"],
|
||||
},
|
||||
|
||||
# Apollo models
|
||||
"apollo_vocals": {
|
||||
"size_mb": 400,
|
||||
"description": "Apollo Vocals - Vocal enhancement and reverb removal (MSST).",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Apollo-Vocal-MSST/epoch=54-val_loss=-17.6221.ckpt", "Apollo-Vocal-MSST/config_apollo_vocals_ep_54.yaml"],
|
||||
},
|
||||
"apollo_edm": {
|
||||
"size_mb": 200,
|
||||
"description": "Apollo EDM - Optimized for EDM vocal enhancement.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Politrees/UVR_resources/models/Apollo/apollo_edm_by_essid.ckpt", "Politrees/UVR_resources/models/Apollo/apollo_edm_by_essid.yaml"],
|
||||
},
|
||||
"apollo_edm_big": {
|
||||
"size_mb": 300,
|
||||
"description": "Apollo EDM Big - Larger EDM vocal enhancement model.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Politrees/UVR_resources/models/Apollo/apollo_edm_big_by_essid.ckpt", "Politrees/UVR_resources/models/Apollo/apollo_edm_big_by_essid.yaml"],
|
||||
},
|
||||
|
||||
# Aspiration models
|
||||
"aspiration_full": {
|
||||
"size_mb": 200,
|
||||
"description": "Aspiration Removal (Full) - Aggressive breath/aspiration removal (SDR 18.98).",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Aspiration-MelBand-Sucial/aspiration_mel_band_roformer_sdr_18.9845.ckpt", "Aspiration-MelBand-Sucial/config_aspiration_mel_band_roformer.yaml"],
|
||||
},
|
||||
"aspiration_light": {
|
||||
"size_mb": 200,
|
||||
"description": "Aspiration Removal (Light) - Gentler breath removal (SDR 18.12).",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Aspiration-MelBand-Sucial/aspiration_mel_band_roformer_less_aggr_sdr_18.1201.ckpt", "Aspiration-MelBand-Sucial/config_aspiration_mel_band_roformer.yaml"],
|
||||
},
|
||||
|
||||
# Bandit models
|
||||
"bandit_v2_multi": {
|
||||
"size_mb": 250,
|
||||
"description": "Bandit v2 Multi-Language - Multi-language vocal separation.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Bandit-v2/checkpoint-multi_state_dict.ckpt", "Bandit-v2/config_dnr_bandit_v2_mus64.yaml"],
|
||||
},
|
||||
"bandit_v2_eng": {
|
||||
"size_mb": 250,
|
||||
"description": "Bandit v2 English - Optimized for English vocals.",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"files": ["Bandit-v2/checkpoint-eng_state_dict.ckpt", "Bandit-v2/config_dnr_bandit_v2_mus64.yaml"],
|
||||
},
|
||||
|
||||
# Ensemble modes (no model files — combine outputs from multiple other models)
|
||||
"ensemble_vocal_best": {
|
||||
"size_mb": 0,
|
||||
"description": "Ensemble combining multiple top vocal separators (BS-RoFormer + SCNet) for best possible vocal quality. Requires constituent models to be downloaded.",
|
||||
"hf_repo": None,
|
||||
"files": [],
|
||||
},
|
||||
"ensemble_4stem": {
|
||||
"size_mb": 0,
|
||||
"description": "4-stem ensemble combining BS-RoFormer and SCNet Large outputs for vocals, drums, bass, and other. Higher quality than any single model.",
|
||||
"hf_repo": None,
|
||||
"files": [],
|
||||
},
|
||||
"ensemble_6stem": {
|
||||
"size_mb": 0,
|
||||
"description": "Full 6-stem ensemble combining multiple models for vocals, drums, bass, guitar, piano, and other. Highest quality multi-instrument separation.",
|
||||
"hf_repo": None,
|
||||
"files": [],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# Explicit mapping from model_id to the exact directory its separator service uses.
|
||||
# Must stay in sync with the DEFAULT_MODEL_DIR / model_dir values in each *_separator.py.
|
||||
MODEL_EXACT_DIRS: Dict[str, Path] = {
|
||||
# ── BS-RoFormer standard (all share the same weights flat in the family dir) ──
|
||||
"bs_roformer_4stem": BS_ROFORMER_MODEL_DIR,
|
||||
"bs_roformer_5stem_guitar": BS_ROFORMER_MODEL_DIR,
|
||||
"bs_roformer_5stem_piano": BS_ROFORMER_MODEL_DIR,
|
||||
"bs_roformer_6stem": BS_ROFORMER_MODEL_DIR,
|
||||
# ── BS-RoFormer specialized ──
|
||||
"bs_roformer_chorus": BS_ROFORMER_MODEL_DIR / "chorus",
|
||||
"bs_roformer_sw_6stem": BS_ROFORMER_MODEL_DIR, # flat; hf_repo=None
|
||||
"bs_roformer_anvuew": MEL_BAND_MODEL_DIR / "vocals_anvuew",
|
||||
"bs_roformer_karaoke_anvuew": MEL_BAND_MODEL_DIR / "karaoke_gabox",
|
||||
# ── MelBand RoFormer ──
|
||||
"mel_band_roformer_4stem": MEL_BAND_MODEL_DIR, # flat; hf_repo=None
|
||||
"mel_band_vocals": MEL_BAND_MODEL_DIR / "vocals",
|
||||
"mel_band_vocals_anvuew": MEL_BAND_MODEL_DIR / "vocals_anvuew",
|
||||
"mel_band_dereverb": MEL_BAND_MODEL_DIR / "dereverb",
|
||||
"mel_band_dereverb_anvuew": MEL_BAND_MODEL_DIR / "dereverb_anvuew",
|
||||
"mel_band_dereverb_room": MEL_BAND_MODEL_DIR / "dereverb_room",
|
||||
"mel_band_karaoke": MEL_BAND_MODEL_DIR / "karaoke",
|
||||
"mel_band_karaoke_gabox": MEL_BAND_MODEL_DIR / "karaoke_gabox",
|
||||
"mel_band_deux_becruily": MEL_BAND_MODEL_DIR / "deux_becruily",
|
||||
# ── MDX-Net (flat family dir — separator uses basename only) ──
|
||||
"mdxnet_voc_ft": MDXNET_MODEL_DIR,
|
||||
"mdxnet_inst_hq_1": MDXNET_MODEL_DIR,
|
||||
"mdxnet_inst_hq_2": MDXNET_MODEL_DIR,
|
||||
"mdxnet_karaoke": MDXNET_MODEL_DIR,
|
||||
"mdxnet_dereverb": MEL_BAND_MODEL_DIR / "dereverb", # routed to MelBand
|
||||
"mdxnet_drumsep": MDXNET_MODEL_DIR,
|
||||
# ── SCNet (per-model subdir) ──
|
||||
"scnet_xl": SCNET_MODEL_DIR / "scnet_xl",
|
||||
"scnet_xl_ihf": SCNET_MODEL_DIR / "scnet_xl_ihf",
|
||||
"scnet_large": SCNET_MODEL_DIR / "scnet_large",
|
||||
"scnet_large_starrytong": SCNET_MODEL_DIR / "scnet_large_starrytong",
|
||||
# ── MedleyVox (per-model subdir) ──
|
||||
"medleyvox": MEDLEYVOX_MODEL_DIR / "medleyvox",
|
||||
"medleyvox_55ep": MEDLEYVOX_MODEL_DIR / "medleyvox_55ep",
|
||||
"medleyvox_66ep": MEDLEYVOX_MODEL_DIR / "medleyvox_66ep",
|
||||
"medleyvox_88ep": MEDLEYVOX_MODEL_DIR / "medleyvox_88ep",
|
||||
"medleyvox_100ep": MEDLEYVOX_MODEL_DIR / "medleyvox_100ep",
|
||||
# ── Apollo (per-model subdir) ──
|
||||
"apollo_vocals": APOLLO_MODEL_DIR / "apollo_vocals",
|
||||
"apollo_edm": APOLLO_MODEL_DIR / "apollo_edm",
|
||||
"apollo_edm_big": APOLLO_MODEL_DIR / "apollo_edm_big",
|
||||
# ── Aspiration (per-model subdir) ──
|
||||
"aspiration_full": ASPIRATION_MODEL_DIR / "aspiration_full",
|
||||
"aspiration_light": ASPIRATION_MODEL_DIR / "aspiration_light",
|
||||
# ── Bandit (per-model subdir) ──
|
||||
"bandit_v2_multi": BANDIT_MODEL_DIR / "bandit_v2_multi",
|
||||
"bandit_v2_eng": BANDIT_MODEL_DIR / "bandit_v2_eng",
|
||||
}
|
||||
|
||||
|
||||
def get_model_directory(model_id: str) -> Path:
|
||||
"""Get the exact directory where a model's files are stored on disk."""
|
||||
return MODEL_EXACT_DIRS.get(model_id, BS_ROFORMER_MODEL_DIR / model_id)
|
||||
|
||||
|
||||
def is_model_downloaded(model_id: str) -> bool:
|
||||
"""Check if a model is already downloaded."""
|
||||
if model_id not in MODEL_DOWNLOAD_INFO:
|
||||
return False
|
||||
|
||||
model_dir = get_model_directory(model_id)
|
||||
info = MODEL_DOWNLOAD_INFO[model_id]
|
||||
|
||||
for file_path in info["files"]:
|
||||
flat = model_dir / Path(file_path).name # flat download (our convention)
|
||||
deep = model_dir / file_path # subdirectory structure
|
||||
if not flat.exists() and not deep.exists():
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class ModelInfo(BaseModel):
|
||||
id: str
|
||||
label: str
|
||||
group: str
|
||||
description: str
|
||||
size_mb: int
|
||||
status: str # not_downloaded, downloaded, downloading, paused, error
|
||||
progress_percent: float = 0.0
|
||||
|
||||
|
||||
class ModelDownloadRequest(BaseModel):
|
||||
model_id: str
|
||||
|
||||
|
||||
class ModelDownloadResponse(BaseModel):
|
||||
success: bool
|
||||
message: str
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/list", response_model=List[ModelInfo])
|
||||
def list_models():
|
||||
"""List all available models with their download status."""
|
||||
result = []
|
||||
|
||||
for model in ALL_MODELS:
|
||||
model_id = model["id"]
|
||||
info = MODEL_DOWNLOAD_INFO.get(model_id, {})
|
||||
|
||||
# Check current download status
|
||||
if model_id in download_progress:
|
||||
prog = download_progress[model_id]
|
||||
status = prog.status.value
|
||||
progress = prog.progress_percent
|
||||
elif is_model_downloaded(model_id):
|
||||
status = ModelDownloadStatus.COMPLETED.value
|
||||
progress = 100.0
|
||||
else:
|
||||
status = ModelDownloadStatus.NOT_DOWNLOADED.value
|
||||
progress = 0.0
|
||||
|
||||
result.append(ModelInfo(
|
||||
id=model_id,
|
||||
label=model["label"],
|
||||
group=model.get("group", "Other"),
|
||||
description=info.get("description", "No description available."),
|
||||
size_mb=info.get("size_mb", 0),
|
||||
status=status,
|
||||
progress_percent=progress,
|
||||
))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/download", response_model=ModelDownloadResponse)
|
||||
def download_model(body: ModelDownloadRequest, background_tasks: BackgroundTasks):
|
||||
"""Start downloading a model in the background."""
|
||||
model_id = body.model_id
|
||||
|
||||
if model_id not in MODEL_DOWNLOAD_INFO:
|
||||
raise HTTPException(status_code=404, detail=f"Unknown model: {model_id}")
|
||||
|
||||
# Check if already downloaded
|
||||
if is_model_downloaded(model_id):
|
||||
return ModelDownloadResponse(success=True, message=f"Model {model_id} is already downloaded.")
|
||||
|
||||
# Check if already downloading
|
||||
if model_id in download_progress:
|
||||
prog = download_progress[model_id]
|
||||
if prog.status == ModelDownloadStatus.DOWNLOADING:
|
||||
return ModelDownloadResponse(success=True, message=f"Download already in progress for {model_id}.")
|
||||
elif prog.status == ModelDownloadStatus.PAUSED:
|
||||
# Resume
|
||||
prog._pause_event.set()
|
||||
prog.status = ModelDownloadStatus.DOWNLOADING
|
||||
return ModelDownloadResponse(success=True, message=f"Resuming download for {model_id}.")
|
||||
|
||||
# Start new download
|
||||
progress = DownloadProgress(
|
||||
model_id=model_id,
|
||||
status=ModelDownloadStatus.DOWNLOADING,
|
||||
total_bytes=MODEL_DOWNLOAD_INFO[model_id]["size_mb"] * 1024 * 1024,
|
||||
)
|
||||
download_progress[model_id] = progress
|
||||
|
||||
# Start background download
|
||||
background_tasks.add_task(_download_model_task, model_id, progress)
|
||||
|
||||
return ModelDownloadResponse(success=True, message=f"Started downloading {model_id}.")
|
||||
|
||||
|
||||
@router.post("/pause")
|
||||
def pause_download(body: ModelDownloadRequest):
|
||||
"""Pause an ongoing download."""
|
||||
model_id = body.model_id
|
||||
|
||||
if model_id not in download_progress:
|
||||
raise HTTPException(status_code=404, detail=f"No active download for {model_id}")
|
||||
|
||||
prog = download_progress[model_id]
|
||||
if prog.status != ModelDownloadStatus.DOWNLOADING:
|
||||
raise HTTPException(status_code=400, detail=f"Download is not in progress (status: {prog.status.value})")
|
||||
|
||||
prog._pause_event.clear()
|
||||
prog.status = ModelDownloadStatus.PAUSED
|
||||
|
||||
return {"success": True, "message": f"Paused download for {model_id}"}
|
||||
|
||||
|
||||
@router.post("/resume")
|
||||
def resume_download(body: ModelDownloadRequest):
|
||||
"""Resume a paused download."""
|
||||
model_id = body.model_id
|
||||
|
||||
if model_id not in download_progress:
|
||||
raise HTTPException(status_code=404, detail=f"No download to resume for {model_id}")
|
||||
|
||||
prog = download_progress[model_id]
|
||||
if prog.status != ModelDownloadStatus.PAUSED:
|
||||
raise HTTPException(status_code=400, detail=f"Download is not paused (status: {prog.status.value})")
|
||||
|
||||
prog._pause_event.set()
|
||||
prog.status = ModelDownloadStatus.DOWNLOADING
|
||||
|
||||
return {"success": True, "message": f"Resumed download for {model_id}"}
|
||||
|
||||
|
||||
@router.post("/cancel")
|
||||
def cancel_download(body: ModelDownloadRequest):
|
||||
"""Cancel an ongoing or paused download."""
|
||||
model_id = body.model_id
|
||||
|
||||
if model_id not in download_progress:
|
||||
raise HTTPException(status_code=404, detail=f"No active download for {model_id}")
|
||||
|
||||
prog = download_progress[model_id]
|
||||
|
||||
# Signal cancellation
|
||||
prog._cancel_event.set()
|
||||
prog.status = ModelDownloadStatus.CANCELLED
|
||||
|
||||
# Clean up partial downloads
|
||||
_cleanup_partial_download(model_id)
|
||||
|
||||
return {"success": True, "message": f"Cancelled download for {model_id}"}
|
||||
|
||||
|
||||
@router.get("/progress/{model_id}")
|
||||
def get_progress(model_id: str):
|
||||
"""Get download progress for a specific model."""
|
||||
if model_id not in download_progress:
|
||||
# Check if downloaded
|
||||
if is_model_downloaded(model_id):
|
||||
return {
|
||||
"model_id": model_id,
|
||||
"status": ModelDownloadStatus.COMPLETED.value,
|
||||
"progress_percent": 100.0,
|
||||
"bytes_downloaded": MODEL_DOWNLOAD_INFO.get(model_id, {}).get("size_mb", 0) * 1024 * 1024,
|
||||
"total_bytes": MODEL_DOWNLOAD_INFO.get(model_id, {}).get("size_mb", 0) * 1024 * 1024,
|
||||
"speed_mbps": 0.0,
|
||||
}
|
||||
raise HTTPException(status_code=404, detail=f"No download found for {model_id}")
|
||||
|
||||
return download_progress[model_id].to_dict()
|
||||
|
||||
|
||||
@router.get("/progress")
|
||||
def get_all_progress():
|
||||
"""Get download progress for all models."""
|
||||
return {model_id: prog.to_dict() for model_id, prog in download_progress.items()}
|
||||
|
||||
|
||||
@router.delete("/{model_id}")
|
||||
def delete_model(model_id: str):
|
||||
"""Delete a downloaded model to free up space."""
|
||||
if model_id not in MODEL_DOWNLOAD_INFO:
|
||||
raise HTTPException(status_code=404, detail=f"Unknown model: {model_id}")
|
||||
|
||||
if not is_model_downloaded(model_id):
|
||||
raise HTTPException(status_code=400, detail=f"Model {model_id} is not downloaded")
|
||||
|
||||
# Remove files
|
||||
model_dir = get_model_directory(model_id)
|
||||
info = MODEL_DOWNLOAD_INFO[model_id]
|
||||
|
||||
deleted = 0
|
||||
for file_path in info["files"]:
|
||||
# Try flat first (our download convention), then deep path
|
||||
for candidate in (model_dir / Path(file_path).name, model_dir / file_path):
|
||||
if candidate.exists():
|
||||
candidate.unlink()
|
||||
deleted += 1
|
||||
break
|
||||
|
||||
# Clean up progress tracking
|
||||
if model_id in download_progress:
|
||||
del download_progress[model_id]
|
||||
|
||||
return {"success": True, "message": f"Deleted {deleted} files for {model_id}"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Background download task
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _download_model_task(model_id: str, progress: DownloadProgress):
|
||||
"""Background task to download a model from HuggingFace."""
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
info = MODEL_DOWNLOAD_INFO[model_id]
|
||||
hf_repo = info.get("hf_repo")
|
||||
|
||||
# Reject placeholder / unconfigured repos immediately
|
||||
if not hf_repo:
|
||||
progress.status = ModelDownloadStatus.ERROR
|
||||
progress.error_message = (
|
||||
"No download URL configured for this model. "
|
||||
"Install the weights manually into the model directory."
|
||||
)
|
||||
return
|
||||
|
||||
model_dir = get_model_directory(model_id)
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
total_files = len(info["files"])
|
||||
|
||||
for idx, file_path in enumerate(info["files"]):
|
||||
# Check for cancellation
|
||||
if progress._cancel_event.is_set():
|
||||
progress.status = ModelDownloadStatus.CANCELLED
|
||||
return
|
||||
|
||||
# Block here until resumed (threading.Event.wait() works correctly in threads)
|
||||
progress._pause_event.wait()
|
||||
|
||||
if progress._cancel_event.is_set():
|
||||
progress.status = ModelDownloadStatus.CANCELLED
|
||||
return
|
||||
|
||||
dest = model_dir / Path(file_path).name
|
||||
if dest.exists():
|
||||
# Already present — count it and skip
|
||||
progress.bytes_downloaded += dest.stat().st_size
|
||||
progress.progress_percent = (idx + 1) / total_files * 100
|
||||
continue
|
||||
|
||||
try:
|
||||
# Download to a temp directory first, then move flat into model_dir.
|
||||
# This ensures is_model_downloaded() finds files at the expected path
|
||||
# regardless of the subdirectory structure inside the HF repo.
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp_path = hf_hub_download(
|
||||
repo_id=hf_repo,
|
||||
filename=file_path,
|
||||
local_dir=tmp,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
shutil.copy2(tmp_path, str(dest))
|
||||
|
||||
progress.bytes_downloaded += dest.stat().st_size
|
||||
progress.progress_percent = (idx + 1) / total_files * 100
|
||||
|
||||
except Exception as e:
|
||||
progress.status = ModelDownloadStatus.ERROR
|
||||
progress.error_message = str(e)
|
||||
return
|
||||
|
||||
progress.progress_percent = 100.0
|
||||
progress.status = ModelDownloadStatus.COMPLETED
|
||||
|
||||
except Exception as e:
|
||||
progress.status = ModelDownloadStatus.ERROR
|
||||
progress.error_message = str(e)
|
||||
|
||||
|
||||
def _cleanup_partial_download(model_id: str):
|
||||
"""Remove partially downloaded files."""
|
||||
model_dir = get_model_directory(model_id)
|
||||
if model_dir.exists():
|
||||
import shutil
|
||||
shutil.rmtree(model_dir, ignore_errors=True)
|
||||
Executable
+53
@@ -0,0 +1,53 @@
|
||||
"""Full pipeline endpoint: separate → transcribe → ABC."""
|
||||
import shutil
|
||||
|
||||
from fastapi import APIRouter, File, Form, UploadFile, HTTPException
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import (
|
||||
UPLOAD_DIR, ALL_MODEL_IDS, OUTPUT_FORMATS, TRANSCRIBE_MODES,
|
||||
DEFAULT_MODEL, DEFAULT_OUTPUT_FORMAT, DEFAULT_TRANSCRIBE_MODE,
|
||||
)
|
||||
from ..services.pipeline_service import run_pipeline
|
||||
|
||||
router = APIRouter(prefix="/api/pipeline", tags=["pipeline"])
|
||||
|
||||
VALID_FORMATS = {f["id"] for f in OUTPUT_FORMATS}
|
||||
VALID_MODES = {m["id"] for m in TRANSCRIBE_MODES}
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def pipeline(
|
||||
file: UploadFile = File(...),
|
||||
sep_model: str = Form(DEFAULT_MODEL),
|
||||
sep_format: str = Form(DEFAULT_OUTPUT_FORMAT),
|
||||
transcribe_mode: str = Form(DEFAULT_TRANSCRIBE_MODE),
|
||||
song_title: str = Form(""),
|
||||
):
|
||||
if sep_model not in ALL_MODEL_IDS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {sep_model}")
|
||||
if sep_format not in VALID_FORMATS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown format: {sep_format}")
|
||||
if transcribe_mode not in VALID_MODES:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown mode: {transcribe_mode}")
|
||||
|
||||
# Fall back to filename as song title
|
||||
effective_title = song_title.strip() or file.filename.rsplit(".", 1)[0]
|
||||
|
||||
job_id = jm.create_job("pipeline")
|
||||
upload_path = UPLOAD_DIR / job_id / file.filename
|
||||
upload_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(upload_path, "wb") as f:
|
||||
shutil.copyfileobj(file.file, f)
|
||||
|
||||
jm.submit_gpu(
|
||||
run_pipeline,
|
||||
job_id,
|
||||
str(upload_path),
|
||||
sep_model,
|
||||
sep_format,
|
||||
transcribe_mode,
|
||||
effective_title,
|
||||
)
|
||||
return {"job_id": job_id}
|
||||
Executable
+749
@@ -0,0 +1,749 @@
|
||||
"""
|
||||
Project management endpoints.
|
||||
|
||||
A Project is created automatically when a separation job completes.
|
||||
It links the separation job to downstream jobs (transcription, ABC, mix).
|
||||
"""
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import PROJECTS_DIR, UPLOAD_DIR, TRANSCRIBE_MODES, DEFAULT_TRANSCRIBE_MODE, STATIC_OPTIONS
|
||||
from ..services.project_metadata import write_project_metadata, read_project_metadata
|
||||
from ..services.transcriber_service import run_transcription
|
||||
from ..services.abc_service import run_abc_conversion
|
||||
from ..services.mix_service import run_mix
|
||||
from ..services.lrc_service import run_lrc_alignment, clean_lrc_content
|
||||
from ..services.lyric_transcriber_service import (
|
||||
run_lyric_transcription,
|
||||
WHISPER_MODELS,
|
||||
DEFAULT_WHISPER_MODEL,
|
||||
)
|
||||
from ..job_manager import set_project_trans_job, set_project_abc_job, set_project_lrc_job
|
||||
|
||||
router = APIRouter(prefix="/api/projects", tags=["projects"])
|
||||
|
||||
VALID_TRANSCRIBE_MODES = {m["id"] for m in TRANSCRIBE_MODES}
|
||||
VALID_STATIC_OPTIONS = {o["id"] for o in STATIC_OPTIONS}
|
||||
_AUDIO_EXTS = {".mp3", ".wav", ".flac"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request bodies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TranscribeRequest(BaseModel):
|
||||
stem_name: str # which stem to transcribe (key from sep job)
|
||||
mode: str = DEFAULT_TRANSCRIBE_MODE
|
||||
|
||||
|
||||
class AbcRequest(BaseModel):
|
||||
stem_name: str # which stem's MIDI to convert
|
||||
song_title: Optional[str] = None
|
||||
|
||||
|
||||
class LrcRequest(BaseModel):
|
||||
stem_name: str # which stem to align (usually vocals)
|
||||
lyrics_source: str = "text" # text | auto | tag
|
||||
lyrics_text: Optional[str] = None # required when lyrics_source='text'
|
||||
output_filename: Optional[str] = None
|
||||
style_options: Optional[dict] = None
|
||||
timing_precision: str = "word" # 'word' | 'syllable'
|
||||
lrc_power_preset: str = "standard" # 'eco' | 'standard' | 'performance'
|
||||
|
||||
|
||||
class MixRequest(BaseModel):
|
||||
preset_type: str # 'static' | 'dynamic'
|
||||
static_option: Optional[str] = None
|
||||
dynamic_levels: Optional[dict] = None # stem_name → float volume
|
||||
output_format: Optional[str] = None # inferred from stems if absent
|
||||
|
||||
|
||||
class LyricTranscribeRequest(BaseModel):
|
||||
"""Request body for AI-based lyric transcription using Whisper."""
|
||||
stem_name: str # which stem to transcribe (usually vocals)
|
||||
model: str = "large-v2" # whisper model: tiny, base, small, medium, large, large-v2, large-v3
|
||||
language: Optional[str] = None # ISO-639-1 code (auto-detect if None)
|
||||
task: str = "transcribe" # 'transcribe' or 'translate' (to English)
|
||||
output_formats: list = ["elrc", "json", "txt"] # lrc, elrc, json, txt, srt, vtt
|
||||
initial_prompt: Optional[str] = None # optional prompt to guide the model
|
||||
temperature: float = 0.0 # sampling temperature
|
||||
condition_on_previous_text: bool = False # if True, pass previous output as prompt; can cause hallucination loops
|
||||
engine: str = "whisperx" # 'whisperx' (only supported engine)
|
||||
diarize: bool = False # run speaker diarization (WhisperX only)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _get_project_or_404(project_id: str) -> dict:
|
||||
project = jm.get_project(project_id)
|
||||
if project is None:
|
||||
raise HTTPException(status_code=404, detail=f"Project not found: {project_id}")
|
||||
return project
|
||||
|
||||
|
||||
def _get_job_or_none(job_id: Optional[str]) -> Optional[dict]:
|
||||
if not job_id:
|
||||
return None
|
||||
return jm.get_job(job_id)
|
||||
|
||||
|
||||
def _stem_files_from_job(sep_job: dict) -> dict:
|
||||
"""Return {stem_name: abs_path} for all stem files in the separation job."""
|
||||
return {
|
||||
f["stem"]: f["path"]
|
||||
for f in sep_job.get("files", [])
|
||||
if f.get("type") == "stem" and f.get("stem") and Path(f.get("path", "")).exists()
|
||||
}
|
||||
|
||||
|
||||
def _infer_output_format(stem_files: dict) -> str:
|
||||
"""Infer output format from stem file extensions."""
|
||||
for path in stem_files.values():
|
||||
ext = Path(path).suffix.lower().lstrip(".")
|
||||
if ext in ("mp3", "flac", "wav"):
|
||||
return ext
|
||||
return "mp3"
|
||||
|
||||
|
||||
def _project_output_dir(project: dict) -> Path:
|
||||
"""Return the project's root directory under MEDIA_DIR."""
|
||||
sep_job = jm.get_job(project["sep_job_id"]) or {}
|
||||
result = sep_job.get("result") or {}
|
||||
project_path = result.get("project_path")
|
||||
if project_path and Path(project_path).is_dir():
|
||||
return Path(project_path)
|
||||
# Fallback: PROJECTS_DIR / project.name
|
||||
return PROJECTS_DIR / project["name"]
|
||||
|
||||
|
||||
def _find_stem_path(stem_files: dict, stem_name: str) -> Optional[str]:
|
||||
"""Return path for the named stem, matching by key or by file basename/stem."""
|
||||
if stem_name in stem_files:
|
||||
return stem_files[stem_name]
|
||||
for path in stem_files.values():
|
||||
p = Path(path)
|
||||
if p.name == stem_name or p.stem == stem_name:
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("")
|
||||
def list_projects():
|
||||
"""Return all projects, newest first. Migrates orphaned separation jobs."""
|
||||
return jm.list_projects()
|
||||
|
||||
|
||||
@router.get("/{project_id}")
|
||||
def get_project_detail(project_id: str):
|
||||
"""Return project with all linked jobs expanded."""
|
||||
project = _get_project_or_404(project_id)
|
||||
trans_jobs_expanded = {
|
||||
stem: jm.get_job(jid)
|
||||
for stem, jid in (project.get("trans_jobs") or {}).items()
|
||||
}
|
||||
abc_jobs_expanded = {
|
||||
stem: jm.get_job(jid)
|
||||
for stem, jid in (project.get("abc_jobs") or {}).items()
|
||||
}
|
||||
lrc_jobs_expanded = {
|
||||
stem: jm.get_job(jid)
|
||||
for stem, jid in (project.get("lrc_jobs") or {}).items()
|
||||
}
|
||||
return {
|
||||
**project,
|
||||
"sep_job": jm.get_job(project["sep_job_id"]),
|
||||
"trans_jobs": trans_jobs_expanded,
|
||||
"abc_jobs": abc_jobs_expanded,
|
||||
"lrc_jobs": lrc_jobs_expanded,
|
||||
"mix_job": _get_job_or_none(project["mix_job_id"]),
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/{project_id}")
|
||||
def delete_project(project_id: str):
|
||||
"""Remove the project record and all associated files from disk."""
|
||||
project = _get_project_or_404(project_id)
|
||||
|
||||
# Delete project directory from disk
|
||||
project_dir = _project_output_dir(project)
|
||||
if project_dir.exists():
|
||||
shutil.rmtree(project_dir, ignore_errors=True)
|
||||
|
||||
# Delete the upload directory for the originating separation job
|
||||
upload_dir = UPLOAD_DIR / project["sep_job_id"]
|
||||
if upload_dir.exists():
|
||||
shutil.rmtree(upload_dir, ignore_errors=True)
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
from ..job_manager import engine, ProjectRecord
|
||||
with Session(engine) as session:
|
||||
record = session.get(ProjectRecord, project_id)
|
||||
if record:
|
||||
session.delete(record)
|
||||
session.commit()
|
||||
return {"detail": "Project deleted"}
|
||||
|
||||
|
||||
@router.post("/{project_id}/transcribe")
|
||||
def transcribe_project(project_id: str, body: TranscribeRequest):
|
||||
"""Transcribe a specific stem from this project to MIDI."""
|
||||
if body.mode not in VALID_TRANSCRIBE_MODES:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown transcribe mode: {body.mode}")
|
||||
|
||||
project = _get_project_or_404(project_id)
|
||||
sep_job = jm.get_job(project["sep_job_id"])
|
||||
if sep_job is None:
|
||||
raise HTTPException(status_code=404, detail="Separation job not found")
|
||||
|
||||
stem_files = _stem_files_from_job(sep_job)
|
||||
if not stem_files:
|
||||
raise HTTPException(status_code=404, detail="No stem files found in separation job")
|
||||
|
||||
stem_path = _find_stem_path(stem_files, body.stem_name)
|
||||
if stem_path is None:
|
||||
raise HTTPException(status_code=404, detail=f"Stem not found: {body.stem_name}")
|
||||
|
||||
job_id = jm.create_job("transcribe")
|
||||
set_project_trans_job(project_id, body.stem_name, job_id)
|
||||
jm.submit_gpu(run_transcription, job_id, stem_path, body.mode)
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/{project_id}/abc")
|
||||
def abc_project(project_id: str, body: AbcRequest):
|
||||
"""Convert MIDI from a specific stem's transcription job to ABC notation."""
|
||||
project = _get_project_or_404(project_id)
|
||||
|
||||
trans_jobs_dict = project.get("trans_jobs") or {}
|
||||
trans_job_id = trans_jobs_dict.get(body.stem_name)
|
||||
if not trans_job_id:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"No transcription job for stem '{body.stem_name}'",
|
||||
)
|
||||
|
||||
trans_job = jm.get_job(trans_job_id)
|
||||
if trans_job is None:
|
||||
raise HTTPException(status_code=404, detail="Transcription job not found")
|
||||
if trans_job["status"] != "done":
|
||||
raise HTTPException(status_code=400, detail="Transcription job has not completed successfully")
|
||||
|
||||
midi_paths = [
|
||||
f["path"]
|
||||
for f in trans_job.get("files", [])
|
||||
if Path(f.get("path", "")).suffix.lower() == ".mid"
|
||||
and Path(f.get("path", "")).exists()
|
||||
]
|
||||
if not midi_paths:
|
||||
raise HTTPException(status_code=404, detail="No MIDI files found in transcription job")
|
||||
|
||||
title = (body.song_title or "").strip() or project["name"]
|
||||
job_id = jm.create_job("abc-convert")
|
||||
set_project_abc_job(project_id, body.stem_name, job_id)
|
||||
jm.submit(run_abc_conversion, job_id, midi_paths, title)
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/{project_id}/mix")
|
||||
def mix_project(project_id: str, body: MixRequest):
|
||||
"""Create a static or dynamic mix from this project's stems."""
|
||||
if body.preset_type not in ("static", "dynamic"):
|
||||
raise HTTPException(status_code=400, detail="preset_type must be 'static' or 'dynamic'")
|
||||
if body.preset_type == "static" and body.static_option and body.static_option not in VALID_STATIC_OPTIONS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown static_option: {body.static_option}")
|
||||
|
||||
project = _get_project_or_404(project_id)
|
||||
sep_job = jm.get_job(project["sep_job_id"])
|
||||
if sep_job is None:
|
||||
raise HTTPException(status_code=404, detail="Separation job not found")
|
||||
|
||||
stem_files = _stem_files_from_job(sep_job)
|
||||
if not stem_files:
|
||||
raise HTTPException(status_code=404, detail="No stem files found in separation job")
|
||||
|
||||
output_format = body.output_format or _infer_output_format(stem_files)
|
||||
ext = "mp3" if output_format.startswith("mp3") else output_format
|
||||
|
||||
# Determine output directory and filename
|
||||
project_dir = _project_output_dir(project)
|
||||
if body.preset_type == "static":
|
||||
sub = "Static"
|
||||
suffix = _static_label(body.static_option)
|
||||
else:
|
||||
sub = "Dynamic"
|
||||
suffix = "Dynamic Mix"
|
||||
|
||||
mix_dir = project_dir / sub
|
||||
mix_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_path = str(mix_dir / f"{project['name']} - {suffix}.{ext}")
|
||||
|
||||
job_id = jm.create_job("mix")
|
||||
jm.update_project(project_id, mix_job_id=job_id)
|
||||
jm.submit(
|
||||
run_mix,
|
||||
job_id,
|
||||
stem_files,
|
||||
body.preset_type,
|
||||
output_path,
|
||||
output_format,
|
||||
body.static_option,
|
||||
{str(k): float(v) for k, v in (body.dynamic_levels or {}).items()},
|
||||
)
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/{project_id}/lrc")
|
||||
def lrc_project(project_id: str, body: LrcRequest):
|
||||
"""Align lyrics with a specific stem from this project to produce an ELRC file."""
|
||||
if body.lyrics_source not in ("auto", "tag", "text"):
|
||||
raise HTTPException(status_code=400, detail=f"Invalid lyrics_source: {body.lyrics_source}")
|
||||
if body.lyrics_source == "text" and not (body.lyrics_text or "").strip():
|
||||
raise HTTPException(status_code=400, detail="lyrics_text required when lyrics_source='text'")
|
||||
|
||||
project = _get_project_or_404(project_id)
|
||||
sep_job = jm.get_job(project["sep_job_id"])
|
||||
if sep_job is None:
|
||||
raise HTTPException(status_code=404, detail="Separation job not found")
|
||||
|
||||
project_dir = _project_output_dir(project)
|
||||
|
||||
# Find the source audio file (for output filename derivation)
|
||||
source_file = None
|
||||
for f in sep_job.get("files", []):
|
||||
if f.get("type") == "source":
|
||||
source_file = f
|
||||
break
|
||||
|
||||
if source_file is None:
|
||||
raise HTTPException(status_code=404, detail="Source audio file not found in project")
|
||||
|
||||
base_name = Path(source_file["name"]).stem
|
||||
output_filename = (body.output_filename or "").strip() or f"{base_name}.elrc"
|
||||
|
||||
# Prefer the already-separated stem file to avoid re-running Demucs.
|
||||
# If the stem file exists on disk, pass it directly with is_vocals=True.
|
||||
stem_files = _stem_files_from_job(sep_job)
|
||||
stem_path = _find_stem_path(stem_files, body.stem_name)
|
||||
if stem_path and Path(stem_path).exists():
|
||||
audio_path = stem_path
|
||||
is_vocals = True
|
||||
else:
|
||||
# Stem file missing — fall back to the original mix and let the
|
||||
# alignment service extract vocals via Demucs.
|
||||
audio_path = source_file["path"]
|
||||
is_vocals = False
|
||||
|
||||
job_id = jm.create_job("lrc_align")
|
||||
set_project_lrc_job(project_id, body.stem_name, job_id)
|
||||
jm.submit_gpu(
|
||||
run_lrc_alignment,
|
||||
job_id,
|
||||
audio_path,
|
||||
lyrics_source=body.lyrics_source,
|
||||
lyrics_content=body.lyrics_text,
|
||||
output_filename=output_filename,
|
||||
style_options=body.style_options,
|
||||
is_vocals=is_vocals,
|
||||
output_dir=str(project_dir),
|
||||
timing_precision=body.timing_precision,
|
||||
lrc_power_preset=body.lrc_power_preset,
|
||||
)
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/{project_id}/lyric-transcribe")
|
||||
def lyric_transcribe_project(project_id: str, body: LyricTranscribeRequest):
|
||||
"""
|
||||
Transcribe lyrics from a vocal stem using AI (Whisper).
|
||||
|
||||
Unlike /lrc which aligns existing lyrics, this endpoint generates
|
||||
lyrics from audio using speech recognition. It produces:
|
||||
- ELRC with word-level timestamps
|
||||
- LRC with line-level timestamps
|
||||
- JSON with full transcription data
|
||||
- Plain text lyrics
|
||||
"""
|
||||
# Validate model
|
||||
if body.model not in WHISPER_MODELS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {body.model}")
|
||||
|
||||
if body.task not in ("transcribe", "translate"):
|
||||
raise HTTPException(status_code=400, detail=f"Invalid task: {body.task}")
|
||||
|
||||
project = _get_project_or_404(project_id)
|
||||
sep_job = jm.get_job(project["sep_job_id"])
|
||||
if sep_job is None:
|
||||
raise HTTPException(status_code=404, detail="Separation job not found")
|
||||
|
||||
# Find the stem file
|
||||
stem_files = _stem_files_from_job(sep_job)
|
||||
if body.stem_name not in stem_files:
|
||||
raise HTTPException(status_code=404, detail=f"Stem not found: {body.stem_name}")
|
||||
|
||||
stem_path = stem_files[body.stem_name]
|
||||
|
||||
# Create output directory
|
||||
project_dir = _project_output_dir(project)
|
||||
output_dir = project_dir / "Transcriptions"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
job_id = jm.create_job("lyric-transcribe")
|
||||
|
||||
# Store job reference keyed by stem name; same key for both engines so the
|
||||
# frontend always shows the most-recent AI transcription for that stem.
|
||||
set_project_lrc_job(project_id, f"{body.stem_name}_whisper", job_id)
|
||||
|
||||
jm.submit_gpu(
|
||||
run_lyric_transcription,
|
||||
job_id,
|
||||
stem_path,
|
||||
model=body.model,
|
||||
language=body.language,
|
||||
task=body.task,
|
||||
output_formats=body.output_formats,
|
||||
initial_prompt=body.initial_prompt,
|
||||
temperature=body.temperature,
|
||||
condition_on_previous_text=body.condition_on_previous_text,
|
||||
output_dir=str(output_dir),
|
||||
engine=body.engine,
|
||||
diarize=body.diarize,
|
||||
)
|
||||
return {"job_id": job_id, "output_dir": str(output_dir)}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stem Reprocessing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class ReprocessRequest(BaseModel):
|
||||
stem_name: str
|
||||
model: str
|
||||
output_format: str = "mp3"
|
||||
|
||||
|
||||
@router.post("/{project_id}/reprocess")
|
||||
def reprocess_stem_endpoint(project_id: str, body: ReprocessRequest):
|
||||
"""
|
||||
Reprocess an individual stem using a specialized model.
|
||||
Useful for applying per-instrument models like MDX-Net or MelBand.
|
||||
"""
|
||||
from ..config import ALL_MODEL_IDS
|
||||
from ..services.separator_service import run_separation
|
||||
|
||||
if body.model not in ALL_MODEL_IDS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {body.model}")
|
||||
|
||||
project = _get_project_or_404(project_id)
|
||||
sep_job = jm.get_job(project["sep_job_id"])
|
||||
if sep_job is None:
|
||||
raise HTTPException(status_code=404, detail="Separation job not found")
|
||||
|
||||
stem_files = _stem_files_from_job(sep_job)
|
||||
stem_path = _find_stem_path(stem_files, body.stem_name)
|
||||
if stem_path is None:
|
||||
raise HTTPException(status_code=404, detail=f"Stem not found: {body.stem_name}")
|
||||
|
||||
# Create a new separation job that processes just this stem
|
||||
job_id = jm.create_job("separate")
|
||||
|
||||
# Submit to GPU executor - reprocess the stem file as if it were a source
|
||||
jm.submit_gpu(
|
||||
run_separation,
|
||||
job_id,
|
||||
stem_path,
|
||||
body.model,
|
||||
body.output_format,
|
||||
preset_type="stems_only",
|
||||
project_name_override=f"{project['name']}_{body.stem_name}_reprocessed",
|
||||
)
|
||||
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stem Renaming
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class RenameStemRequest(BaseModel):
|
||||
stem_name: str
|
||||
new_name: str
|
||||
|
||||
|
||||
@router.post("/{project_id}/rename-stem")
|
||||
def rename_stem_endpoint(project_id: str, body: RenameStemRequest):
|
||||
"""Rename a stem file and update the job record."""
|
||||
import os
|
||||
|
||||
project = _get_project_or_404(project_id)
|
||||
sep_job = jm.get_job(project["sep_job_id"])
|
||||
if sep_job is None:
|
||||
raise HTTPException(status_code=404, detail="Separation job not found")
|
||||
|
||||
# Find the stem file
|
||||
stem_files = sep_job.get("files", [])
|
||||
stem_file = None
|
||||
stem_idx = None
|
||||
|
||||
for idx, f in enumerate(stem_files):
|
||||
if f.get("stem") == body.stem_name or f.get("name") == body.stem_name:
|
||||
stem_file = f
|
||||
stem_idx = idx
|
||||
break
|
||||
|
||||
if stem_file is None:
|
||||
raise HTTPException(status_code=404, detail=f"Stem not found: {body.stem_name}")
|
||||
|
||||
old_path = Path(stem_file["path"])
|
||||
if not old_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Stem file not found on disk: {old_path}")
|
||||
|
||||
# Generate new filename
|
||||
new_name = body.new_name.strip()
|
||||
if not new_name:
|
||||
raise HTTPException(status_code=400, detail="New name cannot be empty")
|
||||
|
||||
# Keep the same extension
|
||||
new_filename = f"{new_name}{old_path.suffix}"
|
||||
new_path = old_path.parent / new_filename
|
||||
|
||||
# Avoid overwriting existing files
|
||||
if new_path.exists() and new_path != old_path:
|
||||
raise HTTPException(status_code=409, detail=f"A file with name '{new_filename}' already exists")
|
||||
|
||||
try:
|
||||
# Rename the file on disk
|
||||
os.rename(str(old_path), str(new_path))
|
||||
|
||||
# Update the job record
|
||||
stem_files[stem_idx]["name"] = new_filename
|
||||
stem_files[stem_idx]["stem"] = new_name
|
||||
stem_files[stem_idx]["path"] = str(new_path)
|
||||
|
||||
jm.update_job(project["sep_job_id"], files=stem_files)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"old_name": body.stem_name,
|
||||
"new_name": new_name,
|
||||
"new_path": str(new_path),
|
||||
}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Failed to rename stem: {e}")
|
||||
|
||||
|
||||
def _static_label(static_option: Optional[str]) -> str:
|
||||
labels = {
|
||||
"no_vocals": "No Vocals",
|
||||
"no_drums": "No Drums",
|
||||
"no_bass": "No Bass",
|
||||
"no_vocals_no_bass": "No Vocals No Bass",
|
||||
"no_drums_no_bass": "No Drums No Bass",
|
||||
}
|
||||
return labels.get(static_option or "", "Static Mix")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cleanup / maintenance endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/cleanup/verify")
|
||||
def cleanup_verify():
|
||||
"""
|
||||
Check every project's stem files are actually on disk.
|
||||
Returns a list of projects with missing files — no mutations performed.
|
||||
"""
|
||||
projects = jm.list_projects()
|
||||
results = []
|
||||
for proj in projects:
|
||||
sep_job = jm.get_job(proj["sep_job_id"]) or {}
|
||||
files = sep_job.get("files", [])
|
||||
missing = [
|
||||
f["path"]
|
||||
for f in files
|
||||
if f.get("type") == "stem" and not Path(f.get("path", "")).exists()
|
||||
]
|
||||
if missing:
|
||||
results.append({
|
||||
"project_id": proj["id"],
|
||||
"name": proj["name"],
|
||||
"missing_files": missing,
|
||||
})
|
||||
return {"projects_with_missing_files": results, "total": len(results)}
|
||||
|
||||
|
||||
@router.post("/cleanup/write-metadata")
|
||||
def cleanup_write_metadata():
|
||||
"""
|
||||
Write/refresh project.json for every project in the database.
|
||||
Uses current DB state so trans_jobs, lrc_jobs etc. are included.
|
||||
"""
|
||||
projects = jm.list_projects()
|
||||
written = 0
|
||||
errors = []
|
||||
for proj in projects:
|
||||
try:
|
||||
project_dir = _project_output_dir(proj)
|
||||
if not project_dir.exists():
|
||||
continue
|
||||
|
||||
sep_job = jm.get_job(proj["sep_job_id"]) or {}
|
||||
files = sep_job.get("files", [])
|
||||
stem_files = {
|
||||
f["stem"]: f["path"]
|
||||
for f in files
|
||||
if f.get("type") == "stem" and f.get("stem")
|
||||
}
|
||||
source_file = next(
|
||||
(f["path"] for f in files if f.get("type") == "source"), ""
|
||||
)
|
||||
result = sep_job.get("result") or {}
|
||||
|
||||
write_project_metadata(
|
||||
project_dir,
|
||||
project_id=proj["id"],
|
||||
name=proj["name"],
|
||||
model=result.get("model_label", ""),
|
||||
model_label=result.get("model_label", ""),
|
||||
format=result.get("format_label", ""),
|
||||
format_label=result.get("format_label", ""),
|
||||
source_file=source_file,
|
||||
stems=stem_files,
|
||||
sep_job_id=proj["sep_job_id"],
|
||||
trans_jobs=proj.get("trans_jobs") or {},
|
||||
abc_jobs=proj.get("abc_jobs") or {},
|
||||
lrc_jobs=proj.get("lrc_jobs") or {},
|
||||
mix_job_id=proj.get("mix_job_id"),
|
||||
created_at=proj.get("created_at"),
|
||||
)
|
||||
written += 1
|
||||
except Exception as e:
|
||||
errors.append({"project": proj.get("name", proj["id"]), "error": str(e)})
|
||||
|
||||
return {"written": written, "errors": errors}
|
||||
|
||||
|
||||
@router.post("/cleanup/rename-stems")
|
||||
def cleanup_rename_stems():
|
||||
"""
|
||||
Rename all stem files that still use the old 'filename - Vocals.ext' format
|
||||
to the new 'Artist - Title (vocals).ext' format.
|
||||
Updates the DB job file records after each rename.
|
||||
"""
|
||||
from ..services.separator_service import _stem_filename
|
||||
|
||||
projects = jm.list_projects()
|
||||
renamed_total = 0
|
||||
errors = []
|
||||
|
||||
for proj in projects:
|
||||
sep_job = jm.get_job(proj["sep_job_id"])
|
||||
if not sep_job:
|
||||
continue
|
||||
|
||||
files = list(sep_job.get("files", []))
|
||||
proj_name = proj["name"]
|
||||
changed = False
|
||||
|
||||
for i, f in enumerate(files):
|
||||
if f.get("type") != "stem" or not f.get("stem"):
|
||||
continue
|
||||
|
||||
old_path = Path(f["path"])
|
||||
if not old_path.exists():
|
||||
continue
|
||||
|
||||
# Build the expected new name and skip if already correct
|
||||
new_filename = _stem_filename(proj_name, f["stem"], old_path.suffix)
|
||||
new_path = old_path.parent / new_filename
|
||||
if old_path == new_path:
|
||||
continue
|
||||
|
||||
try:
|
||||
old_path.rename(new_path)
|
||||
files[i] = {**f, "name": new_filename, "path": str(new_path)}
|
||||
changed = True
|
||||
renamed_total += 1
|
||||
except Exception as e:
|
||||
errors.append({
|
||||
"project": proj_name,
|
||||
"file": str(old_path),
|
||||
"error": str(e),
|
||||
})
|
||||
|
||||
if changed:
|
||||
jm.update_job(proj["sep_job_id"], files=files)
|
||||
|
||||
return {"renamed": renamed_total, "errors": errors}
|
||||
|
||||
|
||||
@router.post("/cleanup/purge-orphaned")
|
||||
def cleanup_purge_orphaned():
|
||||
"""
|
||||
Delete separation job rows (job_type='separate') that have no matching
|
||||
ProjectRecord. Also deletes jobs of any type whose parent project is gone.
|
||||
"""
|
||||
project_job_ids = {
|
||||
proj["sep_job_id"]
|
||||
for proj in jm.list_projects()
|
||||
if proj.get("sep_job_id")
|
||||
}
|
||||
all_jobs = jm.list_all_jobs()
|
||||
purged = 0
|
||||
for job in all_jobs:
|
||||
if job.get("job_type") == "separate" and job["id"] not in project_job_ids:
|
||||
jm.delete_job(job["id"])
|
||||
purged += 1
|
||||
|
||||
return {"purged": purged}
|
||||
|
||||
|
||||
@router.post("/cleanup/clean-lrc-files")
|
||||
def cleanup_clean_lrc_files():
|
||||
"""
|
||||
Scan every LRC/ELRC output file in all lrc jobs and strip section-marker
|
||||
lines (e.g. [01:30.00][Chorus]). Returns counts of files and lines affected.
|
||||
"""
|
||||
all_jobs = jm.list_all_jobs()
|
||||
files_modified = 0
|
||||
lines_removed = 0
|
||||
errors = []
|
||||
|
||||
for job in all_jobs:
|
||||
if job.get("job_type") != "lrc":
|
||||
continue
|
||||
for file_entry in (job.get("files") or []):
|
||||
ftype = file_entry.get("type", "")
|
||||
if ftype not in ("elrc", "lrc"):
|
||||
continue
|
||||
fpath = file_entry.get("path")
|
||||
if not fpath:
|
||||
continue
|
||||
p = Path(fpath)
|
||||
if not p.exists():
|
||||
continue
|
||||
try:
|
||||
content = p.read_text(encoding="utf-8")
|
||||
cleaned, removed = clean_lrc_content(content)
|
||||
if removed > 0:
|
||||
p.write_text(cleaned, encoding="utf-8")
|
||||
files_modified += 1
|
||||
lines_removed += removed
|
||||
except Exception as exc:
|
||||
errors.append({"file": fpath, "error": str(exc)})
|
||||
|
||||
return {
|
||||
"files_modified": files_modified,
|
||||
"lines_removed": lines_removed,
|
||||
"errors": errors,
|
||||
}
|
||||
Executable
+277
@@ -0,0 +1,277 @@
|
||||
"""Stem separation endpoint with preset support."""
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, File, Form, UploadFile, HTTPException
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import (
|
||||
UPLOAD_DIR, ALL_MODELS, ALL_MODEL_IDS, OUTPUT_FORMATS,
|
||||
DEFAULT_MODEL, DEFAULT_OUTPUT_FORMAT, PRESET_TYPES, STATIC_OPTIONS,
|
||||
IMPORTS_DIR,
|
||||
)
|
||||
from ..services.separator_service import run_separation
|
||||
|
||||
router = APIRouter(prefix="/api/separate", tags=["separate"])
|
||||
|
||||
VALID_FORMATS = {f["id"] for f in OUTPUT_FORMATS}
|
||||
VALID_PRESETS = {p["id"] for p in PRESET_TYPES}
|
||||
VALID_STATIC_OPTIONS = {o["id"] for o in STATIC_OPTIONS}
|
||||
|
||||
|
||||
@router.get("/models")
|
||||
def get_models():
|
||||
return ALL_MODELS
|
||||
|
||||
|
||||
@router.get("/formats")
|
||||
def get_formats():
|
||||
return OUTPUT_FORMATS
|
||||
|
||||
|
||||
@router.get("/presets")
|
||||
def get_presets():
|
||||
"""Get available preset types and options."""
|
||||
return {
|
||||
"preset_types": PRESET_TYPES,
|
||||
"static_options": STATIC_OPTIONS,
|
||||
}
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def separate(
|
||||
file: UploadFile = File(...),
|
||||
model: str = Form(DEFAULT_MODEL),
|
||||
output_format: str = Form(DEFAULT_OUTPUT_FORMAT),
|
||||
# Preset options
|
||||
preset_type: str = Form("stems_only"),
|
||||
static_option: Optional[str] = Form(None),
|
||||
dynamic_levels: Optional[str] = Form(None), # JSON string: {"vocals": 1.0, ...}
|
||||
# Project naming
|
||||
project_name: Optional[str] = Form(None),
|
||||
# Power preset
|
||||
sep_power_preset: str = Form("standard"),
|
||||
):
|
||||
"""
|
||||
Separate audio into stems with optional preset mixing.
|
||||
|
||||
Args:
|
||||
file: Audio file to process
|
||||
model: Separation model ID
|
||||
output_format: Output audio format
|
||||
preset_type: Type of preset mix to create (stems_only, instrumental, static, dynamic)
|
||||
static_option: For static preset, which stems to remove
|
||||
dynamic_levels: For dynamic preset, JSON dict of stem name to volume multiplier (0.0-2.0)
|
||||
project_name: Override auto-generated project name
|
||||
"""
|
||||
# Validate inputs
|
||||
if model not in ALL_MODEL_IDS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {model}")
|
||||
if output_format not in VALID_FORMATS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown format: {output_format}")
|
||||
if preset_type not in VALID_PRESETS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown preset type: {preset_type}")
|
||||
|
||||
if preset_type == 'static' and static_option and static_option not in VALID_STATIC_OPTIONS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown static option: {static_option}")
|
||||
|
||||
# Parse dynamic levels JSON
|
||||
parsed_dynamic_levels = None
|
||||
if dynamic_levels:
|
||||
try:
|
||||
parsed_dynamic_levels = json.loads(dynamic_levels)
|
||||
except json.JSONDecodeError:
|
||||
raise HTTPException(status_code=400, detail="Invalid dynamic_levels JSON")
|
||||
|
||||
# Create job
|
||||
job_id = jm.create_job("separate")
|
||||
upload_dir = UPLOAD_DIR / job_id
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Save uploaded file
|
||||
upload_path = upload_dir / file.filename
|
||||
with open(upload_path, "wb") as f:
|
||||
shutil.copyfileobj(file.file, f)
|
||||
|
||||
# Submit job with all options
|
||||
jm.submit_gpu(
|
||||
run_separation,
|
||||
job_id,
|
||||
str(upload_path),
|
||||
model,
|
||||
output_format,
|
||||
preset_type=preset_type,
|
||||
static_option=static_option,
|
||||
dynamic_levels=parsed_dynamic_levels,
|
||||
project_name_override=project_name,
|
||||
sep_power_preset=sep_power_preset,
|
||||
)
|
||||
|
||||
return {
|
||||
"job_id": job_id,
|
||||
"preset_type": preset_type,
|
||||
"project_name": project_name,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/existing")
|
||||
async def separate_existing(
|
||||
project_name: str = Form(..., description="Project name to process"),
|
||||
model: str = Form(DEFAULT_MODEL),
|
||||
output_format: str = Form(DEFAULT_OUTPUT_FORMAT),
|
||||
preset_type: str = Form("stems_only"),
|
||||
static_option: Optional[str] = Form(None),
|
||||
dynamic_levels: Optional[str] = Form(None),
|
||||
sep_power_preset: str = Form("standard"),
|
||||
):
|
||||
"""
|
||||
Re-run separation on an existing project (if source audio exists).
|
||||
"""
|
||||
from ..config import PROJECTS_DIR
|
||||
|
||||
if model not in ALL_MODEL_IDS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {model}")
|
||||
if output_format not in VALID_FORMATS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown format: {output_format}")
|
||||
if preset_type not in VALID_PRESETS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown preset type: {preset_type}")
|
||||
|
||||
project_dir = PROJECTS_DIR / project_name
|
||||
if not project_dir.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Project not found: {project_name}")
|
||||
|
||||
# Look for source audio file
|
||||
source_file = None
|
||||
for ext in ['.mp3', '.flac', '.wav', '.m4a', '.ogg']:
|
||||
candidate = project_dir / f"{project_name}{ext}"
|
||||
if candidate.exists():
|
||||
source_file = candidate
|
||||
break
|
||||
# Check for any file with source-like name
|
||||
for f in project_dir.iterdir():
|
||||
if f.suffix.lower() == ext and 'source' in f.name.lower():
|
||||
source_file = f
|
||||
break
|
||||
if source_file:
|
||||
break
|
||||
|
||||
if not source_file:
|
||||
# Check uploads for the original file
|
||||
upload_dirs = list(UPLOAD_DIR.iterdir()) if UPLOAD_DIR.exists() else []
|
||||
for ud in sorted(upload_dirs, key=lambda x: x.stat().st_mtime, reverse=True):
|
||||
for f in ud.iterdir():
|
||||
if f.suffix.lower() in ['.mp3', '.flac', '.wav', '.m4a', '.ogg']:
|
||||
source_file = f
|
||||
break
|
||||
if source_file:
|
||||
break
|
||||
|
||||
if not source_file:
|
||||
raise HTTPException(status_code=404, detail="Source audio file not found")
|
||||
|
||||
# Parse dynamic levels
|
||||
parsed_dynamic_levels = None
|
||||
if dynamic_levels:
|
||||
try:
|
||||
parsed_dynamic_levels = json.loads(dynamic_levels)
|
||||
except json.JSONDecodeError:
|
||||
raise HTTPException(status_code=400, detail="Invalid dynamic_levels JSON")
|
||||
|
||||
# Create job
|
||||
job_id = jm.create_job("separate")
|
||||
|
||||
jm.submit(
|
||||
run_separation,
|
||||
job_id,
|
||||
str(source_file),
|
||||
model,
|
||||
output_format,
|
||||
preset_type=preset_type,
|
||||
static_option=static_option,
|
||||
dynamic_levels=parsed_dynamic_levels,
|
||||
project_name_override=project_name,
|
||||
sep_power_preset=sep_power_preset,
|
||||
)
|
||||
|
||||
return {
|
||||
"job_id": job_id,
|
||||
"preset_type": preset_type,
|
||||
"project_name": project_name,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/from-path")
|
||||
async def separate_from_path(
|
||||
file_path: str = Form(...),
|
||||
model: str = Form(DEFAULT_MODEL),
|
||||
output_format: str = Form(DEFAULT_OUTPUT_FORMAT),
|
||||
preset_type: str = Form("stems_only"),
|
||||
static_option: Optional[str] = Form(None),
|
||||
dynamic_levels: Optional[str] = Form(None),
|
||||
project_name: Optional[str] = Form(None),
|
||||
sep_power_preset: str = Form("standard"),
|
||||
):
|
||||
"""
|
||||
Separate a server-side audio file (e.g., imported from Subsonic).
|
||||
Only files inside IMPORTS_DIR are accepted for security.
|
||||
"""
|
||||
if model not in ALL_MODEL_IDS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {model}")
|
||||
if output_format not in VALID_FORMATS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown format: {output_format}")
|
||||
if preset_type not in VALID_PRESETS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown preset type: {preset_type}")
|
||||
|
||||
source = Path(file_path)
|
||||
if not source.exists() or not source.is_file():
|
||||
raise HTTPException(status_code=404, detail="File not found")
|
||||
|
||||
# Security: reject paths that escape IMPORTS_DIR
|
||||
try:
|
||||
source.resolve().relative_to(IMPORTS_DIR.resolve())
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=403, detail="File is not in the imports directory")
|
||||
|
||||
parsed_dynamic_levels = None
|
||||
if dynamic_levels:
|
||||
try:
|
||||
parsed_dynamic_levels = json.loads(dynamic_levels)
|
||||
except json.JSONDecodeError:
|
||||
raise HTTPException(status_code=400, detail="Invalid dynamic_levels JSON")
|
||||
|
||||
job_id = jm.create_job("separate")
|
||||
jm.submit_gpu(
|
||||
run_separation,
|
||||
job_id,
|
||||
str(source),
|
||||
model,
|
||||
output_format,
|
||||
preset_type=preset_type,
|
||||
static_option=static_option,
|
||||
dynamic_levels=parsed_dynamic_levels,
|
||||
project_name_override=project_name or source.stem,
|
||||
sep_power_preset=sep_power_preset,
|
||||
)
|
||||
|
||||
return {"job_id": job_id, "preset_type": preset_type}
|
||||
|
||||
|
||||
# Per-instrument models for targeted reprocessing
|
||||
PER_INSTRUMENT_MODELS = [
|
||||
{"id": "mdxnet_voc_ft", "label": "MDX-Net Vocals FT", "instrument": "vocals", "description": "Specialized vocals separation model"},
|
||||
{"id": "mdxnet_inst_hq_1", "label": "MDX-Net Inst HQ 1", "instrument": "instrumental", "description": "High-quality instrumental separation"},
|
||||
{"id": "mdxnet_inst_hq_2", "label": "MDX-Net Inst HQ 2", "instrument": "instrumental", "description": "Alternative high-quality instrumental model"},
|
||||
{"id": "mdxnet_karaoke", "label": "MDX-Net Karaoke", "instrument": "vocals", "description": "Optimized for karaoke vocal removal"},
|
||||
{"id": "mel_band_vocals", "label": "MelBand Vocals", "instrument": "vocals", "description": "MelBand RoFormer vocals separation"},
|
||||
{"id": "mel_band_dereverb", "label": "MelBand Dereverb", "instrument": "vocals", "description": "Remove reverb from vocals"},
|
||||
{"id": "mel_band_karaoke", "label": "MelBand Karaoke", "instrument": "vocals", "description": "Lead/backing vocals separation"},
|
||||
{"id": "bs_roformer_chorus", "label": "BS-RoFormer Chorus", "instrument": "vocals", "description": "Male/female chorus separation"},
|
||||
]
|
||||
|
||||
|
||||
@router.get("/per-instrument-models")
|
||||
def get_per_instrument_models():
|
||||
"""Get models suitable for per-instrument/stem reprocessing."""
|
||||
return PER_INSTRUMENT_MODELS
|
||||
Executable
+111
@@ -0,0 +1,111 @@
|
||||
"""Subsonic server integration — search, import, and config management."""
|
||||
import json
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ..config import SUBSONIC_CONFIG_PATH
|
||||
from ..services import subsonic_service
|
||||
|
||||
router = APIRouter(prefix="/api/subsonic", tags=["subsonic"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class SubsonicConfigIn(BaseModel):
|
||||
url: str
|
||||
username: str
|
||||
password: str
|
||||
|
||||
|
||||
@router.get("/config")
|
||||
def get_config():
|
||||
"""Return connection status (never exposes the password)."""
|
||||
if not SUBSONIC_CONFIG_PATH.exists():
|
||||
return {"configured": False}
|
||||
try:
|
||||
cfg = json.loads(SUBSONIC_CONFIG_PATH.read_text())
|
||||
return {
|
||||
"configured": True,
|
||||
"url": cfg.get("url", ""),
|
||||
"username": cfg.get("username", ""),
|
||||
}
|
||||
except Exception:
|
||||
return {"configured": False}
|
||||
|
||||
|
||||
@router.post("/config")
|
||||
def save_config(body: SubsonicConfigIn):
|
||||
"""Validate credentials against the server, then persist them."""
|
||||
cfg = {"url": body.url, "username": body.username, "password": body.password}
|
||||
if not subsonic_service.test_connection(cfg):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Could not connect to Subsonic server — check URL and credentials.",
|
||||
)
|
||||
subsonic_service.save_config(cfg)
|
||||
return {"configured": True, "url": body.url, "username": body.username}
|
||||
|
||||
|
||||
@router.delete("/config")
|
||||
def remove_config():
|
||||
"""Remove stored Subsonic credentials."""
|
||||
subsonic_service.delete_config()
|
||||
return {"configured": False}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Search & Import
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/search")
|
||||
def search(q: str = "", count: int = 20):
|
||||
"""Search songs on the configured Subsonic server."""
|
||||
if not q.strip():
|
||||
return []
|
||||
try:
|
||||
return subsonic_service.search_songs(q.strip(), count)
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=502, detail=f"Subsonic search failed: {exc}")
|
||||
|
||||
|
||||
class ImportRequest(BaseModel):
|
||||
id: str
|
||||
title: str
|
||||
artist: str = ""
|
||||
suffix: str = "mp3"
|
||||
|
||||
|
||||
@router.post("/import")
|
||||
def import_song(req: ImportRequest):
|
||||
"""Download a Subsonic song to the local imports directory."""
|
||||
try:
|
||||
return subsonic_service.import_song(req.id, req.title, req.artist, req.suffix)
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=502, detail=f"Import failed: {exc}")
|
||||
|
||||
|
||||
@router.get("/imports")
|
||||
def list_imports():
|
||||
"""List all locally imported audio files."""
|
||||
return subsonic_service.list_imports()
|
||||
|
||||
|
||||
@router.delete("/imports/{filename}")
|
||||
def delete_import(filename: str):
|
||||
"""Delete an imported file from the imports directory."""
|
||||
from ..config import IMPORTS_DIR
|
||||
# Security: reject any path traversal
|
||||
if ".." in filename or "/" in filename or "\\" in filename:
|
||||
raise HTTPException(status_code=400, detail="Invalid filename")
|
||||
target = IMPORTS_DIR / filename
|
||||
if not target.exists():
|
||||
raise HTTPException(status_code=404, detail="File not found")
|
||||
target.unlink()
|
||||
return {"detail": f"Deleted {filename}"}
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
"""System information endpoints (GPU hardware)."""
|
||||
from fastapi import APIRouter
|
||||
|
||||
router = APIRouter(prefix="/api/system", tags=["system"])
|
||||
|
||||
|
||||
@router.post("/gpu-reset")
|
||||
def gpu_reset():
|
||||
"""
|
||||
Re-enable GPU acceleration for all services without restarting the container.
|
||||
|
||||
After a CUDA OOM or 'unknown error', the lyric transcriber marks the GPU
|
||||
context as failed and falls back to CPU. The separator may also fail with
|
||||
a CUDA context error. Calling this endpoint clears the CUDA allocation
|
||||
cache and resets the Whisper device flag, so both services can retry on GPU.
|
||||
|
||||
NOTE: if the CUDA context is truly corrupted (persistent 'unknown error'),
|
||||
a container restart is still required.
|
||||
"""
|
||||
from ..services.lyric_transcriber_service import reset_cuda
|
||||
return reset_cuda()
|
||||
|
||||
|
||||
@router.get("/gpu-info")
|
||||
def gpu_info():
|
||||
"""Return GPU name, VRAM figures, and cuda_ok health flag."""
|
||||
try:
|
||||
from ..services.lyric_transcriber_service import _cuda_ok as whisper_ok
|
||||
except Exception:
|
||||
whisper_ok = True
|
||||
|
||||
try:
|
||||
import torch # type: ignore
|
||||
if not torch.cuda.is_available():
|
||||
return {"cuda_available": False, "cuda_ok": whisper_ok, "name": None, "total_mb": 0, "free_mb": 0}
|
||||
props = torch.cuda.get_device_properties(0)
|
||||
free_bytes, total_bytes = torch.cuda.mem_get_info(0)
|
||||
return {
|
||||
"cuda_available": True,
|
||||
"cuda_ok": whisper_ok,
|
||||
"name": props.name,
|
||||
"total_mb": total_bytes // (1024 * 1024),
|
||||
"free_mb": free_bytes // (1024 * 1024),
|
||||
}
|
||||
except Exception as exc: # pragma: no cover
|
||||
return {"cuda_available": False, "cuda_ok": False, "name": None, "total_mb": 0, "free_mb": 0, "error": str(exc)}
|
||||
Executable
+88
@@ -0,0 +1,88 @@
|
||||
"""MIDI transcription endpoint."""
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, File, Form, UploadFile, HTTPException
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import UPLOAD_DIR, PROJECTS_DIR, TRANSCRIBE_MODES, DEFAULT_TRANSCRIBE_MODE
|
||||
from ..services.transcriber_service import run_transcription
|
||||
|
||||
router = APIRouter(prefix="/api/transcribe", tags=["transcribe"])
|
||||
|
||||
VALID_MODES = {m["id"] for m in TRANSCRIBE_MODES}
|
||||
|
||||
# Preferred stem order for transcription (drums are skipped; vocals last since
|
||||
# Omnizart's "music" mode is most useful on pitched instruments).
|
||||
_STEM_PREFERENCE = ["other", "piano", "guitar", "bass", "vocals"]
|
||||
_AUDIO_EXTS = {".mp3", ".wav", ".flac"}
|
||||
|
||||
|
||||
@router.get("/modes")
|
||||
def get_modes():
|
||||
return TRANSCRIBE_MODES
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def transcribe(
|
||||
file: UploadFile = File(...),
|
||||
mode: str = Form(DEFAULT_TRANSCRIBE_MODE),
|
||||
):
|
||||
if mode not in VALID_MODES:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown mode: {mode}")
|
||||
|
||||
job_id = jm.create_job("transcribe")
|
||||
upload_path = UPLOAD_DIR / job_id / file.filename
|
||||
upload_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(upload_path, "wb") as f:
|
||||
shutil.copyfileobj(file.file, f)
|
||||
|
||||
jm.submit_gpu(run_transcription, job_id, str(upload_path), mode)
|
||||
return {"job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/from-project")
|
||||
async def transcribe_from_project(
|
||||
project_name: str = Form(...),
|
||||
mode: str = Form(DEFAULT_TRANSCRIBE_MODE),
|
||||
):
|
||||
"""
|
||||
Transcribe a stem from an existing separation project.
|
||||
Picks the best non-drum stem (other > piano > guitar > bass > vocals).
|
||||
"""
|
||||
if mode not in VALID_MODES:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown mode: {mode}")
|
||||
|
||||
stems_dir = PROJECTS_DIR / project_name / "Stems"
|
||||
if not stems_dir.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Stems folder not found for project: {project_name}")
|
||||
|
||||
# Find the best non-drum stem in order of preference
|
||||
candidates: dict[str, Path] = {}
|
||||
for f in stems_dir.iterdir():
|
||||
if f.is_file() and f.suffix.lower() in _AUDIO_EXTS and "drum" not in f.name.lower():
|
||||
name_lower = f.stem.lower()
|
||||
for pref in _STEM_PREFERENCE:
|
||||
if pref in name_lower:
|
||||
candidates.setdefault(pref, f)
|
||||
|
||||
stem_file: Path | None = None
|
||||
for pref in _STEM_PREFERENCE:
|
||||
if pref in candidates:
|
||||
stem_file = candidates[pref]
|
||||
break
|
||||
|
||||
# Fallback: any non-drum audio file
|
||||
if stem_file is None:
|
||||
for f in stems_dir.iterdir():
|
||||
if f.is_file() and f.suffix.lower() in _AUDIO_EXTS and "drum" not in f.name.lower():
|
||||
stem_file = f
|
||||
break
|
||||
|
||||
if stem_file is None:
|
||||
raise HTTPException(status_code=404, detail="No suitable (non-drum) stem found in project")
|
||||
|
||||
job_id = jm.create_job("transcribe")
|
||||
jm.submit_gpu(run_transcription, job_id, str(stem_file), mode)
|
||||
return {"job_id": job_id, "stem": stem_file.name}
|
||||
Executable
Whitespace-only changes.
Executable
+129
@@ -0,0 +1,129 @@
|
||||
from music21 import environment, converter, metadata, stream
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import os
|
||||
import traceback
|
||||
|
||||
|
||||
def _post_process_abc_content(abc_text: str, title: str) -> str:
|
||||
"""Cleans midi2abc output, sets title, removes comments."""
|
||||
lines = abc_text.splitlines()
|
||||
cleaned_lines = []
|
||||
seen_midi_program_in_voice = False
|
||||
|
||||
for line in lines:
|
||||
# The midi2abc tool generates a default title from the filename.
|
||||
if line.startswith('T:'):
|
||||
cleaned_lines.append(f'T: {title}')
|
||||
continue
|
||||
|
||||
if line.strip().startswith('% Last note suggests'):
|
||||
continue
|
||||
|
||||
if line.strip().startswith('V:'):
|
||||
seen_midi_program_in_voice = False
|
||||
if line.strip().startswith('%%MIDI program'):
|
||||
if not seen_midi_program_in_voice:
|
||||
seen_midi_program_in_voice = True
|
||||
cleaned_lines.append(line)
|
||||
continue
|
||||
|
||||
cleaned_lines.append(line)
|
||||
|
||||
return '\n'.join(cleaned_lines)
|
||||
|
||||
|
||||
def convert_midi_to_abc(
|
||||
midi_paths: list, song_title: str
|
||||
) -> dict[str, str] | None:
|
||||
"""Converts MIDI files to a dictionary of ABC strings."""
|
||||
us = environment.UserSettings()
|
||||
us['directoryScratch'] = '/tmp'
|
||||
|
||||
temp_dir = Path(us['directoryScratch'])
|
||||
|
||||
abc_results = {}
|
||||
|
||||
for midi_path in midi_paths:
|
||||
part_name = Path(midi_path).stem
|
||||
part_title = f"{song_title} ({part_name})"
|
||||
print(f" -> Processing part: {part_name}.mid")
|
||||
|
||||
temp_midi_path = temp_dir / f"{part_title}.mid"
|
||||
temp_abc_path = temp_dir / f"{part_title}.abc"
|
||||
|
||||
try:
|
||||
score_part = converter.parse(midi_path, forceSource=True)
|
||||
if not score_part or not score_part.flatten().notesAndRests:
|
||||
print(f" -> Skipping empty MIDI: {Path(midi_path).name}") # noqa
|
||||
continue
|
||||
|
||||
single_part_score = stream.Score()
|
||||
single_part_score.insert(0, metadata.Metadata())
|
||||
single_part_score.metadata.title = part_title
|
||||
score_part.parts[0].id = part_name
|
||||
single_part_score.insert(0, score_part.parts[0])
|
||||
|
||||
single_part_score.write('midi', fp=str(temp_midi_path))
|
||||
|
||||
cmd = ['midi2abc', str(temp_midi_path),
|
||||
'-o', str(temp_abc_path)]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, check=False)
|
||||
|
||||
if result.returncode != 0:
|
||||
print(f" -> ERROR: midi2abc failed for {part_name}: "
|
||||
f"{result.stderr}")
|
||||
continue
|
||||
|
||||
if not temp_abc_path.exists() or temp_abc_path.stat().st_size == 0:
|
||||
print(f" -> ERROR: midi2abc created an empty file for {part_name}.")
|
||||
continue
|
||||
|
||||
with open(temp_abc_path, 'r', encoding='utf-8') as f:
|
||||
abc_text_content = f.read()
|
||||
|
||||
abc_results[part_name] = _post_process_abc_content(
|
||||
abc_text_content, part_title)
|
||||
print(f" -> Successfully converted {part_name} to ABC.")
|
||||
|
||||
except Exception as e:
|
||||
print(
|
||||
f" -> ERROR during ABC conversion for {part_name}: {e}")
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
if temp_midi_path.exists():
|
||||
os.remove(temp_midi_path)
|
||||
if temp_abc_path.exists():
|
||||
os.remove(temp_abc_path)
|
||||
|
||||
return abc_results if abc_results else None
|
||||
|
||||
|
||||
def generate_mix_abc(mix_midi_path: str, song_title: str) -> str | None:
|
||||
"""Generates ABC notation for a single 'mix' MIDI file."""
|
||||
us = environment.UserSettings()
|
||||
us['directoryScratch'] = '/tmp'
|
||||
temp_dir = Path(us['directoryScratch'])
|
||||
mix_title = f"{song_title} (Mix)"
|
||||
temp_abc_path = temp_dir / f"{mix_title}.abc"
|
||||
|
||||
try:
|
||||
print("\n -> Creating combined 'Mix' ABC score...")
|
||||
cmd = ['midi2abc', mix_midi_path, '-o', str(temp_abc_path)]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, check=False)
|
||||
|
||||
if result.returncode == 0 and temp_abc_path.exists():
|
||||
raw_mix_abc = temp_abc_path.read_text(encoding='utf-8')
|
||||
cleaned_mix_abc = _post_process_abc_content(raw_mix_abc, mix_title)
|
||||
print(" -> Successfully created 'Mix' ABC.")
|
||||
return cleaned_mix_abc
|
||||
else:
|
||||
print(
|
||||
f" -> ERROR: midi2abc failed for Mix: {result.stderr}")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f" -> ERROR during Mix ABC conversion: {e}")
|
||||
return None
|
||||
finally:
|
||||
if temp_abc_path.exists():
|
||||
os.remove(temp_abc_path)
|
||||
Executable
+70
@@ -0,0 +1,70 @@
|
||||
"""
|
||||
ABC notation conversion via abc_generator (backend/services/abc_generator.py).
|
||||
"""
|
||||
from .. import job_manager as jm
|
||||
from ..config import JOBS_DIR
|
||||
from . import abc_generator
|
||||
|
||||
|
||||
def run_abc_conversion(job_id: str, midi_paths: list[str], song_title: str):
|
||||
"""
|
||||
Background task: convert MIDI file(s) to ABC notation.
|
||||
midi_paths: list of MIDI file paths
|
||||
song_title: title to use in ABC headers
|
||||
"""
|
||||
output_dir = JOBS_DIR / job_id
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Converting MIDI to ABC...")
|
||||
|
||||
try:
|
||||
abc_results = abc_generator.convert_midi_to_abc(midi_paths, song_title)
|
||||
|
||||
if not abc_results:
|
||||
raise RuntimeError("No ABC output produced — MIDI may be empty or unreadable.")
|
||||
|
||||
files = []
|
||||
for part_name, abc_text in abc_results.items():
|
||||
safe_name = part_name.replace("/", "_").replace("\\", "_")
|
||||
filename = f"{safe_name}.abc"
|
||||
dest = output_dir / filename
|
||||
dest.write_text(abc_text, encoding="utf-8")
|
||||
files.append({"name": filename, "path": str(dest)})
|
||||
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.DONE,
|
||||
progress="ABC conversion complete.",
|
||||
files=files,
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
|
||||
|
||||
|
||||
def run_mix_abc_conversion(job_id: str, mix_midi_path: str, song_title: str):
|
||||
"""Convert a single mix MIDI to ABC notation."""
|
||||
output_dir = JOBS_DIR / job_id
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Converting mix MIDI to ABC...")
|
||||
|
||||
try:
|
||||
abc_text = abc_generator.generate_mix_abc(mix_midi_path, song_title)
|
||||
|
||||
if not abc_text:
|
||||
raise RuntimeError("No ABC output produced.")
|
||||
|
||||
dest = output_dir / "mix.abc"
|
||||
dest.write_text(abc_text, encoding="utf-8")
|
||||
|
||||
files = [{"name": "mix.abc", "path": str(dest)}]
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.DONE,
|
||||
progress="ABC conversion complete.",
|
||||
files=files,
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
|
||||
Executable
+155
@@ -0,0 +1,155 @@
|
||||
"""
|
||||
Apollo separator - Vocal enhancement and reverb removal
|
||||
Models from MVSEP-Central-Backup: https://huggingface.co/miercolesv/MVSEP-Central-Backup
|
||||
"""
|
||||
import os
|
||||
import torch
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
from ..config import APOLLO_MODEL_DIR, CPU_SEPARATION
|
||||
|
||||
|
||||
# Model configurations
|
||||
APOLLO_MODELS = {
|
||||
"apollo_vocals": {
|
||||
"model_path": "epoch=54-val_loss=-17.6221.ckpt",
|
||||
"config_path": "config_apollo_vocals_ep_54.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "Apollo-Vocal-MSST",
|
||||
"stems": ["dry_vocals", "reverb"],
|
||||
"description": "Apollo vocal enhancement/reverb removal",
|
||||
},
|
||||
"apollo_edm": {
|
||||
"model_path": "apollo_edm_by_essid.ckpt",
|
||||
"config_path": "apollo_edm_by_essid.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "Politrees/UVR_resources/models/Apollo",
|
||||
"stems": ["enhanced", "residual"],
|
||||
"description": "Apollo EDM enhancement",
|
||||
},
|
||||
"apollo_edm_big": {
|
||||
"model_path": "apollo_edm_big_by_essid.ckpt",
|
||||
"config_path": "apollo_edm_big_by_essid.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "Politrees/UVR_resources/models/Apollo",
|
||||
"stems": ["enhanced", "residual"],
|
||||
"description": "Apollo EDM Big enhancement",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class ApolloSeparator:
|
||||
"""
|
||||
Apollo separator for vocal enhancement and reverb removal.
|
||||
|
||||
Apollo models separate vocals into dry vocals and reverb,
|
||||
useful for cleaning up vocal recordings.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_id: str = "apollo_vocals",
|
||||
cpu_separation: bool = CPU_SEPARATION,
|
||||
output_format: str = "mp3",
|
||||
mp3_bitrate: str = "256k",
|
||||
):
|
||||
if model_id not in APOLLO_MODELS:
|
||||
raise ValueError(f"Unknown Apollo model: {model_id}")
|
||||
|
||||
self.model_id = model_id
|
||||
self.model_config = APOLLO_MODELS[model_id]
|
||||
self.cpu_separation = cpu_separation
|
||||
self.output_format = output_format
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self._model = None
|
||||
|
||||
def _ensure_model(self):
|
||||
"""Download model from HuggingFace if not present."""
|
||||
model_dir = APOLLO_MODEL_DIR / self.model_id
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
model_path = model_dir / self.model_config["model_path"]
|
||||
config_path = model_dir / self.model_config["config_path"]
|
||||
|
||||
if not model_path.exists() or not config_path.exists():
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
print(f"Downloading Apollo model: {self.model_id}")
|
||||
|
||||
if not model_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.model_config["hf_repo"],
|
||||
filename=f"{self.model_config['hf_subfolder']}/{self.model_config['model_path']}",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
if not config_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.model_config["hf_repo"],
|
||||
filename=f"{self.model_config['hf_subfolder']}/{self.model_config['config_path']}",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to download Apollo model: {e}")
|
||||
|
||||
return str(model_path), str(config_path)
|
||||
|
||||
def _load_model(self):
|
||||
"""Lazy-load the model."""
|
||||
if self._model is None:
|
||||
model_path, config_path = self._ensure_model()
|
||||
self._model = None # Placeholder for actual model loading
|
||||
return self._model
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
||||
"""
|
||||
Separate vocals into dry vocals and reverb.
|
||||
|
||||
Returns:
|
||||
Dict mapping stem names to output file paths
|
||||
"""
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# For now, use audio-separator as fallback
|
||||
# Full implementation would load the actual Apollo model
|
||||
try:
|
||||
from audio_separator.separator import Separator
|
||||
|
||||
sep = Separator(
|
||||
model_file_dir=str(APOLLO_MODEL_DIR),
|
||||
output_dir=str(output_path),
|
||||
output_format=self.output_format.upper(),
|
||||
output_bitrate=self.mp3_bitrate if self.output_format == "mp3" else None,
|
||||
)
|
||||
|
||||
# Use dereverb model as approximation
|
||||
sep.load_model()
|
||||
result = sep.separate(input_path)
|
||||
|
||||
stems = self.model_config.get("stems", ["dry_vocals", "reverb"])
|
||||
output_paths = {}
|
||||
|
||||
if isinstance(result, list):
|
||||
for i, path in enumerate(result):
|
||||
if i < len(stems):
|
||||
stem_name = stems[i]
|
||||
final_path = output_path / f"{stem_name}.{self.output_format}"
|
||||
Path(path).rename(final_path)
|
||||
output_paths[stem_name] = str(final_path)
|
||||
else:
|
||||
final_path = output_path / f"dry_vocals.{self.output_format}"
|
||||
Path(result).rename(final_path)
|
||||
output_paths["dry_vocals"] = str(final_path)
|
||||
|
||||
return output_paths
|
||||
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"audio-separator is required for Apollo separation. "
|
||||
"Install with: pip install audio-separator"
|
||||
)
|
||||
Executable
+144
@@ -0,0 +1,144 @@
|
||||
"""
|
||||
Aspiration separator - Breath/aspiration removal from vocals
|
||||
Models from MVSEP-Central-Backup: https://huggingface.co/miercolesv/MVSEP-Central-Backup
|
||||
"""
|
||||
import os
|
||||
import torch
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
from ..config import ASPIRATION_MODEL_DIR, CPU_SEPARATION
|
||||
|
||||
|
||||
# Model configurations
|
||||
ASPIRATION_MODELS = {
|
||||
"aspiration_full": {
|
||||
"model_path": "aspiration_mel_band_roformer_sdr_18.9845.ckpt",
|
||||
"config_path": "config_aspiration_mel_band_roformer.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "Aspiration-MelBand-Sucial",
|
||||
"stems": ["clean_vocals", "aspiration"],
|
||||
"aggression": "full",
|
||||
},
|
||||
"aspiration_light": {
|
||||
"model_path": "aspiration_mel_band_roformer_less_aggr_sdr_18.1201.ckpt",
|
||||
"config_path": "config_aspiration_mel_band_roformer.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "Aspiration-MelBand-Sucial",
|
||||
"stems": ["clean_vocals", "aspiration"],
|
||||
"aggression": "light",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class AspirationSeparator:
|
||||
"""
|
||||
Aspiration separator for breath/aspiration removal from vocals.
|
||||
|
||||
Useful for cleaning vocal recordings by separating breath sounds
|
||||
from the main vocal performance.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_id: str = "aspiration_full",
|
||||
cpu_separation: bool = CPU_SEPARATION,
|
||||
output_format: str = "mp3",
|
||||
mp3_bitrate: str = "256k",
|
||||
):
|
||||
if model_id not in ASPIRATION_MODELS:
|
||||
raise ValueError(f"Unknown Aspiration model: {model_id}")
|
||||
|
||||
self.model_id = model_id
|
||||
self.model_config = ASPIRATION_MODELS[model_id]
|
||||
self.cpu_separation = cpu_separation
|
||||
self.output_format = output_format
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self._model = None
|
||||
|
||||
def _ensure_model(self):
|
||||
"""Download model from HuggingFace if not present."""
|
||||
model_dir = ASPIRATION_MODEL_DIR / self.model_id
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
model_path = model_dir / self.model_config["model_path"]
|
||||
config_path = model_dir / self.model_config["config_path"]
|
||||
|
||||
if not model_path.exists() or not config_path.exists():
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
print(f"Downloading Aspiration model: {self.model_id}")
|
||||
|
||||
if not model_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.model_config["hf_repo"],
|
||||
filename=f"{self.model_config['hf_subfolder']}/{self.model_config['model_path']}",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
if not config_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.model_config["hf_repo"],
|
||||
filename=f"{self.model_config['hf_subfolder']}/{self.model_config['config_path']}",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to download Aspiration model: {e}")
|
||||
|
||||
return str(model_path), str(config_path)
|
||||
|
||||
def _load_model(self):
|
||||
"""Lazy-load the model."""
|
||||
if self._model is None:
|
||||
model_path, config_path = self._ensure_model()
|
||||
self._model = None # Placeholder for actual model loading
|
||||
return self._model
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
||||
"""
|
||||
Separate vocals into clean vocals and aspiration (breath).
|
||||
|
||||
Returns:
|
||||
Dict mapping stem names to output file paths
|
||||
"""
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
from audio_separator.separator import Separator
|
||||
|
||||
sep = Separator(
|
||||
model_file_dir=str(ASPIRATION_MODEL_DIR),
|
||||
output_dir=str(output_path),
|
||||
output_format=self.output_format.upper(),
|
||||
output_bitrate=self.mp3_bitrate if self.output_format == "mp3" else None,
|
||||
)
|
||||
|
||||
sep.load_model()
|
||||
result = sep.separate(input_path)
|
||||
|
||||
stems = self.model_config.get("stems", ["clean_vocals", "aspiration"])
|
||||
output_paths = {}
|
||||
|
||||
if isinstance(result, list):
|
||||
for i, path in enumerate(result):
|
||||
if i < len(stems):
|
||||
stem_name = stems[i]
|
||||
final_path = output_path / f"{stem_name}.{self.output_format}"
|
||||
Path(path).rename(final_path)
|
||||
output_paths[stem_name] = str(final_path)
|
||||
else:
|
||||
final_path = output_path / f"clean_vocals.{self.output_format}"
|
||||
Path(result).rename(final_path)
|
||||
output_paths["clean_vocals"] = str(final_path)
|
||||
|
||||
return output_paths
|
||||
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"audio-separator is required for Aspiration separation. "
|
||||
"Install with: pip install audio-separator"
|
||||
)
|
||||
Executable
+150
@@ -0,0 +1,150 @@
|
||||
"""
|
||||
Bandit-v2 separator - Multi-language/multi-stem separation
|
||||
Models from MVSEP-Central-Backup: https://huggingface.co/miercolesv/MVSEP-Central-Backup
|
||||
"""
|
||||
import os
|
||||
import torch
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
from ..config import BANDIT_MODEL_DIR, CPU_SEPARATION
|
||||
|
||||
|
||||
# Model configurations
|
||||
BANDIT_MODELS = {
|
||||
"bandit_v2_multi": {
|
||||
"model_path": "checkpoint-multi_state_dict.ckpt",
|
||||
"config_path": "config_dnr_bandit_v2_mus64.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "Bandit-v2",
|
||||
"stems": ["vocals", "drums", "bass", "other"],
|
||||
"language": "multi",
|
||||
},
|
||||
"bandit_v2_eng": {
|
||||
"model_path": "checkpoint-eng_state_dict.ckpt",
|
||||
"config_path": "config_dnr_bandit_v2_mus64.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "Bandit-v2",
|
||||
"stems": ["vocals", "drums", "bass", "other"],
|
||||
"language": "english",
|
||||
},
|
||||
# Additional language models available:
|
||||
# - checkpoint-cmn_state_dict.ckpt (Chinese)
|
||||
# - checkpoint-deu_state_dict.ckpt (German)
|
||||
# - checkpoint-fao_state_dict.ckpt (Faroese)
|
||||
# - checkpoint-fra_state_dict.ckpt (French)
|
||||
# - checkpoint-spa_state_dict.ckpt (Spanish)
|
||||
}
|
||||
|
||||
|
||||
class BanditSeparator:
|
||||
"""
|
||||
Bandit-v2 separator for multi-language/multi-stem separation.
|
||||
|
||||
Bandit models are trained on different languages and provide
|
||||
high-quality separation optimized for specific language characteristics.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_id: str = "bandit_v2_multi",
|
||||
cpu_separation: bool = CPU_SEPARATION,
|
||||
output_format: str = "mp3",
|
||||
mp3_bitrate: str = "256k",
|
||||
):
|
||||
if model_id not in BANDIT_MODELS:
|
||||
raise ValueError(f"Unknown Bandit model: {model_id}")
|
||||
|
||||
self.model_id = model_id
|
||||
self.model_config = BANDIT_MODELS[model_id]
|
||||
self.cpu_separation = cpu_separation
|
||||
self.output_format = output_format
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self._model = None
|
||||
|
||||
def _ensure_model(self):
|
||||
"""Download model from HuggingFace if not present."""
|
||||
model_dir = BANDIT_MODEL_DIR / self.model_id
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
model_path = model_dir / self.model_config["model_path"]
|
||||
config_path = model_dir / self.model_config["config_path"]
|
||||
|
||||
if not model_path.exists() or not config_path.exists():
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
print(f"Downloading Bandit model: {self.model_id}")
|
||||
|
||||
if not model_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.model_config["hf_repo"],
|
||||
filename=f"{self.model_config['hf_subfolder']}/{self.model_config['model_path']}",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
if not config_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.model_config["hf_repo"],
|
||||
filename=f"{self.model_config['hf_subfolder']}/{self.model_config['config_path']}",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to download Bandit model: {e}")
|
||||
|
||||
return str(model_path), str(config_path)
|
||||
|
||||
def _load_model(self):
|
||||
"""Lazy-load the model."""
|
||||
if self._model is None:
|
||||
model_path, config_path = self._ensure_model()
|
||||
self._model = None # Placeholder for actual model loading
|
||||
return self._model
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
||||
"""
|
||||
Separate audio into stems using Bandit-v2.
|
||||
|
||||
Returns:
|
||||
Dict mapping stem names to output file paths
|
||||
"""
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
from audio_separator.separator import Separator
|
||||
|
||||
sep = Separator(
|
||||
model_file_dir=str(BANDIT_MODEL_DIR),
|
||||
output_dir=str(output_path),
|
||||
output_format=self.output_format.upper(),
|
||||
output_bitrate=self.mp3_bitrate if self.output_format == "mp3" else None,
|
||||
)
|
||||
|
||||
sep.load_model()
|
||||
result = sep.separate(input_path)
|
||||
|
||||
stems = self.model_config.get("stems", ["vocals", "drums", "bass", "other"])
|
||||
output_paths = {}
|
||||
|
||||
if isinstance(result, list):
|
||||
for i, path in enumerate(result):
|
||||
if i < len(stems):
|
||||
stem_name = stems[i]
|
||||
final_path = output_path / f"{stem_name}.{self.output_format}"
|
||||
Path(path).rename(final_path)
|
||||
output_paths[stem_name] = str(final_path)
|
||||
else:
|
||||
final_path = output_path / f"vocals.{self.output_format}"
|
||||
Path(result).rename(final_path)
|
||||
output_paths["vocals"] = str(final_path)
|
||||
|
||||
return output_paths
|
||||
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"audio-separator is required for Bandit separation. "
|
||||
"Install with: pip install audio-separator"
|
||||
)
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
from .bs_roformer import BSRoformer
|
||||
|
||||
__all__ = ['BSRoformer']
|
||||
Executable
+116
@@ -0,0 +1,116 @@
|
||||
from functools import wraps
|
||||
from packaging import version
|
||||
from collections import namedtuple
|
||||
|
||||
import torch
|
||||
from torch import nn, einsum
|
||||
import torch.nn.functional as F
|
||||
|
||||
from einops import rearrange, reduce
|
||||
|
||||
"""
|
||||
Code adapted from https://github.com/Anjok07/ultimatevocalremovergui/blob/v5.6.0_roformer_add%2Bdirectml/lib_v5/bs_roformer.py and https://github.com/lucidrains/BS-RoFormer which are copyrighted under the terms of the MIT license.
|
||||
"""
|
||||
|
||||
# constants
|
||||
|
||||
FlashAttentionConfig = namedtuple('FlashAttentionConfig', ['enable_flash', 'enable_math', 'enable_mem_efficient'])
|
||||
|
||||
# helpers
|
||||
|
||||
def exists(val):
|
||||
return val is not None
|
||||
|
||||
def once(fn):
|
||||
called = False
|
||||
@wraps(fn)
|
||||
def inner(x):
|
||||
nonlocal called
|
||||
if called:
|
||||
return
|
||||
called = True
|
||||
return fn(x)
|
||||
return inner
|
||||
|
||||
print_once = once(print)
|
||||
|
||||
# main class
|
||||
|
||||
class Attend(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
dropout = 0.,
|
||||
flash = False,
|
||||
scale = None
|
||||
):
|
||||
super().__init__()
|
||||
self.dropout = dropout
|
||||
self.attn_dropout = nn.Dropout(dropout)
|
||||
self.scale = scale
|
||||
|
||||
self.flash = flash
|
||||
assert not (flash and version.parse(torch.__version__) < version.parse('2.0.0')), 'in order to use flash attention, you must be using pytorch 2.0 or above'
|
||||
|
||||
# determine efficient attention configs for cuda and cpu
|
||||
|
||||
self.cpu_config = FlashAttentionConfig(True, True, True)
|
||||
self.cuda_config = None
|
||||
|
||||
if not torch.cuda.is_available() or not flash:
|
||||
return
|
||||
|
||||
device_properties = torch.cuda.get_device_properties(torch.device('cuda'))
|
||||
|
||||
if device_properties.major == 8 and device_properties.minor == 0:
|
||||
print_once('A100 GPU detected, using flash attention if input tensor is on cuda')
|
||||
self.cuda_config = FlashAttentionConfig(True, False, False)
|
||||
else:
|
||||
self.cuda_config = FlashAttentionConfig(False, True, True)
|
||||
|
||||
def flash_attn(self, q, k, v):
|
||||
_, heads, q_len, _, k_len, is_cuda, device = *q.shape, k.shape[-2], q.is_cuda, q.device
|
||||
|
||||
# Check if there is a compatible device for flash attention
|
||||
|
||||
config = self.cuda_config if is_cuda else self.cpu_config
|
||||
|
||||
# pytorch 2.0 flash attn: q, k, v, mask, dropout, softmax_scale
|
||||
|
||||
with torch.backends.cuda.sdp_kernel(**config._asdict()):
|
||||
out = F.scaled_dot_product_attention(
|
||||
q, k, v,
|
||||
dropout_p = self.dropout if self.training else 0.
|
||||
)
|
||||
|
||||
return out
|
||||
|
||||
def forward(self, q, k, v):
|
||||
"""
|
||||
einstein notation
|
||||
b - batch
|
||||
h - heads
|
||||
n, i, j - sequence length (base sequence length, source, target)
|
||||
d - feature dimension
|
||||
"""
|
||||
|
||||
q_len, k_len, device = q.shape[-2], k.shape[-2], q.device
|
||||
|
||||
scale = self.scale if self.scale is not None else q.shape[-1] ** -0.5
|
||||
|
||||
if self.flash:
|
||||
return self.flash_attn(q, k, v)
|
||||
|
||||
# similarity
|
||||
|
||||
sim = einsum(f"b h i d, b h j d -> b h i j", q, k) * scale
|
||||
|
||||
# attention
|
||||
|
||||
attn = sim.softmax(dim=-1)
|
||||
attn = self.attn_dropout(attn)
|
||||
|
||||
# aggregate values
|
||||
|
||||
out = einsum(f"b h i j, b h j d -> b h i d", attn, v)
|
||||
|
||||
return out
|
||||
Executable
+604
@@ -0,0 +1,604 @@
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
from torch import nn, einsum, Tensor
|
||||
from torch.nn import Module, ModuleList
|
||||
import torch.nn.functional as F
|
||||
|
||||
from .attend import Attend
|
||||
|
||||
from beartype.typing import Tuple, Optional, List, Callable
|
||||
from beartype import beartype
|
||||
|
||||
from rotary_embedding_torch import RotaryEmbedding
|
||||
|
||||
from einops import rearrange, pack, unpack
|
||||
from einops.layers.torch import Rearrange
|
||||
|
||||
"""
|
||||
Code adapted from https://github.com/Anjok07/ultimatevocalremovergui/blob/v5.6.0_roformer_add%2Bdirectml/lib_v5/bs_roformer.py and https://github.com/lucidrains/BS-RoFormer which are copyrighted under the terms of the MIT license.
|
||||
"""
|
||||
|
||||
# helper functions
|
||||
|
||||
def exists(val):
|
||||
return val is not None
|
||||
|
||||
|
||||
def default(v, d):
|
||||
return v if exists(v) else d
|
||||
|
||||
|
||||
def pack_one(t, pattern):
|
||||
return pack([t], pattern)
|
||||
|
||||
|
||||
def unpack_one(t, ps, pattern):
|
||||
return unpack(t, ps, pattern)[0]
|
||||
|
||||
|
||||
# norm
|
||||
|
||||
def l2norm(t):
|
||||
return F.normalize(t, dim = -1, p = 2)
|
||||
|
||||
|
||||
class RMSNorm(Module):
|
||||
def __init__(self, dim):
|
||||
super().__init__()
|
||||
self.scale = dim ** 0.5
|
||||
self.gamma = nn.Parameter(torch.ones(dim))
|
||||
|
||||
def forward(self, x):
|
||||
x = x.to(self.gamma.device)
|
||||
return F.normalize(x, dim=-1) * self.scale * self.gamma
|
||||
|
||||
|
||||
# attention
|
||||
|
||||
class FeedForward(Module):
|
||||
def __init__(
|
||||
self,
|
||||
dim,
|
||||
mult=4,
|
||||
dropout=0.
|
||||
):
|
||||
super().__init__()
|
||||
dim_inner = int(dim * mult)
|
||||
self.net = nn.Sequential(
|
||||
RMSNorm(dim),
|
||||
nn.Linear(dim, dim_inner),
|
||||
nn.GELU(),
|
||||
nn.Dropout(dropout),
|
||||
nn.Linear(dim_inner, dim),
|
||||
nn.Dropout(dropout)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
return self.net(x)
|
||||
|
||||
|
||||
class Attention(Module):
|
||||
def __init__(
|
||||
self,
|
||||
dim,
|
||||
heads=8,
|
||||
dim_head=64,
|
||||
dropout=0.,
|
||||
rotary_embed=None,
|
||||
flash=True
|
||||
):
|
||||
super().__init__()
|
||||
self.heads = heads
|
||||
self.scale = dim_head ** -0.5
|
||||
dim_inner = heads * dim_head
|
||||
|
||||
self.rotary_embed = rotary_embed
|
||||
|
||||
self.attend = Attend(flash=flash, dropout=dropout)
|
||||
|
||||
self.norm = RMSNorm(dim)
|
||||
self.to_qkv = nn.Linear(dim, dim_inner * 3, bias=False)
|
||||
|
||||
self.to_gates = nn.Linear(dim, heads)
|
||||
|
||||
self.to_out = nn.Sequential(
|
||||
nn.Linear(dim_inner, dim, bias=False),
|
||||
nn.Dropout(dropout)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.norm(x)
|
||||
|
||||
q, k, v = rearrange(self.to_qkv(x), 'b n (qkv h d) -> qkv b h n d', qkv=3, h=self.heads)
|
||||
|
||||
if exists(self.rotary_embed):
|
||||
q = self.rotary_embed.rotate_queries_or_keys(q)
|
||||
k = self.rotary_embed.rotate_queries_or_keys(k)
|
||||
|
||||
out = self.attend(q, k, v)
|
||||
|
||||
gates = self.to_gates(x)
|
||||
out = out * rearrange(gates, 'b n h -> b h n 1').sigmoid()
|
||||
|
||||
out = rearrange(out, 'b h n d -> b n (h d)')
|
||||
return self.to_out(out)
|
||||
|
||||
|
||||
class LinearAttention(Module):
|
||||
"""
|
||||
this flavor of linear attention proposed in https://arxiv.org/abs/2106.09681 by El-Nouby et al.
|
||||
"""
|
||||
|
||||
@beartype
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
dim,
|
||||
dim_head=32,
|
||||
heads=8,
|
||||
scale=8,
|
||||
flash=False,
|
||||
dropout=0.
|
||||
):
|
||||
super().__init__()
|
||||
dim_inner = dim_head * heads
|
||||
self.norm = RMSNorm(dim)
|
||||
|
||||
self.to_qkv = nn.Sequential(
|
||||
nn.Linear(dim, dim_inner * 3, bias=False),
|
||||
Rearrange('b n (qkv h d) -> qkv b h d n', qkv=3, h=heads)
|
||||
)
|
||||
|
||||
self.temperature = nn.Parameter(torch.ones(heads, 1, 1))
|
||||
|
||||
self.attend = Attend(
|
||||
scale=scale,
|
||||
dropout=dropout,
|
||||
flash=flash
|
||||
)
|
||||
|
||||
self.to_out = nn.Sequential(
|
||||
Rearrange('b h d n -> b n (h d)'),
|
||||
nn.Linear(dim_inner, dim, bias=False)
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x
|
||||
):
|
||||
x = self.norm(x)
|
||||
|
||||
q, k, v = self.to_qkv(x)
|
||||
|
||||
q, k = map(l2norm, (q, k))
|
||||
q = q * self.temperature.exp()
|
||||
|
||||
out = self.attend(q, k, v)
|
||||
|
||||
return self.to_out(out)
|
||||
|
||||
|
||||
class Transformer(Module):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
dim,
|
||||
depth,
|
||||
dim_head=64,
|
||||
heads=8,
|
||||
attn_dropout=0.,
|
||||
ff_dropout=0.,
|
||||
ff_mult=4,
|
||||
norm_output=True,
|
||||
rotary_embed=None,
|
||||
flash_attn=True,
|
||||
linear_attn=False
|
||||
):
|
||||
super().__init__()
|
||||
self.layers = ModuleList([])
|
||||
|
||||
for _ in range(depth):
|
||||
if linear_attn:
|
||||
attn = LinearAttention(dim=dim, dim_head=dim_head, heads=heads, dropout=attn_dropout, flash=flash_attn)
|
||||
else:
|
||||
attn = Attention(dim=dim, dim_head=dim_head, heads=heads, dropout=attn_dropout,
|
||||
rotary_embed=rotary_embed, flash=flash_attn)
|
||||
|
||||
self.layers.append(ModuleList([
|
||||
attn,
|
||||
FeedForward(dim=dim, mult=ff_mult, dropout=ff_dropout)
|
||||
]))
|
||||
|
||||
self.norm = RMSNorm(dim) if norm_output else nn.Identity()
|
||||
|
||||
def forward(self, x):
|
||||
|
||||
for attn, ff in self.layers:
|
||||
x = attn(x) + x
|
||||
x = ff(x) + x
|
||||
|
||||
return self.norm(x)
|
||||
|
||||
|
||||
# bandsplit module
|
||||
|
||||
class BandSplit(Module):
|
||||
@beartype
|
||||
def __init__(
|
||||
self,
|
||||
dim,
|
||||
dim_inputs: Tuple[int, ...]
|
||||
):
|
||||
super().__init__()
|
||||
self.dim_inputs = dim_inputs
|
||||
self.to_features = ModuleList([])
|
||||
|
||||
for dim_in in dim_inputs:
|
||||
net = nn.Sequential(
|
||||
RMSNorm(dim_in),
|
||||
nn.Linear(dim_in, dim)
|
||||
)
|
||||
|
||||
self.to_features.append(net)
|
||||
|
||||
def forward(self, x):
|
||||
x = x.split(self.dim_inputs, dim=-1)
|
||||
|
||||
outs = []
|
||||
for split_input, to_feature in zip(x, self.to_features):
|
||||
split_output = to_feature(split_input)
|
||||
outs.append(split_output)
|
||||
|
||||
return torch.stack(outs, dim=-2)
|
||||
|
||||
|
||||
def MLP(
|
||||
dim_in,
|
||||
dim_out,
|
||||
dim_hidden=None,
|
||||
depth=1,
|
||||
activation=nn.Tanh
|
||||
):
|
||||
dim_hidden = default(dim_hidden, dim_in)
|
||||
|
||||
net = []
|
||||
dims = (dim_in, *((dim_hidden,) * (depth - 1)), dim_out)
|
||||
|
||||
for ind, (layer_dim_in, layer_dim_out) in enumerate(zip(dims[:-1], dims[1:])):
|
||||
is_last = ind == (len(dims) - 2)
|
||||
|
||||
net.append(nn.Linear(layer_dim_in, layer_dim_out))
|
||||
|
||||
if is_last:
|
||||
continue
|
||||
|
||||
net.append(activation())
|
||||
|
||||
return nn.Sequential(*net)
|
||||
|
||||
|
||||
class MaskEstimator(Module):
|
||||
@beartype
|
||||
def __init__(
|
||||
self,
|
||||
dim,
|
||||
dim_inputs: Tuple[int, ...],
|
||||
depth,
|
||||
mlp_expansion_factor=4
|
||||
):
|
||||
super().__init__()
|
||||
self.dim_inputs = dim_inputs
|
||||
self.to_freqs = ModuleList([])
|
||||
dim_hidden = dim * mlp_expansion_factor
|
||||
|
||||
for dim_in in dim_inputs:
|
||||
net = []
|
||||
|
||||
mlp = nn.Sequential(
|
||||
MLP(dim, dim_in * 2, dim_hidden=dim_hidden, depth=depth),
|
||||
nn.GLU(dim=-1)
|
||||
)
|
||||
|
||||
self.to_freqs.append(mlp)
|
||||
|
||||
def forward(self, x):
|
||||
x = x.unbind(dim=-2)
|
||||
|
||||
outs = []
|
||||
|
||||
for band_features, mlp in zip(x, self.to_freqs):
|
||||
freq_out = mlp(band_features)
|
||||
outs.append(freq_out)
|
||||
|
||||
return torch.cat(outs, dim=-1)
|
||||
|
||||
|
||||
# main class
|
||||
|
||||
DEFAULT_FREQS_PER_BANDS = (
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2,
|
||||
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
|
||||
12, 12, 12, 12, 12, 12, 12, 12,
|
||||
24, 24, 24, 24, 24, 24, 24, 24,
|
||||
48, 48, 48, 48, 48, 48, 48, 48,
|
||||
128, 129,
|
||||
)
|
||||
|
||||
|
||||
class BSRoformer(Module):
|
||||
|
||||
@beartype
|
||||
def __init__(
|
||||
self,
|
||||
dim,
|
||||
*,
|
||||
depth,
|
||||
stereo=False,
|
||||
num_stems=1,
|
||||
time_transformer_depth=2,
|
||||
freq_transformer_depth=2,
|
||||
linear_transformer_depth=0,
|
||||
freqs_per_bands: Tuple[int, ...] = DEFAULT_FREQS_PER_BANDS,
|
||||
# in the paper, they divide into ~60 bands, test with 1 for starters
|
||||
dim_head=64,
|
||||
heads=8,
|
||||
attn_dropout=0.,
|
||||
ff_dropout=0.,
|
||||
flash_attn=True,
|
||||
dim_freqs_in=1025,
|
||||
stft_n_fft=2048,
|
||||
stft_hop_length=512,
|
||||
# 10ms at 44100Hz, from sections 4.1, 4.4 in the paper - @faroit recommends // 2 or // 4 for better reconstruction
|
||||
stft_win_length=2048,
|
||||
stft_normalized=False,
|
||||
stft_window_fn: Optional[Callable] = None,
|
||||
mask_estimator_depth=2,
|
||||
multi_stft_resolution_loss_weight=1.,
|
||||
multi_stft_resolutions_window_sizes: Tuple[int, ...] = (4096, 2048, 1024, 512, 256),
|
||||
multi_stft_hop_size=147,
|
||||
multi_stft_normalized=False,
|
||||
multi_stft_window_fn: Callable = torch.hann_window,
|
||||
# Extra kwargs to ignore from config
|
||||
**kwargs
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.stereo = stereo
|
||||
self.audio_channels = 2 if stereo else 1
|
||||
self.num_stems = num_stems
|
||||
|
||||
self.layers = ModuleList([])
|
||||
|
||||
transformer_kwargs = dict(
|
||||
dim=dim,
|
||||
heads=heads,
|
||||
dim_head=dim_head,
|
||||
attn_dropout=attn_dropout,
|
||||
ff_dropout=ff_dropout,
|
||||
flash_attn=flash_attn,
|
||||
norm_output=False
|
||||
)
|
||||
|
||||
time_rotary_embed = RotaryEmbedding(dim=dim_head)
|
||||
freq_rotary_embed = RotaryEmbedding(dim=dim_head)
|
||||
|
||||
for _ in range(depth):
|
||||
tran_modules = []
|
||||
if linear_transformer_depth > 0:
|
||||
tran_modules.append(Transformer(depth=linear_transformer_depth, linear_attn=True, **transformer_kwargs))
|
||||
tran_modules.append(
|
||||
Transformer(depth=time_transformer_depth, rotary_embed=time_rotary_embed, **transformer_kwargs)
|
||||
)
|
||||
tran_modules.append(
|
||||
Transformer(depth=freq_transformer_depth, rotary_embed=freq_rotary_embed, **transformer_kwargs)
|
||||
)
|
||||
self.layers.append(nn.ModuleList(tran_modules))
|
||||
|
||||
self.final_norm = RMSNorm(dim)
|
||||
|
||||
self.stft_kwargs = dict(
|
||||
n_fft=stft_n_fft,
|
||||
hop_length=stft_hop_length,
|
||||
win_length=stft_win_length,
|
||||
normalized=stft_normalized
|
||||
)
|
||||
|
||||
self.stft_window_fn = partial(default(stft_window_fn, torch.hann_window), stft_win_length)
|
||||
|
||||
freqs = torch.stft(torch.randn(1, 4096), **self.stft_kwargs, return_complex=True).shape[1]
|
||||
|
||||
assert len(freqs_per_bands) > 1
|
||||
assert sum(
|
||||
freqs_per_bands) == freqs, f'the number of freqs in the bands must equal {freqs} based on the STFT settings, but got {sum(freqs_per_bands)}'
|
||||
|
||||
freqs_per_bands_with_complex = tuple(2 * f * self.audio_channels for f in freqs_per_bands)
|
||||
|
||||
self.band_split = BandSplit(
|
||||
dim=dim,
|
||||
dim_inputs=freqs_per_bands_with_complex
|
||||
)
|
||||
|
||||
self.mask_estimators = nn.ModuleList([])
|
||||
|
||||
for _ in range(num_stems):
|
||||
mask_estimator = MaskEstimator(
|
||||
dim=dim,
|
||||
dim_inputs=freqs_per_bands_with_complex,
|
||||
depth=mask_estimator_depth
|
||||
)
|
||||
|
||||
self.mask_estimators.append(mask_estimator)
|
||||
|
||||
# for the multi-resolution stft loss
|
||||
|
||||
self.multi_stft_resolution_loss_weight = multi_stft_resolution_loss_weight
|
||||
self.multi_stft_resolutions_window_sizes = multi_stft_resolutions_window_sizes
|
||||
self.multi_stft_n_fft = stft_n_fft
|
||||
self.multi_stft_window_fn = multi_stft_window_fn
|
||||
|
||||
self.multi_stft_kwargs = dict(
|
||||
hop_length=multi_stft_hop_size,
|
||||
normalized=multi_stft_normalized
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
raw_audio,
|
||||
target=None,
|
||||
return_loss_breakdown=False
|
||||
):
|
||||
"""
|
||||
einops
|
||||
|
||||
b - batch
|
||||
f - freq
|
||||
t - time
|
||||
s - audio channel (1 for mono, 2 for stereo)
|
||||
n - number of 'stems'
|
||||
c - complex (2)
|
||||
d - feature dimension
|
||||
"""
|
||||
|
||||
original_device = raw_audio.device
|
||||
|
||||
x_is_mps = True if original_device.type == 'mps' else False
|
||||
|
||||
if x_is_mps:
|
||||
raw_audio = raw_audio.cpu()
|
||||
|
||||
device = raw_audio.device
|
||||
|
||||
if raw_audio.ndim == 2:
|
||||
raw_audio = rearrange(raw_audio, 'b t -> b 1 t')
|
||||
|
||||
channels = raw_audio.shape[1]
|
||||
assert (not self.stereo and channels == 1) or (
|
||||
self.stereo and channels == 2), 'stereo needs to be set to True if passing in audio signal that is stereo (channel dimension of 2). also need to be False if mono (channel dimension of 1)'
|
||||
|
||||
# to stft
|
||||
|
||||
raw_audio, batch_audio_channel_packed_shape = pack_one(raw_audio, '* t')
|
||||
|
||||
stft_window = self.stft_window_fn(device=device)
|
||||
|
||||
stft_repr = torch.stft(raw_audio, **self.stft_kwargs, window=stft_window, return_complex=True)
|
||||
stft_repr = torch.view_as_real(stft_repr)
|
||||
|
||||
stft_repr = unpack_one(stft_repr, batch_audio_channel_packed_shape, '* f t c')
|
||||
stft_repr = rearrange(stft_repr,
|
||||
'b s f t c -> b (f s) t c') # merge stereo / mono into the frequency, with frequency leading dimension, for band splitting
|
||||
|
||||
x = rearrange(stft_repr, 'b f t c -> b t (f c)')
|
||||
|
||||
x = self.band_split(x)
|
||||
|
||||
# axial / hierarchical attention
|
||||
|
||||
for transformer_block in self.layers:
|
||||
|
||||
if len(transformer_block) == 3:
|
||||
linear_transformer, time_transformer, freq_transformer = transformer_block
|
||||
|
||||
x, ft_ps = pack([x], 'b * d')
|
||||
x = linear_transformer(x)
|
||||
x, = unpack(x, ft_ps, 'b * d')
|
||||
else:
|
||||
time_transformer, freq_transformer = transformer_block
|
||||
|
||||
x = rearrange(x, 'b t f d -> b f t d')
|
||||
x, ps = pack([x], '* t d')
|
||||
|
||||
x = time_transformer(x)
|
||||
|
||||
x, = unpack(x, ps, '* t d')
|
||||
x = rearrange(x, 'b f t d -> b t f d')
|
||||
x, ps = pack([x], '* f d')
|
||||
|
||||
x = freq_transformer(x)
|
||||
|
||||
x, = unpack(x, ps, '* f d')
|
||||
|
||||
x = self.final_norm(x)
|
||||
|
||||
num_stems = len(self.mask_estimators)
|
||||
|
||||
mask = torch.stack([fn(x) for fn in self.mask_estimators], dim=1)
|
||||
mask = rearrange(mask, 'b n t (f c) -> b n f t c', c=2)
|
||||
|
||||
if x_is_mps:
|
||||
mask = mask.to('cpu')
|
||||
|
||||
# modulate frequency representation
|
||||
|
||||
stft_repr = rearrange(stft_repr, 'b f t c -> b 1 f t c')
|
||||
|
||||
# complex number multiplication
|
||||
|
||||
stft_repr = torch.view_as_complex(stft_repr)
|
||||
mask = torch.view_as_complex(mask)
|
||||
|
||||
stft_repr = stft_repr * mask
|
||||
|
||||
# istft
|
||||
|
||||
stft_repr = rearrange(stft_repr, 'b n (f s) t -> (b n s) f t', s=self.audio_channels)
|
||||
|
||||
recon_audio = torch.istft(stft_repr, **self.stft_kwargs, window=stft_window, return_complex=False)
|
||||
|
||||
recon_audio = rearrange(recon_audio, '(b n s) t -> b n s t', s=self.audio_channels, n=num_stems)
|
||||
|
||||
if num_stems == 1:
|
||||
recon_audio = rearrange(recon_audio, 'b 1 s t -> b s t')
|
||||
|
||||
# if a target is passed in, calculate loss for learning
|
||||
|
||||
if not exists(target):
|
||||
return recon_audio
|
||||
|
||||
if self.num_stems > 1:
|
||||
assert target.ndim == 4 and target.shape[1] == self.num_stems
|
||||
|
||||
if target.ndim == 2:
|
||||
target = rearrange(target, '... t -> ... 1 t')
|
||||
|
||||
target = target[..., :recon_audio.shape[-1]]
|
||||
|
||||
loss = F.l1_loss(recon_audio, target)
|
||||
|
||||
multi_stft_resolution_loss = 0.
|
||||
|
||||
for window_size in self.multi_stft_resolutions_window_sizes:
|
||||
res_stft_kwargs = dict(
|
||||
n_fft=max(window_size, self.multi_stft_n_fft),
|
||||
win_length=window_size,
|
||||
return_complex=True,
|
||||
window=self.multi_stft_window_fn(window_size, device=device),
|
||||
**self.multi_stft_kwargs,
|
||||
)
|
||||
|
||||
recon_Y = torch.stft(rearrange(recon_audio, '... s t -> (... s) t'), **res_stft_kwargs)
|
||||
target_Y = torch.stft(rearrange(target, '... s t -> (... s) t'), **res_stft_kwargs)
|
||||
|
||||
multi_stft_resolution_loss = multi_stft_resolution_loss + F.l1_loss(recon_Y, target_Y)
|
||||
|
||||
weighted_multi_resolution_loss = multi_stft_resolution_loss * self.multi_stft_resolution_loss_weight
|
||||
|
||||
total_loss = loss + weighted_multi_resolution_loss
|
||||
|
||||
|
||||
if not return_loss_breakdown:
|
||||
# Move the result back to the original device if it was moved to CPU for MPS compatibility
|
||||
if x_is_mps:
|
||||
total_loss = total_loss.to(original_device)
|
||||
return total_loss
|
||||
|
||||
# For detailed loss breakdown, ensure all components are moved back to the original device for MPS
|
||||
if x_is_mps:
|
||||
loss = loss.to(original_device)
|
||||
multi_stft_resolution_loss = multi_stft_resolution_loss.to(original_device)
|
||||
weighted_multi_resolution_loss = weighted_multi_resolution_loss.to(original_device)
|
||||
|
||||
return total_loss, (loss, multi_stft_resolution_loss)
|
||||
+258
@@ -0,0 +1,258 @@
|
||||
"""
|
||||
BS-RoFormer Chorus separator for male/female vocals separation.
|
||||
Uses the Chorus_Male_Female_BS_Roformer model.
|
||||
"""
|
||||
import gc
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import yaml
|
||||
from omegaconf import OmegaConf
|
||||
from tqdm import tqdm
|
||||
|
||||
from .bs_roformer import BSRoformer
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
# HuggingFace model coordinates
|
||||
HF_REPO_ID = 'Sucial/Chorus_Male_Female_BS_Roformer'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
|
||||
def _try_download_model(model_dir: Path, model_path: Path, config_path: Path) -> None:
|
||||
"""Download model from HuggingFace if not present."""
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
from huggingface_hub import snapshot_download
|
||||
snapshot_download(
|
||||
repo_id=HF_REPO_ID,
|
||||
revision=HF_REVISION,
|
||||
local_dir=str(model_dir),
|
||||
)
|
||||
|
||||
|
||||
def _load_audio(path: str) -> np.ndarray:
|
||||
"""Load audio as (channels, samples) float32 array."""
|
||||
import soundfile as sf
|
||||
|
||||
try:
|
||||
data, sr = sf.read(path, dtype='float32', always_2d=True)
|
||||
data = data.T
|
||||
except Exception:
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', path, '-ar', str(SAMPLE_RATE),
|
||||
'-ac', '2', '-f', 'wav', tmp_path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
data, sr = sf.read(tmp_path, dtype='float32', always_2d=True)
|
||||
data = data.T
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
if sr != SAMPLE_RATE:
|
||||
import librosa
|
||||
data = np.array([
|
||||
librosa.resample(ch, orig_sr=sr, target_sr=SAMPLE_RATE)
|
||||
for ch in data
|
||||
], dtype=np.float32)
|
||||
|
||||
if data.shape[0] == 1:
|
||||
data = np.vstack([data, data])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def _save_audio(path: str, data: np.ndarray, output_format: str, bitrate: str = '256k') -> None:
|
||||
"""Save (samples, channels) array to audio file."""
|
||||
import soundfile as sf
|
||||
|
||||
ext = output_format.lower()
|
||||
|
||||
if ext in ('wav', 'flac'):
|
||||
sf.write(path, data, SAMPLE_RATE, format=ext.upper(),
|
||||
subtype='PCM_24' if ext == 'wav' else 'PCM_16')
|
||||
else:
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
sf.write(tmp_path, data, SAMPLE_RATE, format='WAV', subtype='PCM_16')
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', tmp_path, '-b:a', bitrate, path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
|
||||
class BSRoformerChorusSeparator:
|
||||
"""
|
||||
BS-RoFormer separator for male/female chorus separation.
|
||||
Separates male and female voices in choir singing.
|
||||
|
||||
Note: This is an experimental model trained primarily on Chinese songs.
|
||||
Works best when male and female sing simultaneously (not alternately).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
):
|
||||
self.model_dir = Path(model_dir) if model_dir else Path('/opt/bs_roformer_models/chorus')
|
||||
self.model_path = self.model_dir / 'model_chorus_bs_roformer_ep_267_sdr_24.1275.ckpt'
|
||||
self.config_path = self.model_dir / 'config.yaml'
|
||||
self.device = 'cpu' if cpu_separation else 'cuda'
|
||||
self.output_format = output_format.lower()
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self._config = None
|
||||
|
||||
def _ensure_model(self):
|
||||
_try_download_model(self.model_dir, self.model_path, self.config_path)
|
||||
|
||||
def _load_config(self):
|
||||
if self._config is None:
|
||||
with open(self.config_path, 'r') as f:
|
||||
config = yaml.load(f, Loader=yaml.UnsafeLoader)
|
||||
self._config = OmegaConf.create(config)
|
||||
return self._config
|
||||
|
||||
def get_model(self) -> BSRoformer:
|
||||
self._ensure_model()
|
||||
config = self._load_config()
|
||||
model_cfg = OmegaConf.to_container(config.model, resolve=True)
|
||||
|
||||
# Convert lists to tuples where needed
|
||||
for key in ('freqs_per_bands', 'multi_stft_resolutions_window_sizes'):
|
||||
if key in model_cfg and isinstance(model_cfg[key], list):
|
||||
model_cfg[key] = tuple(model_cfg[key])
|
||||
|
||||
model = BSRoformer(**model_cfg)
|
||||
checkpoint = torch.load(str(self.model_path), map_location='cpu', weights_only=False)
|
||||
|
||||
if 'state_dict' in checkpoint:
|
||||
state_dict = checkpoint['state_dict']
|
||||
elif 'model' in checkpoint:
|
||||
state_dict = checkpoint['model']
|
||||
else:
|
||||
state_dict = checkpoint
|
||||
|
||||
model.load_state_dict(state_dict)
|
||||
model.to(self.device).eval()
|
||||
return model
|
||||
|
||||
def _demix(self, mix: np.ndarray, model: BSRoformer,
|
||||
batch_size_override: Optional[int] = None,
|
||||
num_overlap_override: Optional[int] = None) -> Dict[str, np.ndarray]:
|
||||
"""Separate into male and female vocals."""
|
||||
config = self._load_config()
|
||||
hop_length = config.model.stft_hop_length
|
||||
segment_size = getattr(config.inference, 'dim_t', 2000)
|
||||
C = hop_length * (segment_size - 1)
|
||||
N = num_overlap_override if num_overlap_override is not None else getattr(config.inference, 'num_overlap', 4)
|
||||
step = int(C // N)
|
||||
fade_size = C // 10
|
||||
batch_size = batch_size_override if batch_size_override is not None else getattr(config.inference, 'batch_size', 1)
|
||||
device = self.device
|
||||
|
||||
mix_t = torch.tensor(mix, dtype=torch.float32)
|
||||
length_init = mix_t.shape[-1]
|
||||
|
||||
if length_init > 2 * (C - step) and (C - step > 0):
|
||||
mix_t = nn.functional.pad(mix_t, (C - step, C - step), mode='reflect')
|
||||
|
||||
fadein = torch.linspace(0, 1, fade_size).to(device)
|
||||
fadeout = torch.linspace(1, 0, fade_size).to(device)
|
||||
window_start = torch.ones(C).to(device)
|
||||
window_start[-fade_size:] *= fadeout
|
||||
window_finish = torch.ones(C).to(device)
|
||||
window_finish[:fade_size] *= fadein
|
||||
window_middle = torch.ones(C).to(device)
|
||||
window_middle[:fade_size] *= fadein
|
||||
window_middle[-fade_size:] *= fadeout
|
||||
|
||||
# This model outputs: instrumental, male, female (3 stems)
|
||||
stems = ['instrumental', 'male', 'female']
|
||||
num_stems = len(stems)
|
||||
req_shape = (num_stems,) + tuple(mix_t.shape)
|
||||
|
||||
with torch.inference_mode():
|
||||
result = torch.zeros(req_shape, dtype=torch.float32, device=device)
|
||||
counter = torch.zeros(req_shape, dtype=torch.float32, device=device)
|
||||
batch_data, batch_locs = [], []
|
||||
i = 0
|
||||
|
||||
total_seg = sum(1 for _ in range(0, mix_t.shape[1], step))
|
||||
|
||||
with tqdm(total=total_seg, desc='Separating', unit='segment', ncols=100) as pbar:
|
||||
while i < mix_t.shape[1]:
|
||||
part = mix_t[:, i:i + C].to(device)
|
||||
l = part.shape[-1]
|
||||
if l < C:
|
||||
part = nn.functional.pad(
|
||||
part,
|
||||
(0, C - l) if l > C // 2 + 1 else (0, C - l, 0, 0),
|
||||
mode='reflect' if l > C // 2 + 1 else 'constant',
|
||||
)
|
||||
batch_data.append(part)
|
||||
batch_locs.append((i, l))
|
||||
i += step
|
||||
|
||||
if len(batch_data) >= batch_size or i >= mix_t.shape[1]:
|
||||
arr = torch.stack(batch_data, dim=0)
|
||||
x = model(arr)
|
||||
for j, (start, ln) in enumerate(batch_locs):
|
||||
window = (window_start if start == 0
|
||||
else window_finish if i >= mix_t.shape[1]
|
||||
else window_middle)
|
||||
result[..., start:start + ln] += x[j][..., :ln] * window[..., :ln]
|
||||
counter[..., start:start + ln] += window[..., :ln]
|
||||
pbar.update(len(batch_data))
|
||||
batch_data, batch_locs = [], []
|
||||
|
||||
estimated = result / counter.clamp(min=1e-10)
|
||||
if length_init > 2 * (C - step) and (C - step > 0):
|
||||
estimated = estimated[..., (C - step):-(C - step)]
|
||||
|
||||
sources = {}
|
||||
for idx, name in enumerate(stems):
|
||||
sources[name] = estimated[idx].cpu().numpy()
|
||||
return sources
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str,
|
||||
batch_size_override: Optional[int] = None,
|
||||
num_overlap_override: Optional[int] = None) -> Dict[str, str]:
|
||||
"""Separate audio and write stems."""
|
||||
output_dir = Path(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
audio = _load_audio(input_path)
|
||||
model = self.get_model()
|
||||
sources = self._demix(audio, model,
|
||||
batch_size_override=batch_size_override,
|
||||
num_overlap_override=num_overlap_override)
|
||||
|
||||
del model
|
||||
torch.cuda.empty_cache()
|
||||
gc.collect()
|
||||
|
||||
ext = self.output_format
|
||||
result = {}
|
||||
for name, data in sources.items():
|
||||
dest = str(output_dir / f'{name}.{ext}')
|
||||
_save_audio(dest, data.T, ext, self.mp3_bitrate)
|
||||
result[name] = dest
|
||||
return result
|
||||
Executable
+322
@@ -0,0 +1,322 @@
|
||||
"""
|
||||
Standalone BS-RoFormer separator — no Django, no Spleeter/TensorFlow dependency.
|
||||
|
||||
Audio I/O uses soundfile (WAV/FLAC) + subprocess ffmpeg (MP3 encoding).
|
||||
Model architecture is unchanged from the original spleeter-web implementation.
|
||||
"""
|
||||
import gc
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import yaml
|
||||
from omegaconf import OmegaConf
|
||||
from tqdm import tqdm
|
||||
|
||||
from .bs_roformer import BSRoformer
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HuggingFace model coordinates
|
||||
# ---------------------------------------------------------------------------
|
||||
HF_REPO_ID = 'jarredou/BS-ROFO-SW-Fixed'
|
||||
HF_REVISION = 'f99fc5c2e4bc201413cfc1ca5767cf62ddb1a9a4'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Default model paths (overridable via env var BS_ROFORMER_MODEL_DIR)
|
||||
# ---------------------------------------------------------------------------
|
||||
_MODEL_DIR_ENV = os.environ.get('BS_ROFORMER_MODEL_DIR', '/opt/bs_roformer_models')
|
||||
DEFAULT_MODEL_DIR = Path(_MODEL_DIR_ENV)
|
||||
DEFAULT_MODEL_PATH = DEFAULT_MODEL_DIR / 'BS-Rofo-SW-Fixed.ckpt'
|
||||
DEFAULT_CONFIG_PATH = DEFAULT_MODEL_DIR / 'BS-Rofo-SW-Fixed.yaml'
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _try_download_model(
|
||||
model_dir: Path = DEFAULT_MODEL_DIR,
|
||||
model_path: Path = DEFAULT_MODEL_PATH,
|
||||
config_path: Path = DEFAULT_CONFIG_PATH,
|
||||
) -> None:
|
||||
"""Download BS-RoFormer model from HuggingFace if not already present."""
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
from huggingface_hub import snapshot_download
|
||||
snapshot_download(
|
||||
repo_id=HF_REPO_ID,
|
||||
revision=HF_REVISION,
|
||||
local_dir=str(model_dir),
|
||||
)
|
||||
|
||||
|
||||
def _load_config(config_path: Path):
|
||||
"""Load YAML config, supporting !!python/tuple tags."""
|
||||
with open(config_path, 'r') as f:
|
||||
config = yaml.load(f, Loader=yaml.UnsafeLoader)
|
||||
return OmegaConf.create(config)
|
||||
|
||||
|
||||
def _load_audio(path: str) -> np.ndarray:
|
||||
"""
|
||||
Load audio file and return as float32 numpy array of shape (channels, samples).
|
||||
Uses soundfile for WAV/FLAC; falls back to ffmpeg decode for MP3 and other formats.
|
||||
"""
|
||||
import soundfile as sf
|
||||
|
||||
try:
|
||||
data, sr = sf.read(path, dtype='float32', always_2d=True)
|
||||
# soundfile returns (samples, channels); transpose to (channels, samples)
|
||||
data = data.T
|
||||
except Exception:
|
||||
# Fallback: decode via ffmpeg to a temporary WAV
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', path, '-ar', str(SAMPLE_RATE),
|
||||
'-ac', '2', '-f', 'wav', tmp_path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
data, sr = sf.read(tmp_path, dtype='float32', always_2d=True)
|
||||
data = data.T
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
# Resample if needed
|
||||
if sr != SAMPLE_RATE:
|
||||
import librosa
|
||||
data = np.array([
|
||||
librosa.resample(ch, orig_sr=sr, target_sr=SAMPLE_RATE)
|
||||
for ch in data
|
||||
], dtype=np.float32)
|
||||
|
||||
# Ensure stereo
|
||||
if data.shape[0] == 1:
|
||||
data = np.vstack([data, data])
|
||||
|
||||
return data # (channels, samples)
|
||||
|
||||
|
||||
def _save_audio(path: str, data: np.ndarray, output_format: str, bitrate: str = '256k') -> None:
|
||||
"""
|
||||
Save (samples, channels) float32 array as audio file.
|
||||
WAV and FLAC use soundfile directly; MP3 encodes via ffmpeg.
|
||||
"""
|
||||
import soundfile as sf
|
||||
|
||||
ext = output_format.lower()
|
||||
|
||||
if ext in ('wav', 'flac'):
|
||||
sf.write(path, data, SAMPLE_RATE, format=ext.upper(),
|
||||
subtype='PCM_24' if ext == 'wav' else 'PCM_16')
|
||||
else:
|
||||
# Encode to MP3 via ffmpeg
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
sf.write(tmp_path, data, SAMPLE_RATE, format='WAV', subtype='PCM_16')
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', tmp_path,
|
||||
'-b:a', bitrate, path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Separator class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class BSRoformerSeparator:
|
||||
"""
|
||||
Performs source separation using the BS-RoFormer model.
|
||||
Pure PyTorch — no TensorFlow / Spleeter dependency.
|
||||
"""
|
||||
|
||||
STEM_MODE_4 = '4stem'
|
||||
STEM_MODE_5_GUITAR = '5stem_guitar'
|
||||
STEM_MODE_5_PIANO = '5stem_piano'
|
||||
STEM_MODE_6 = '6stem'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_path: Optional[str] = None,
|
||||
config_path: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3', # 'mp3' | 'flac' | 'wav'
|
||||
mp3_bitrate: str = '256k',
|
||||
stem_mode: str = '4stem',
|
||||
):
|
||||
self.model_path = Path(model_path) if model_path else DEFAULT_MODEL_PATH
|
||||
self.config_path = Path(config_path) if config_path else DEFAULT_CONFIG_PATH
|
||||
self.device = 'cpu' if cpu_separation else 'cuda'
|
||||
self.output_format = output_format.lower()
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self.stem_mode = stem_mode
|
||||
self._config = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Model loading
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _ensure_model(self):
|
||||
_try_download_model(DEFAULT_MODEL_DIR, self.model_path, self.config_path)
|
||||
|
||||
def _load_config(self):
|
||||
if self._config is None:
|
||||
self._config = _load_config(self.config_path)
|
||||
return self._config
|
||||
|
||||
def get_model(self) -> BSRoformer:
|
||||
self._ensure_model()
|
||||
config = self._load_config()
|
||||
model_cfg = OmegaConf.to_container(config.model, resolve=True)
|
||||
|
||||
# OmegaConf loads tuples as lists; the model requires tuples
|
||||
for key in ('freqs_per_bands', 'multi_stft_resolutions_window_sizes'):
|
||||
if key in model_cfg and isinstance(model_cfg[key], list):
|
||||
model_cfg[key] = tuple(model_cfg[key])
|
||||
|
||||
model = BSRoformer(**model_cfg)
|
||||
checkpoint = torch.load(str(self.model_path), map_location='cpu', weights_only=False)
|
||||
model.load_state_dict(checkpoint)
|
||||
model.to(self.device).eval()
|
||||
return model
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Inference
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _demix(self, mix: np.ndarray, model: BSRoformer,
|
||||
batch_size_override: Optional[int] = None,
|
||||
num_overlap_override: Optional[int] = None) -> Dict[str, np.ndarray]:
|
||||
"""Separate (channels, samples) array into a dict of stem arrays."""
|
||||
config = self._load_config()
|
||||
hop_length = config.model.stft_hop_length
|
||||
segment_size = config.inference.dim_t
|
||||
C = hop_length * (segment_size - 1)
|
||||
N = num_overlap_override if num_overlap_override is not None else config.inference.num_overlap
|
||||
step = int(C // N)
|
||||
fade_size = C // 10
|
||||
batch_size = batch_size_override if batch_size_override is not None else config.inference.batch_size
|
||||
device = self.device
|
||||
|
||||
mix_t = torch.tensor(mix, dtype=torch.float32)
|
||||
length_init = mix_t.shape[-1]
|
||||
|
||||
if length_init > 2 * (C - step) and (C - step > 0):
|
||||
mix_t = nn.functional.pad(mix_t, (C - step, C - step), mode='reflect')
|
||||
|
||||
fadein = torch.linspace(0, 1, fade_size).to(device)
|
||||
fadeout = torch.linspace(1, 0, fade_size).to(device)
|
||||
window_start = torch.ones(C).to(device); window_start[-fade_size:] *= fadeout
|
||||
window_finish = torch.ones(C).to(device); window_finish[:fade_size] *= fadein
|
||||
window_middle = torch.ones(C).to(device)
|
||||
window_middle[:fade_size] *= fadein; window_middle[-fade_size:] *= fadeout
|
||||
|
||||
S = len(config.training.instruments)
|
||||
req_shape = (S,) + tuple(mix_t.shape)
|
||||
|
||||
with torch.inference_mode():
|
||||
result = torch.zeros(req_shape, dtype=torch.float32, device=device)
|
||||
counter = torch.zeros(req_shape, dtype=torch.float32, device=device)
|
||||
batch_data, batch_locs = [], []
|
||||
i = 0
|
||||
|
||||
# count total segments for tqdm
|
||||
total_seg = sum(1 for _ in range(0, mix_t.shape[1], step))
|
||||
|
||||
with tqdm(total=total_seg, desc='Separating', unit='segment', ncols=100) as pbar:
|
||||
while i < mix_t.shape[1]:
|
||||
part = mix_t[:, i:i + C].to(device)
|
||||
l = part.shape[-1]
|
||||
if l < C:
|
||||
part = nn.functional.pad(
|
||||
part,
|
||||
(0, C - l) if l > C // 2 + 1 else (0, C - l, 0, 0),
|
||||
mode='reflect' if l > C // 2 + 1 else 'constant',
|
||||
)
|
||||
batch_data.append(part)
|
||||
batch_locs.append((i, l))
|
||||
i += step
|
||||
|
||||
if len(batch_data) >= batch_size or i >= mix_t.shape[1]:
|
||||
arr = torch.stack(batch_data, dim=0)
|
||||
x = model(arr)
|
||||
for j, (start, ln) in enumerate(batch_locs):
|
||||
window = (window_start if start == 0
|
||||
else window_finish if i >= mix_t.shape[1]
|
||||
else window_middle)
|
||||
result[..., start:start + ln] += x[j][..., :ln] * window[..., :ln]
|
||||
counter[..., start:start + ln] += window[..., :ln]
|
||||
pbar.update(len(batch_data))
|
||||
batch_data, batch_locs = [], []
|
||||
|
||||
estimated = result / counter.clamp(min=1e-10)
|
||||
if length_init > 2 * (C - step) and (C - step > 0):
|
||||
estimated = estimated[..., (C - step):-(C - step)]
|
||||
|
||||
sources = {}
|
||||
for idx, name in enumerate(config.training.instruments):
|
||||
sources[name] = estimated[idx].cpu().numpy()
|
||||
return sources
|
||||
|
||||
def _merge_stems(self, sources_6: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]:
|
||||
"""Map 6-stem model output to the requested stem_mode output."""
|
||||
v, d, b, o, g, p = (
|
||||
sources_6['vocals'], sources_6['drums'], sources_6['bass'],
|
||||
sources_6['other'], sources_6['guitar'], sources_6['piano'],
|
||||
)
|
||||
if self.stem_mode == self.STEM_MODE_4:
|
||||
return {'vocals': v, 'drums': d, 'bass': b, 'other': o + g + p}
|
||||
if self.stem_mode == self.STEM_MODE_5_GUITAR:
|
||||
return {'vocals': v, 'drums': d, 'bass': b, 'guitar': g, 'other': o + p}
|
||||
if self.stem_mode == self.STEM_MODE_5_PIANO:
|
||||
return {'vocals': v, 'drums': d, 'bass': b, 'piano': p, 'other': o + g}
|
||||
# 6stem — return all
|
||||
return sources_6
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str,
|
||||
batch_size_override: Optional[int] = None,
|
||||
num_overlap_override: Optional[int] = None) -> Dict[str, str]:
|
||||
"""
|
||||
Separate audio at input_path and write one file per stem into output_dir.
|
||||
Returns {stem_name: output_file_path}.
|
||||
"""
|
||||
output_dir = Path(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
audio = _load_audio(input_path) # (channels, samples)
|
||||
model = self.get_model()
|
||||
sources_6 = self._demix(audio, model,
|
||||
batch_size_override=batch_size_override,
|
||||
num_overlap_override=num_overlap_override)
|
||||
|
||||
del model
|
||||
torch.cuda.empty_cache()
|
||||
gc.collect()
|
||||
|
||||
stems = self._merge_stems(sources_6)
|
||||
ext = self.output_format
|
||||
result = {}
|
||||
for name, data in stems.items():
|
||||
dest = str(output_dir / f'{name}.{ext}')
|
||||
_save_audio(dest, data.T, ext, self.mp3_bitrate) # T: (channels,samples)→(samples,channels)
|
||||
result[name] = dest
|
||||
return result
|
||||
Executable
+106
@@ -0,0 +1,106 @@
|
||||
"""
|
||||
BS-RoFormer SW separator - Swin Transformer variant for 6-stem separation
|
||||
Models from mvsep.com / audio-separator
|
||||
"""
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
try:
|
||||
from audio_separator.separator import Separator
|
||||
AUDIO_SEPARATOR_AVAILABLE = True
|
||||
except ImportError:
|
||||
AUDIO_SEPARATOR_AVAILABLE = False
|
||||
|
||||
from ..config import BS_ROFORMER_MODEL_DIR, CPU_SEPARATION
|
||||
|
||||
|
||||
class BSRoformerSWSeparator:
|
||||
"""
|
||||
BS-RoFormer SW (Swin Transformer) separator.
|
||||
|
||||
This variant uses Swin Transformer architecture and provides
|
||||
6-stem separation: vocals, drums, bass, guitar, piano, other.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
cpu_separation: bool = CPU_SEPARATION,
|
||||
output_format: str = "mp3",
|
||||
mp3_bitrate: str = "256k",
|
||||
):
|
||||
if not AUDIO_SEPARATOR_AVAILABLE:
|
||||
raise ImportError(
|
||||
"audio-separator is required for BS-RoFormer SW separation. "
|
||||
"Install with: pip install audio-separator"
|
||||
)
|
||||
|
||||
self.cpu_separation = cpu_separation
|
||||
self.output_format = output_format
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self._separator = None
|
||||
|
||||
def _get_separator(self):
|
||||
"""Lazy-load the separator with SW model."""
|
||||
if self._separator is None:
|
||||
# BS-RoFormer SW model is typically available in audio-separator
|
||||
# as a pre-configured model
|
||||
self._separator = Separator(
|
||||
model_file_dir=str(BS_ROFORMER_MODEL_DIR),
|
||||
output_dir=None,
|
||||
output_format=self.output_format,
|
||||
output_bitrate=self.mp3_bitrate if self.output_format == "mp3" else None,
|
||||
normalization_threshold=0.9,
|
||||
enable_denoise=False,
|
||||
cpu_separation=self.cpu_separation,
|
||||
)
|
||||
|
||||
# Load the SW model - this may need specific model name
|
||||
# audio-separator typically provides this as "model_bs_roformer_sw"
|
||||
try:
|
||||
self._separator.load_model()
|
||||
except Exception as e:
|
||||
# Fallback: try to load specific model file
|
||||
sw_model_path = BS_ROFORMER_MODEL_DIR / "bs_roformer_sw.ckpt"
|
||||
if sw_model_path.exists():
|
||||
self._separator.load_model(str(sw_model_path))
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"BS-RoFormer SW model not found. Please download it first. Error: {e}"
|
||||
)
|
||||
|
||||
return self._separator
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
||||
"""
|
||||
Separate audio into 6 stems.
|
||||
|
||||
Returns:
|
||||
Dict mapping stem names to output file paths
|
||||
Stems: vocals, drums, bass, guitar, piano, other
|
||||
"""
|
||||
sep = self._get_separator()
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Run separation
|
||||
result = sep.separate(input_path)
|
||||
|
||||
# Map outputs to 6 stems
|
||||
stems = ["vocals", "drums", "bass", "guitar", "piano", "other"]
|
||||
output_paths = {}
|
||||
|
||||
if isinstance(result, list):
|
||||
for i, path in enumerate(result):
|
||||
if i < len(stems):
|
||||
stem_name = stems[i]
|
||||
final_path = output_path / f"{stem_name}.{self.output_format}"
|
||||
Path(path).rename(final_path)
|
||||
output_paths[stem_name] = str(final_path)
|
||||
else:
|
||||
# Fallback - single output
|
||||
final_path = output_path / f"vocals.{self.output_format}"
|
||||
Path(result).rename(final_path)
|
||||
output_paths["vocals"] = str(final_path)
|
||||
|
||||
return output_paths
|
||||
Executable
+263
@@ -0,0 +1,263 @@
|
||||
"""
|
||||
Ensemble separator - Combines multiple models for best quality separation
|
||||
Uses model averaging/ensembling from multiple high-quality models.
|
||||
"""
|
||||
import os
|
||||
import torch
|
||||
import torchaudio
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
import tempfile
|
||||
import shutil
|
||||
|
||||
from ..config import ENSEMBLE_CACHE_DIR, CPU_SEPARATION
|
||||
|
||||
|
||||
# Ensemble configurations
|
||||
ENSEMBLE_CONFIGS = {
|
||||
"ensemble_vocal_best": {
|
||||
"description": "Best vocal separation using BS-RoFormer x2 + MelBand RoFormer ft + SCNet XL IHF",
|
||||
"models": [
|
||||
{"type": "bs_roformer", "weight": 0.30, "stem_mode": "4stem"},
|
||||
{"type": "mel_band", "weight": 0.25, "variant": "vocals"},
|
||||
{"type": "scnet_xl_ihf", "weight": 0.45},
|
||||
],
|
||||
"stems": ["vocals", "instrumental"],
|
||||
},
|
||||
"ensemble_4stem": {
|
||||
"description": "Best 4-stem separation using multiple models averaged",
|
||||
"models": [
|
||||
{"type": "demucs", "weight": 0.20, "model": "htdemucs_ft"},
|
||||
{"type": "bs_roformer", "weight": 0.30, "stem_mode": "4stem"},
|
||||
{"type": "mel_band", "weight": 0.25, "variant": "4stem"},
|
||||
{"type": "scnet_large", "weight": 0.25},
|
||||
],
|
||||
"stems": ["vocals", "drums", "bass", "other"],
|
||||
},
|
||||
"ensemble_6stem": {
|
||||
"description": "Best 6-stem separation using multiple models averaged",
|
||||
"models": [
|
||||
{"type": "bs_roformer", "weight": 0.35, "stem_mode": "6stem"},
|
||||
{"type": "bs_roformer_sw", "weight": 0.35},
|
||||
{"type": "demucs", "weight": 0.30, "model": "htdemucs_6s"},
|
||||
],
|
||||
"stems": ["vocals", "drums", "bass", "guitar", "piano", "other"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class EnsembleSeparator:
|
||||
"""
|
||||
Ensemble separator that combines multiple models for best quality.
|
||||
|
||||
Uses weighted averaging of outputs from multiple separation models
|
||||
to produce higher quality results than any single model alone.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ensemble_id: str = "ensemble_vocal_best",
|
||||
cpu_separation: bool = CPU_SEPARATION,
|
||||
output_format: str = "mp3",
|
||||
mp3_bitrate: str = "256k",
|
||||
):
|
||||
if ensemble_id not in ENSEMBLE_CONFIGS:
|
||||
raise ValueError(f"Unknown ensemble config: {ensemble_id}")
|
||||
|
||||
self.ensemble_id = ensemble_id
|
||||
self.config = ENSEMBLE_CONFIGS[ensemble_id]
|
||||
self.cpu_separation = cpu_separation
|
||||
self.output_format = output_format
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self.cache_dir = Path(ENSEMBLE_CACHE_DIR)
|
||||
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _load_audio(self, path: str) -> tuple:
|
||||
"""Load audio file and return waveform and sample rate."""
|
||||
waveform, sr = torchaudio.load(path)
|
||||
return waveform, sr
|
||||
|
||||
def _save_audio(self, waveform: torch.Tensor, sr: int, path: str):
|
||||
"""Save audio waveform to file."""
|
||||
torchaudio.save(path, waveform, sr)
|
||||
|
||||
def _weighted_average(self, audio_tensors: List[torch.Tensor], weights: List[float]) -> torch.Tensor:
|
||||
"""Compute weighted average of audio tensors."""
|
||||
if len(audio_tensors) != len(weights):
|
||||
raise ValueError("Number of tensors must match number of weights")
|
||||
|
||||
# Normalize weights
|
||||
total_weight = sum(weights)
|
||||
normalized_weights = [w / total_weight for w in weights]
|
||||
|
||||
# Compute weighted sum
|
||||
result = torch.zeros_like(audio_tensors[0])
|
||||
for tensor, weight in zip(audio_tensors, normalized_weights):
|
||||
result += tensor * weight
|
||||
|
||||
return result
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
||||
"""
|
||||
Run ensemble separation combining multiple models.
|
||||
|
||||
This runs multiple models and averages their outputs for best quality.
|
||||
Note: This takes significantly longer than single-model separation.
|
||||
|
||||
Returns:
|
||||
Dict mapping stem names to output file paths
|
||||
"""
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create temporary directory for intermediate results
|
||||
temp_dir = Path(tempfile.mkdtemp(dir=self.cache_dir))
|
||||
|
||||
try:
|
||||
stems = self.config["stems"]
|
||||
stem_audio_data = {stem: [] for stem in stems}
|
||||
stem_weights = {stem: [] for stem in stems}
|
||||
|
||||
# Run each model in the ensemble
|
||||
for model_config in self.config["models"]:
|
||||
model_type = model_config["type"]
|
||||
weight = model_config["weight"]
|
||||
|
||||
print(f"Running ensemble component: {model_type} (weight: {weight})")
|
||||
|
||||
# Run individual model and get stems
|
||||
model_stems = self._run_model_component(
|
||||
model_config, input_path, temp_dir
|
||||
)
|
||||
|
||||
# Add to collection for averaging
|
||||
for stem_name, stem_path in model_stems.items():
|
||||
if stem_name in stem_audio_data:
|
||||
waveform, sr = self._load_audio(stem_path)
|
||||
stem_audio_data[stem_name].append(waveform)
|
||||
stem_weights[stem_name].append(weight)
|
||||
|
||||
# Compute weighted averages and save final outputs
|
||||
output_paths = {}
|
||||
sample_rate = 44100 # Default, will be updated from actual audio
|
||||
|
||||
for stem_name in stems:
|
||||
if stem_name in stem_audio_data and stem_audio_data[stem_name]:
|
||||
# Compute weighted average
|
||||
averaged = self._weighted_average(
|
||||
stem_audio_data[stem_name],
|
||||
stem_weights[stem_name]
|
||||
)
|
||||
|
||||
# Save result
|
||||
output_file = output_path / f"{stem_name}.{self.output_format}"
|
||||
|
||||
# Convert to desired format
|
||||
if self.output_format == "mp3":
|
||||
# Save as WAV first, then convert
|
||||
temp_wav = temp_dir / f"{stem_name}_temp.wav"
|
||||
self._save_audio(averaged, sample_rate, str(temp_wav))
|
||||
|
||||
# Convert to MP3
|
||||
import subprocess
|
||||
bitrate = self.mp3_bitrate.replace("k", "")
|
||||
subprocess.run([
|
||||
"ffmpeg", "-y", "-i", str(temp_wav),
|
||||
"-b:a", f"{bitrate}k",
|
||||
"-f", "mp3", str(output_file)
|
||||
], check=True, capture_output=True)
|
||||
else:
|
||||
self._save_audio(averaged, sample_rate, str(output_file))
|
||||
|
||||
output_paths[stem_name] = str(output_file)
|
||||
|
||||
return output_paths
|
||||
|
||||
finally:
|
||||
# Cleanup temp directory
|
||||
try:
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
except:
|
||||
pass
|
||||
|
||||
def _run_model_component(
|
||||
self,
|
||||
model_config: Dict,
|
||||
input_path: str,
|
||||
output_dir: Path
|
||||
) -> Dict[str, str]:
|
||||
"""
|
||||
Run a single model component and return its output stems.
|
||||
|
||||
Returns:
|
||||
Dict mapping stem names to file paths
|
||||
"""
|
||||
model_type = model_config["type"]
|
||||
|
||||
if model_type == "bs_roformer":
|
||||
from .bs_roformer_separator import BSRoformerSeparator
|
||||
sep = BSRoformerSeparator(
|
||||
cpu_separation=self.cpu_separation,
|
||||
output_format="wav", # Use WAV for lossless intermediate
|
||||
mp3_bitrate="320k",
|
||||
stem_mode=model_config.get("stem_mode", "4stem"),
|
||||
)
|
||||
return sep.separate_into_parts(input_path, str(output_dir / "bsr"))
|
||||
|
||||
elif model_type == "bs_roformer_sw":
|
||||
from .bs_roformer_sw_separator import BSRoformerSWSeparator
|
||||
sep = BSRoformerSWSeparator(
|
||||
cpu_separation=self.cpu_separation,
|
||||
output_format="wav",
|
||||
mp3_bitrate="320k",
|
||||
)
|
||||
return sep.separate_into_parts(input_path, str(output_dir / "bsr_sw"))
|
||||
|
||||
elif model_type == "mel_band":
|
||||
from .mel_band_specialized_separator import MelBandVocalsSeparator
|
||||
variant = model_config.get("variant", "vocals")
|
||||
if variant == "vocals":
|
||||
sep = MelBandVocalsSeparator(
|
||||
cpu_separation=self.cpu_separation,
|
||||
output_format="wav",
|
||||
mp3_bitrate="320k",
|
||||
)
|
||||
else:
|
||||
from .mel_band_roformer_separator import MelBandRoformerSeparator
|
||||
sep = MelBandRoformerSeparator(
|
||||
cpu_separation=self.cpu_separation,
|
||||
output_format="wav",
|
||||
mp3_bitrate="320k",
|
||||
)
|
||||
return sep.separate_into_parts(input_path, str(output_dir / "mbr"))
|
||||
|
||||
elif model_type == "scnet_xl_ihf":
|
||||
from .scnet_separator import SCNetSeparator
|
||||
sep = SCNetSeparator(
|
||||
model_id="scnet_xl_ihf",
|
||||
cpu_separation=self.cpu_separation,
|
||||
output_format="wav",
|
||||
mp3_bitrate="320k",
|
||||
)
|
||||
return sep.separate_into_parts(input_path, str(output_dir / "scnet"))
|
||||
|
||||
elif model_type == "scnet_large":
|
||||
from .scnet_separator import SCNetSeparator
|
||||
sep = SCNetSeparator(
|
||||
model_id="scnet_large",
|
||||
cpu_separation=self.cpu_separation,
|
||||
output_format="wav",
|
||||
mp3_bitrate="320k",
|
||||
)
|
||||
return sep.separate_into_parts(input_path, str(output_dir / "scnet"))
|
||||
|
||||
elif model_type == "demucs":
|
||||
# For Demucs, we'd need to run via CLI subprocess
|
||||
# This is a placeholder
|
||||
model_name = model_config.get("model", "htdemucs_ft")
|
||||
# Return empty dict for now - would need actual implementation
|
||||
return {}
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown model type in ensemble: {model_type}")
|
||||
Executable
+847
@@ -0,0 +1,847 @@
|
||||
"""
|
||||
LRC alignment service for creating enhanced ELRC files with word-level timestamps.
|
||||
Uses the lyrics-sync module for vocal extraction and forced alignment.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Tuple
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import JOBS_DIR, LRC_SYNC_DIR
|
||||
from .metadata_util import (
|
||||
extract_lyrics_from_file,
|
||||
lyrics_to_plaintext,
|
||||
format_elrc,
|
||||
is_lrc_synced
|
||||
)
|
||||
|
||||
# Add lyrics-sync to path
|
||||
_LRC_SYNC_PATH = Path(__file__).parent.parent.parent / "lyrics-sync"
|
||||
if str(_LRC_SYNC_PATH) not in sys.path:
|
||||
sys.path.insert(0, str(_LRC_SYNC_PATH))
|
||||
|
||||
|
||||
class _LyricsSyncNoExtract:
|
||||
"""
|
||||
Drop-in replacement for LyricsSync that skips Demucs vocal extraction.
|
||||
|
||||
Use this when the input audio is already a separated vocals stem (e.g. from
|
||||
a project's Stems folder), so we avoid loading and running htdemucs again.
|
||||
The alignment pipeline is otherwise identical to the standard LyricsSync.
|
||||
"""
|
||||
|
||||
def __init__(self, lang: str = "en-US", blank_id: int = 0, device: str = None) -> None:
|
||||
from lsync.phoneme_recognizer import PhonemeRecognizer
|
||||
from lsync.lyrics_processor import LyricsProcessor
|
||||
self.phone_rec = PhonemeRecognizer(lang=lang, device=device)
|
||||
self.lp = LyricsProcessor(lang=lang)
|
||||
self.blank_id = blank_id
|
||||
|
||||
def sync(self, audio_fn: str, text_fn: str, save: bool = True, timing_precision: str = 'word'):
|
||||
import os
|
||||
import librosa
|
||||
from lsync.config import TARGET_SR
|
||||
from lsync.alignment import Aligner
|
||||
from lsync.util import read_text, save_lrc, save_words
|
||||
from lsync.lrc_formatter import LrcFormatter
|
||||
|
||||
audio_name = os.path.splitext(os.path.basename(audio_fn))[0]
|
||||
|
||||
# Load vocals directly (skip Demucs), resample to TARGET_SR
|
||||
vocals, sr = librosa.load(audio_fn, sr=None, mono=True)
|
||||
if sr != TARGET_SR:
|
||||
vocals = librosa.resample(vocals, orig_sr=sr, target_sr=TARGET_SR)
|
||||
|
||||
# Rest of alignment pipeline is unchanged
|
||||
emission, _, _, frame_duration = self.phone_rec.recognize(vocals)
|
||||
tokens = self.phone_rec.tokenize(text_fn)
|
||||
path = Aligner.align(emission, tokens, blank_id=self.blank_id)
|
||||
processed_lyrics = self.lp.process(text_fn)
|
||||
|
||||
# Always compute word-level for the basic LRC output
|
||||
words = self.lp.get_words_from_path(processed_lyrics, path, frame_duration)
|
||||
original_lyrics = read_text(text_fn)
|
||||
lrc = LrcFormatter.words2lrc(words, original_lyrics)
|
||||
|
||||
if save:
|
||||
save_words(words, audio_name)
|
||||
save_lrc(lrc, audio_name)
|
||||
|
||||
# Return syllable tokens for ELRC when requested
|
||||
if timing_precision == 'syllable':
|
||||
return_words = self.lp.get_syllables_from_path(processed_lyrics, path, frame_duration)
|
||||
else:
|
||||
return_words = words
|
||||
|
||||
return return_words, lrc
|
||||
|
||||
|
||||
# ── Lyrics cleaning helpers ───────────────────────────────────────────────────
|
||||
|
||||
_SECTION_MARKER_RE = re.compile(r'^\[(?!\d{1,2}:\d{2})[^\]]*\]\s*$')
|
||||
|
||||
def clean_lyrics_text(text: str) -> str:
|
||||
"""Remove section markers (e.g. [Chorus], [Verse 1]) and blank lines from plain lyrics."""
|
||||
cleaned = []
|
||||
for line in text.splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
if _SECTION_MARKER_RE.match(stripped):
|
||||
continue
|
||||
cleaned.append(stripped)
|
||||
return '\n'.join(cleaned)
|
||||
|
||||
|
||||
_LRC_SECTION_LINE_RE = re.compile(
|
||||
r'^\[\d{1,2}:\d{2}[.\d]*\]\s*\[(?!\d)[^\]]*\]\s*$'
|
||||
)
|
||||
|
||||
def clean_lrc_content(content: str) -> tuple:
|
||||
"""Remove section-marker lines from LRC/ELRC content.
|
||||
|
||||
A section-marker line looks like ``[01:30.00][Chorus]`` — a timestamp
|
||||
followed by a non-timestamp bracket label and nothing else.
|
||||
Returns ``(cleaned_str, lines_removed_count)``.
|
||||
"""
|
||||
lines = content.splitlines()
|
||||
cleaned, removed = [], 0
|
||||
for line in lines:
|
||||
if _LRC_SECTION_LINE_RE.match(line):
|
||||
removed += 1
|
||||
else:
|
||||
cleaned.append(line)
|
||||
return '\n'.join(cleaned), removed
|
||||
|
||||
|
||||
def run_lrc_alignment(
|
||||
job_id: str,
|
||||
audio_path: str,
|
||||
lyrics_source: str = "auto", # 'auto', 'tag', 'file', 'text'
|
||||
lyrics_content: Optional[str] = None,
|
||||
lyrics_file_path: Optional[str] = None,
|
||||
output_filename: Optional[str] = None,
|
||||
style_options: Optional[dict] = None,
|
||||
is_vocals: bool = False,
|
||||
output_dir: Optional[str] = None,
|
||||
timing_precision: str = 'word', # 'word' | 'syllable'
|
||||
lrc_power_preset: str = 'standard', # 'eco' | 'standard' | 'performance'
|
||||
):
|
||||
"""
|
||||
Background task: align lyrics with vocals to create ELRC file.
|
||||
|
||||
Args:
|
||||
job_id: Job identifier
|
||||
audio_path: Path to the audio file (with vocals)
|
||||
lyrics_source: Where to get lyrics from
|
||||
- 'auto': Try metadata tags, then external LRC, then fail
|
||||
- 'tag': Only use metadata tags
|
||||
- 'file': Use provided lyrics_file_path
|
||||
- 'text': Use provided lyrics_content
|
||||
lyrics_content: Raw lyrics text (when lyrics_source='text')
|
||||
lyrics_file_path: Path to LRC/txt file (when lyrics_source='file')
|
||||
output_filename: Optional custom filename for output ELRC
|
||||
is_vocals: When True, the audio_path is already a separated vocals stem —
|
||||
skip the internal Demucs extraction step for faster processing.
|
||||
output_dir: Optional custom output directory (defaults to LRC_SYNC_DIR / job_id)
|
||||
"""
|
||||
if output_dir:
|
||||
out_path = Path(output_dir)
|
||||
else:
|
||||
out_path = LRC_SYNC_DIR / job_id
|
||||
out_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
# Import lyrics-sync here to handle any import issues gracefully
|
||||
try:
|
||||
from lsync import LyricsSync
|
||||
except ImportError as e:
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.ERROR,
|
||||
error=f"Lyrics sync module not available: {e}",
|
||||
progress="Error."
|
||||
)
|
||||
return
|
||||
|
||||
# Step 1: Get lyrics
|
||||
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Reading lyrics...")
|
||||
|
||||
lyrics_text = ""
|
||||
lyrics_synced = False
|
||||
|
||||
if lyrics_source == "text" and lyrics_content:
|
||||
lyrics_text = lyrics_content
|
||||
lyrics_synced = is_lrc_synced(lyrics_content)
|
||||
elif lyrics_source == "file" and lyrics_file_path:
|
||||
lyrics_data = extract_lyrics_from_file(lyrics_file_path)
|
||||
lyrics_text = lyrics_data["plain"]
|
||||
lyrics_synced = lyrics_data["synced"]
|
||||
elif lyrics_source in ("auto", "tag"):
|
||||
lyrics_data = extract_lyrics_from_file(audio_path)
|
||||
lyrics_text = lyrics_data["plain"]
|
||||
lyrics_synced = lyrics_data["synced"]
|
||||
|
||||
# If no lyrics in tags, try external LRC file
|
||||
if not lyrics_text and lyrics_source == "auto":
|
||||
audio_dir = Path(audio_path).parent
|
||||
audio_stem = Path(audio_path).stem
|
||||
for ext in ['.lrc', '.LRC', '.txt', '.TXT']:
|
||||
lrc_file = audio_dir / f"{audio_stem}{ext}"
|
||||
if lrc_file.exists():
|
||||
lyrics_data = extract_lyrics_from_file(str(lrc_file))
|
||||
lyrics_text = lyrics_data["plain"]
|
||||
lyrics_synced = lyrics_data["synced"]
|
||||
break
|
||||
|
||||
if not lyrics_text:
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.ERROR,
|
||||
error="No lyrics found. Please provide lyrics text or ensure audio file has lyrics tags.",
|
||||
progress="Error."
|
||||
)
|
||||
return
|
||||
|
||||
# Strip section markers ([Chorus], [Verse 1], etc.) and blank lines
|
||||
if not lyrics_synced:
|
||||
lyrics_text = clean_lyrics_text(lyrics_text)
|
||||
|
||||
# Choose sync class: skip Demucs when audio is already vocals.
|
||||
# In eco mode, force CPU for the phoneme recogniser to avoid GPU contention
|
||||
# that causes display glitches on shared-GPU (WSL2) setups.
|
||||
_align_device = "cpu" if lrc_power_preset == "eco" else None
|
||||
SyncClass = _LyricsSyncNoExtract if is_vocals else LyricsSync
|
||||
|
||||
# If lyrics are already synced (LRC format), convert to ELRC with word-level timestamps
|
||||
if lyrics_synced:
|
||||
jm.update_job(job_id, progress="Lyrics already synced. Enhancing to word-level...")
|
||||
_enhance_existing_lrc(
|
||||
job_id, audio_path, lyrics_text, out_path,
|
||||
output_filename, SyncClass, style_options, timing_precision,
|
||||
align_device=_align_device,
|
||||
)
|
||||
else:
|
||||
# Run full alignment pipeline
|
||||
jm.update_job(job_id, progress="Starting lyrics alignment...")
|
||||
_run_full_alignment(
|
||||
job_id, audio_path, lyrics_text, out_path,
|
||||
output_filename, SyncClass, style_options, timing_precision,
|
||||
align_device=_align_device,
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
|
||||
|
||||
|
||||
def _run_full_alignment(
|
||||
job_id: str,
|
||||
audio_path: str,
|
||||
lyrics_text: str,
|
||||
output_dir: Path,
|
||||
output_filename: Optional[str],
|
||||
LyricsSyncClass,
|
||||
style_options: Optional[dict] = None,
|
||||
timing_precision: str = 'word',
|
||||
align_device: Optional[str] = None,
|
||||
):
|
||||
"""Run full alignment using lyrics-sync."""
|
||||
# Write lyrics to temp file
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False, encoding='utf-8') as f:
|
||||
f.write(lyrics_text)
|
||||
lyrics_temp_path = f.name
|
||||
|
||||
try:
|
||||
# Initialize lyrics sync
|
||||
is_no_extract = LyricsSyncClass is _LyricsSyncNoExtract
|
||||
device_note = f" [{align_device.upper()}]" if align_device else ""
|
||||
jm.update_job(job_id, progress=f"Initializing alignment model{device_note}...")
|
||||
if is_no_extract:
|
||||
lsync = LyricsSyncClass(lang="en-US", device=align_device)
|
||||
else:
|
||||
lsync = LyricsSyncClass(lang="en-US")
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
# Run alignment
|
||||
prec_label = "syllable-level" if timing_precision == 'syllable' else "word-level"
|
||||
if LyricsSyncClass is _LyricsSyncNoExtract:
|
||||
prog_msg = f"Aligning lyrics ({prec_label})..."
|
||||
else:
|
||||
prog_msg = f"Extracting vocals and aligning lyrics ({prec_label})... (this may take a while)"
|
||||
jm.update_job(job_id, progress=prog_msg)
|
||||
words, lrc = lsync.sync(audio_path, lyrics_temp_path, save=False, timing_precision=timing_precision)
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
# Convert to ELRC format
|
||||
elrc_content = _convert_to_elrc(words, lyrics_text, timing_precision)
|
||||
|
||||
# Save ELRC file
|
||||
if not output_filename:
|
||||
output_filename = f"{Path(audio_path).stem}.elrc"
|
||||
elif not output_filename.endswith('.elrc'):
|
||||
output_filename += '.elrc'
|
||||
|
||||
output_path = output_dir / output_filename
|
||||
output_path.write_text(elrc_content, encoding='utf-8')
|
||||
|
||||
# Also save plain LRC version
|
||||
lrc_filename = output_filename.replace('.elrc', '.lrc')
|
||||
lrc_path = output_dir / lrc_filename
|
||||
lrc_path.write_text(lrc, encoding='utf-8')
|
||||
|
||||
files = [
|
||||
{"name": output_path.name, "path": str(output_path), "type": "elrc"},
|
||||
{"name": lrc_path.name, "path": str(lrc_path), "type": "lrc"},
|
||||
]
|
||||
|
||||
# Generate styled HTML preview if style options provided
|
||||
if style_options:
|
||||
jm.update_job(job_id, progress="Generating styled preview...")
|
||||
html_content = _generate_styled_html(elrc_content, style_options, Path(audio_path).stem)
|
||||
html_filename = output_filename.replace('.elrc', '.html')
|
||||
html_path = output_dir / html_filename
|
||||
html_path.write_text(html_content, encoding='utf-8')
|
||||
files.append({"name": html_path.name, "path": str(html_path), "type": "html"})
|
||||
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.DONE,
|
||||
progress="ELRC alignment complete.",
|
||||
files=files
|
||||
)
|
||||
|
||||
finally:
|
||||
# Cleanup temp file
|
||||
try:
|
||||
os.unlink(lyrics_temp_path)
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
def _enhance_existing_lrc(
|
||||
job_id: str,
|
||||
audio_path: str,
|
||||
lrc_content: str,
|
||||
output_dir: Path,
|
||||
output_filename: Optional[str],
|
||||
LyricsSyncClass,
|
||||
style_options: Optional[dict] = None,
|
||||
timing_precision: str = 'word',
|
||||
align_device: Optional[str] = None,
|
||||
):
|
||||
"""Enhance existing LRC with word-level timestamps."""
|
||||
from .metadata_util import parse_lrc
|
||||
|
||||
# Parse existing LRC
|
||||
lines = parse_lrc(lrc_content)
|
||||
|
||||
# Get plain lyrics text
|
||||
plain_lyrics = lyrics_to_plaintext(lrc_content)
|
||||
|
||||
# Run alignment
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False, encoding='utf-8') as f:
|
||||
f.write(plain_lyrics)
|
||||
lyrics_temp_path = f.name
|
||||
|
||||
try:
|
||||
if LyricsSyncClass is _LyricsSyncNoExtract:
|
||||
lsync = LyricsSyncClass(lang="en-US", device=align_device)
|
||||
else:
|
||||
lsync = LyricsSyncClass(lang="en-US")
|
||||
|
||||
prec_label = "syllable-level" if timing_precision == 'syllable' else "word-level"
|
||||
if LyricsSyncClass is _LyricsSyncNoExtract:
|
||||
prog_msg = f"Aligning words ({prec_label})..."
|
||||
else:
|
||||
prog_msg = f"Extracting vocals and aligning words ({prec_label})..."
|
||||
jm.update_job(job_id, progress=prog_msg)
|
||||
words, lrc = lsync.sync(audio_path, lyrics_temp_path, save=False, timing_precision=timing_precision)
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
# Convert to ELRC
|
||||
elrc_content = _convert_to_elrc(words, plain_lyrics, timing_precision)
|
||||
|
||||
# Save files
|
||||
if not output_filename:
|
||||
output_filename = f"{Path(audio_path).stem}.elrc"
|
||||
elif not output_filename.endswith('.elrc'):
|
||||
output_filename += '.elrc'
|
||||
|
||||
output_path = output_dir / output_filename
|
||||
output_path.write_text(elrc_content, encoding='utf-8')
|
||||
|
||||
lrc_filename = output_filename.replace('.elrc', '.lrc')
|
||||
lrc_path = output_dir / lrc_filename
|
||||
lrc_path.write_text(lrc, encoding='utf-8')
|
||||
|
||||
files = [
|
||||
{"name": output_path.name, "path": str(output_path), "type": "elrc"},
|
||||
{"name": lrc_path.name, "path": str(lrc_path), "type": "lrc"},
|
||||
]
|
||||
|
||||
# Generate styled HTML preview if style options provided
|
||||
if style_options:
|
||||
jm.update_job(job_id, progress="Generating styled preview...")
|
||||
html_content = _generate_styled_html(elrc_content, style_options, Path(audio_path).stem)
|
||||
html_filename = output_filename.replace('.elrc', '.html')
|
||||
html_path = output_dir / html_filename
|
||||
html_path.write_text(html_content, encoding='utf-8')
|
||||
files.append({"name": html_path.name, "path": str(html_path), "type": "html"})
|
||||
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.DONE,
|
||||
progress="ELRC enhancement complete.",
|
||||
files=files
|
||||
)
|
||||
|
||||
finally:
|
||||
try:
|
||||
os.unlink(lyrics_temp_path)
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
def _fmt_ts(seconds: float) -> str:
|
||||
"""Format a time value as an ELRC inline timestamp <mm:ss.xx>."""
|
||||
m = int(seconds // 60)
|
||||
s = int(seconds % 60)
|
||||
cs = int((seconds % 1) * 100)
|
||||
return f"<{m:02d}:{s:02d}.{cs:02d}>"
|
||||
|
||||
|
||||
def _line_ts(seconds: float) -> str:
|
||||
"""Format a time value as an ELRC line timestamp [mm:ss.xx]."""
|
||||
m = int(seconds // 60)
|
||||
s = int(seconds % 60)
|
||||
cs = int((seconds % 1) * 100)
|
||||
return f"[{m:02d}:{s:02d}.{cs:02d}]"
|
||||
|
||||
|
||||
def _convert_to_elrc(words, original_lyrics: str, timing_precision: str = 'word') -> str:
|
||||
"""
|
||||
Convert aligned words/syllables to ELRC format.
|
||||
|
||||
Word mode → ``[line_ts]<ts>Hello <ts>world``
|
||||
Syllable mode → ``[line_ts]<ts>Hel<ts>lo <ts>world``
|
||||
(no space between intra-word syllables; space between words)
|
||||
|
||||
Args:
|
||||
words: List of Word objects (label, start, end, is_word_end)
|
||||
original_lyrics: Original lyrics text for line structure
|
||||
timing_precision: 'word' or 'syllable'
|
||||
|
||||
Returns:
|
||||
ELRC formatted string
|
||||
"""
|
||||
original_lines = [line.strip() for line in original_lyrics.split('\n') if line.strip()]
|
||||
|
||||
elrc_lines = []
|
||||
word_idx = 0
|
||||
|
||||
for line_text in original_lines:
|
||||
if word_idx >= len(words):
|
||||
break
|
||||
|
||||
orig_words = line_text.split()
|
||||
if not orig_words:
|
||||
continue
|
||||
|
||||
if timing_precision == 'syllable':
|
||||
# Collect all syllables for this line by consuming until is_word_end
|
||||
# for each original word
|
||||
line_syls = []
|
||||
for _ in orig_words:
|
||||
while word_idx < len(words):
|
||||
syl = words[word_idx]
|
||||
line_syls.append(syl)
|
||||
word_idx += 1
|
||||
if getattr(syl, 'is_word_end', False):
|
||||
break
|
||||
|
||||
if not line_syls:
|
||||
continue
|
||||
|
||||
line_start = line_syls[0].start
|
||||
line_content = ""
|
||||
for i, syl in enumerate(line_syls):
|
||||
# Space before a syllable only when the previous one ended a word
|
||||
prefix = " " if i > 0 and getattr(line_syls[i - 1], 'is_word_end', True) else ""
|
||||
line_content += f"{prefix}{_fmt_ts(syl.start)}{syl.label}"
|
||||
|
||||
else:
|
||||
# Word mode: consume one Word per original word
|
||||
line_aligned = []
|
||||
for i in range(len(orig_words)):
|
||||
if word_idx + i < len(words):
|
||||
w = words[word_idx + i]
|
||||
line_aligned.append(w)
|
||||
|
||||
if not line_aligned:
|
||||
continue
|
||||
|
||||
line_start = line_aligned[0].start
|
||||
line_content = " ".join(f"{_fmt_ts(w.start)}{w.label}" for w in line_aligned)
|
||||
word_idx += len(orig_words)
|
||||
|
||||
elrc_lines.append(f"{_line_ts(line_start)}{line_content}")
|
||||
|
||||
return '\n'.join(elrc_lines)
|
||||
|
||||
|
||||
def _generate_styled_html(elrc_content: str, style_options: dict, title: str) -> str:
|
||||
"""
|
||||
Generate an HTML file with styled lyrics preview.
|
||||
|
||||
Args:
|
||||
elrc_content: ELRC formatted lyrics content
|
||||
style_options: Dictionary with style configuration
|
||||
title: Song title for the HTML page
|
||||
|
||||
Returns:
|
||||
HTML content as string
|
||||
"""
|
||||
from .metadata_util import parse_elrc
|
||||
|
||||
# Extract style options with defaults
|
||||
font_family = style_options.get('fontFamily', 'system-ui')
|
||||
font_size = style_options.get('fontSize', 24)
|
||||
text_color = style_options.get('textColor', '#e2e8f0')
|
||||
highlight_color = style_options.get('highlightColor', '#3b82f6')
|
||||
background_color = style_options.get('backgroundColor', '#0f172a')
|
||||
word_highlight_color = style_options.get('wordHighlightColor', '#60a5fa')
|
||||
line_spacing = style_options.get('lineSpacing', 1.5)
|
||||
text_align = style_options.get('textAlign', 'center')
|
||||
|
||||
# Parse ELRC content
|
||||
lines = parse_elrc(elrc_content)
|
||||
|
||||
# Build HTML lines
|
||||
html_lines = []
|
||||
for line_ts, line_text, words in lines:
|
||||
if words:
|
||||
# Build line with word spans for animation
|
||||
word_spans = []
|
||||
for word_ts, word in words:
|
||||
minutes = int(word_ts // 60)
|
||||
seconds = int(word_ts % 60)
|
||||
millis = int((word_ts % 1) * 1000)
|
||||
time_attr = f"{minutes:02d}:{seconds:02d}.{millis:03d}"
|
||||
word_spans.append(
|
||||
f'<span class="word" data-time="{time_attr}">{word}</span>'
|
||||
)
|
||||
line_html = ' '.join(word_spans)
|
||||
else:
|
||||
line_html = line_text
|
||||
|
||||
minutes = int(line_ts // 60)
|
||||
seconds = int(line_ts % 60)
|
||||
millis = int((line_ts % 1) * 1000)
|
||||
line_time = f"{minutes:02d}:{seconds:02d}.{millis:03d}"
|
||||
|
||||
html_lines.append(
|
||||
f'<div class="lyric-line" data-time="{line_time}">{line_html}</div>'
|
||||
)
|
||||
|
||||
lyrics_html = '\n'.join(html_lines)
|
||||
|
||||
html_template = f'''<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>{title} - Lyrics</title>
|
||||
<style>
|
||||
* {{
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}}
|
||||
|
||||
body {{
|
||||
font-family: {font_family};
|
||||
background-color: {background_color};
|
||||
color: {text_color};
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
padding: 2rem 1rem;
|
||||
}}
|
||||
|
||||
.container {{
|
||||
max-width: 800px;
|
||||
width: 100%;
|
||||
}}
|
||||
|
||||
h1 {{
|
||||
font-size: {font_size * 1.5}px;
|
||||
margin-bottom: 2rem;
|
||||
text-align: {text_align};
|
||||
color: {text_color};
|
||||
}}
|
||||
|
||||
.lyrics-container {{
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: {font_size * line_spacing * 0.5}px;
|
||||
}}
|
||||
|
||||
.lyric-line {{
|
||||
font-size: {font_size}px;
|
||||
line-height: {line_spacing};
|
||||
text-align: {text_align};
|
||||
color: {text_color};
|
||||
opacity: 0.6;
|
||||
transition: all 0.3s ease;
|
||||
padding: 0.5rem;
|
||||
border-radius: 0.5rem;
|
||||
}}
|
||||
|
||||
.lyric-line.active {{
|
||||
color: {highlight_color};
|
||||
opacity: 1;
|
||||
transform: scale(1.02);
|
||||
}}
|
||||
|
||||
.word {{
|
||||
display: inline-block;
|
||||
transition: color 0.2s ease;
|
||||
padding: 0 0.1em;
|
||||
}}
|
||||
|
||||
.word.active {{
|
||||
color: {word_highlight_color};
|
||||
text-shadow: 0 0 10px {word_highlight_color}40;
|
||||
}}
|
||||
|
||||
.controls {{
|
||||
position: fixed;
|
||||
bottom: 2rem;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
background: rgba(0, 0, 0, 0.8);
|
||||
padding: 1rem 2rem;
|
||||
border-radius: 2rem;
|
||||
display: flex;
|
||||
gap: 1rem;
|
||||
align-items: center;
|
||||
backdrop-filter: blur(10px);
|
||||
}}
|
||||
|
||||
button {{
|
||||
background: {highlight_color};
|
||||
color: white;
|
||||
border: none;
|
||||
padding: 0.5rem 1rem;
|
||||
border-radius: 0.5rem;
|
||||
cursor: pointer;
|
||||
font-size: 14px;
|
||||
transition: opacity 0.2s;
|
||||
}}
|
||||
|
||||
button:hover {{
|
||||
opacity: 0.8;
|
||||
}}
|
||||
|
||||
input[type="range"] {{
|
||||
width: 200px;
|
||||
}}
|
||||
|
||||
.time-display {{
|
||||
font-family: monospace;
|
||||
font-size: 14px;
|
||||
color: {text_color};
|
||||
}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>{title}</h1>
|
||||
<div class="lyrics-container">
|
||||
{lyrics_html}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="controls">
|
||||
<button id="playBtn">▶ Play</button>
|
||||
<input type="range" id="progress" min="0" max="100" value="0">
|
||||
<span class="time-display" id="timeDisplay">00:00</span>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const lines = document.querySelectorAll('.lyric-line');
|
||||
const words = document.querySelectorAll('.word');
|
||||
const playBtn = document.getElementById('playBtn');
|
||||
const progress = document.getElementById('progress');
|
||||
const timeDisplay = document.getElementById('timeDisplay');
|
||||
|
||||
let isPlaying = false;
|
||||
let currentTime = 0;
|
||||
// Derive duration from the last timestamp in the ELRC content
|
||||
const allTimestamps = Array.from(document.querySelectorAll('[data-start]'))
|
||||
.map(el => parseFloat(el.getAttribute('data-start')))
|
||||
.filter(t => !isNaN(t));
|
||||
let duration = allTimestamps.length > 0 ? Math.ceil(Math.max(...allTimestamps) + 10) : 300;
|
||||
let animationId = null;
|
||||
|
||||
// Parse time string to seconds
|
||||
function parseTime(timeStr) {{
|
||||
const [mins, secs] = timeStr.split(':');
|
||||
return parseInt(mins) * 60 + parseFloat(secs);
|
||||
}}
|
||||
|
||||
// Format seconds to time string
|
||||
function formatTime(seconds) {{
|
||||
const mins = Math.floor(seconds / 60);
|
||||
const secs = Math.floor(seconds % 60);
|
||||
return `${{mins.toString().padStart(2, '0')}}:${{secs.toString().padStart(2, '0')}}`;
|
||||
}}
|
||||
|
||||
// Track last-active elements to avoid redundant DOM updates
|
||||
let _activeLine = null;
|
||||
let _activeWord = null;
|
||||
|
||||
// Update active line/word based on current time; scroll active line into view
|
||||
function updateActiveElements() {{
|
||||
// Find the last line whose timestamp <= currentTime
|
||||
let newLine = null;
|
||||
lines.forEach(line => {{
|
||||
if (currentTime >= parseTime(line.dataset.time)) newLine = line;
|
||||
}});
|
||||
|
||||
if (newLine !== _activeLine) {{
|
||||
if (_activeLine) _activeLine.classList.remove('active');
|
||||
if (newLine) {{
|
||||
newLine.classList.add('active');
|
||||
const ctrl = document.querySelector('.controls');
|
||||
const ctrlH = ctrl ? ctrl.offsetHeight + 32 : 80;
|
||||
const rect = newLine.getBoundingClientRect();
|
||||
const lineMid = rect.top + rect.height / 2;
|
||||
const availH = window.innerHeight - ctrlH;
|
||||
const target = window.scrollY + lineMid - availH / 2;
|
||||
window.scrollTo({{ top: Math.max(0, target), behavior: 'smooth' }});
|
||||
}}
|
||||
_activeLine = newLine;
|
||||
}}
|
||||
|
||||
// Find the last word whose timestamp <= currentTime
|
||||
let newWord = null;
|
||||
words.forEach(word => {{
|
||||
if (currentTime >= parseTime(word.dataset.time)) newWord = word;
|
||||
}});
|
||||
|
||||
if (newWord !== _activeWord) {{
|
||||
if (_activeWord) _activeWord.classList.remove('active');
|
||||
if (newWord) newWord.classList.add('active');
|
||||
_activeWord = newWord;
|
||||
}}
|
||||
}}
|
||||
|
||||
// Animation loop — use performance.now() delta for accurate wall-clock timing
|
||||
let _lastTs = null;
|
||||
function animate(ts) {{
|
||||
if (isPlaying) {{
|
||||
if (_lastTs !== null) currentTime += (ts - _lastTs) / 1000;
|
||||
_lastTs = ts;
|
||||
if (currentTime > duration) {{
|
||||
currentTime = 0;
|
||||
isPlaying = false;
|
||||
playBtn.textContent = '▶ Play';
|
||||
_lastTs = null;
|
||||
}}
|
||||
progress.value = (currentTime / duration) * 100;
|
||||
timeDisplay.textContent = formatTime(currentTime);
|
||||
updateActiveElements();
|
||||
}} else {{
|
||||
_lastTs = null;
|
||||
}}
|
||||
animationId = requestAnimationFrame(animate);
|
||||
}}
|
||||
|
||||
// Play/Pause toggle
|
||||
playBtn.addEventListener('click', () => {{
|
||||
isPlaying = !isPlaying;
|
||||
playBtn.textContent = isPlaying ? '⏸ Pause' : '▶ Play';
|
||||
}});
|
||||
|
||||
// Progress bar interaction
|
||||
progress.addEventListener('input', (e) => {{
|
||||
currentTime = (e.target.value / 100) * duration;
|
||||
timeDisplay.textContent = formatTime(currentTime);
|
||||
updateActiveElements();
|
||||
}});
|
||||
|
||||
// Start animation loop
|
||||
animate();
|
||||
</script>
|
||||
</body>
|
||||
</html>'''
|
||||
|
||||
return html_template
|
||||
|
||||
|
||||
def extract_vocals_for_alignment(
|
||||
job_id: str,
|
||||
audio_path: str,
|
||||
output_dir: Path
|
||||
) -> str:
|
||||
"""
|
||||
Extract vocals from audio file for alignment purposes.
|
||||
Returns path to extracted vocal file.
|
||||
"""
|
||||
jm.update_job(job_id, progress="Extracting vocals...")
|
||||
|
||||
try:
|
||||
import torch
|
||||
from demucs.pretrained import get_model
|
||||
from demucs.apply import apply_model
|
||||
from demucs.separate import load_track
|
||||
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
model = get_model(name="htdemucs", repo=None)
|
||||
model.to(device)
|
||||
model.eval()
|
||||
|
||||
vocals_idx = model.sources.index("vocals")
|
||||
|
||||
# Load audio
|
||||
wav = load_track(audio_path, 2, model.samplerate)
|
||||
|
||||
# Separate
|
||||
ref = wav.mean(0)
|
||||
wav = (wav - ref.mean()) / ref.std()
|
||||
|
||||
with torch.no_grad():
|
||||
sources = apply_model(
|
||||
model, wav[None], device=device, shifts=1,
|
||||
split=True, overlap=0.25, progress=False
|
||||
)
|
||||
|
||||
vocals = sources[0][vocals_idx]
|
||||
vocals_np = vocals.cpu().numpy()[0, ...]
|
||||
|
||||
# Save vocals
|
||||
import soundfile as sf
|
||||
vocals_path = output_dir / f"{Path(audio_path).stem}_vocals.wav"
|
||||
sf.write(vocals_path, vocals_np, model.samplerate)
|
||||
|
||||
return str(vocals_path)
|
||||
|
||||
except Exception as e:
|
||||
# If vocal extraction fails, return original audio
|
||||
jm.update_job(job_id, progress=f"Vocal extraction failed, using original audio: {e}")
|
||||
return audio_path
|
||||
+1075
File diff suppressed because it is too large.
Load diff
Executable
+355
@@ -0,0 +1,355 @@
|
||||
"""
|
||||
MDX-Net separator using ONNX models from Ultimate Vocal Remover (UVR).
|
||||
Supports various MDX-Net models for vocals/instrumental separation.
|
||||
"""
|
||||
import gc
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional, Tuple
|
||||
from collections import namedtuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from tqdm import tqdm
|
||||
|
||||
# Try to import ONNX runtime
|
||||
try:
|
||||
import onnxruntime as ort
|
||||
ONNX_AVAILABLE = True
|
||||
except ImportError:
|
||||
ONNX_AVAILABLE = False
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
# Model configurations
|
||||
MDXNET_MODELS = {
|
||||
'mdxnet_voc_ft': {
|
||||
'repo_id': 'Politrees/UVR_resources',
|
||||
'path': 'models/MDXNet/UVR-MDX-NET-Voc_FT.onnx',
|
||||
'stems': ['vocals', 'instrumental'],
|
||||
'target_stem': 'vocals',
|
||||
},
|
||||
'mdxnet_inst_hq_1': {
|
||||
'repo_id': 'Politrees/UVR_resources',
|
||||
'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_1.onnx',
|
||||
'stems': ['instrumental', 'vocals'],
|
||||
'target_stem': 'instrumental',
|
||||
},
|
||||
'mdxnet_inst_hq_2': {
|
||||
'repo_id': 'Politrees/UVR_resources',
|
||||
'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_2.onnx',
|
||||
'stems': ['instrumental', 'vocals'],
|
||||
'target_stem': 'instrumental',
|
||||
},
|
||||
'mdxnet_inst_hq_3': {
|
||||
'repo_id': 'Politrees/UVR_resources',
|
||||
'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_3.onnx',
|
||||
'stems': ['instrumental', 'vocals'],
|
||||
'target_stem': 'instrumental',
|
||||
},
|
||||
'mdxnet_karaoke': {
|
||||
'repo_id': 'Politrees/UVR_resources',
|
||||
'path': 'models/MDXNet/UVR_MDXNET_KARA.onnx',
|
||||
'stems': ['lead', 'backing_instrumental'],
|
||||
'target_stem': 'lead',
|
||||
},
|
||||
# New models from MVSEP-Central-Backup
|
||||
'mdxnet_dereverb': {
|
||||
'repo_id': 'miercolesv/MVSEP-Central-Backup',
|
||||
'path': 'Dereverb-MelBand-Anvuew/dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt',
|
||||
'stems': ['dry_vocals', 'reverb'],
|
||||
'target_stem': 'dry_vocals',
|
||||
'use_melband': True, # Uses MelBand architecture, not pure ONNX
|
||||
},
|
||||
'mdxnet_drumsep': {
|
||||
'repo_id': 'miercolesv/MVSEP-Central-Backup',
|
||||
'path': 'Politrees/UVR_resources/models/MDX23C/MDX23C-DrumSep-aufr33-jarredou.ckpt',
|
||||
'stems': ['drums', 'no_drums'],
|
||||
'target_stem': 'drums',
|
||||
'use_melband': True,
|
||||
},
|
||||
}
|
||||
|
||||
# MDX-Net specific constants
|
||||
HOP_LENGTH = 1024
|
||||
N_FFT = 2048
|
||||
DIM_F = 3072
|
||||
DIM_T = 8
|
||||
|
||||
|
||||
def _load_audio(path: str, sample_rate: int = SAMPLE_RATE) -> np.ndarray:
|
||||
"""Load audio as (channels, samples) float32 array."""
|
||||
import soundfile as sf
|
||||
|
||||
try:
|
||||
data, sr = sf.read(path, dtype='float32', always_2d=True)
|
||||
data = data.T
|
||||
except Exception:
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', path, '-ar', str(sample_rate),
|
||||
'-ac', '2', '-f', 'wav', tmp_path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
data, sr = sf.read(tmp_path, dtype='float32', always_2d=True)
|
||||
data = data.T
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
if sr != sample_rate:
|
||||
import librosa
|
||||
data = np.array([
|
||||
librosa.resample(ch, orig_sr=sr, target_sr=sample_rate)
|
||||
for ch in data
|
||||
], dtype=np.float32)
|
||||
|
||||
# Ensure stereo
|
||||
if data.shape[0] == 1:
|
||||
data = np.vstack([data, data])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def _save_audio(path: str, data: np.ndarray, output_format: str, bitrate: str = '256k') -> None:
|
||||
"""Save (samples, channels) array to audio file."""
|
||||
import soundfile as sf
|
||||
|
||||
ext = output_format.lower()
|
||||
|
||||
if ext in ('wav', 'flac'):
|
||||
sf.write(path, data, SAMPLE_RATE, format=ext.upper(),
|
||||
subtype='PCM_24' if ext == 'wav' else 'PCM_16')
|
||||
else:
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
sf.write(tmp_path, data, SAMPLE_RATE, format='WAV', subtype='PCM_16')
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', tmp_path, '-b:a', bitrate, path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
|
||||
def _stft(wave: np.ndarray, n_fft: int = N_FFT, hop_length: int = HOP_LENGTH) -> np.ndarray:
|
||||
"""Compute STFT of audio."""
|
||||
# wave shape: (channels, samples)
|
||||
channels = wave.shape[0]
|
||||
specs = []
|
||||
for ch in range(channels):
|
||||
spec = np.stft(wave[ch], n_fft=n_fft, hop_length=hop_length, window='hann')
|
||||
specs.append(spec)
|
||||
return np.array(specs) # (channels, freq, time, 2) where last dim is real/imag
|
||||
|
||||
|
||||
def _istft(spec: np.ndarray, n_fft: int = N_FFT, hop_length: int = HOP_LENGTH) -> np.ndarray:
|
||||
"""Compute inverse STFT."""
|
||||
channels = spec.shape[0]
|
||||
waves = []
|
||||
for ch in range(channels):
|
||||
wave = np.istft(spec[ch], n_fft=n_fft, hop_length=hop_length, window='hann')
|
||||
waves.append(wave)
|
||||
return np.array(waves)
|
||||
|
||||
|
||||
def _spec_to_batch(spec: np.ndarray, dim_f: int = DIM_F, dim_t: int = DIM_T) -> Tuple[np.ndarray, dict]:
|
||||
"""Convert spectrogram to batch format for MDX-Net."""
|
||||
# spec shape: (channels, freq, time, 2)
|
||||
b, f, t, c = spec.shape
|
||||
|
||||
# Pad if necessary
|
||||
pad_f = dim_f - f if f < dim_f else 0
|
||||
pad_t = dim_t - (t % dim_t) if t % dim_t != 0 else 0
|
||||
|
||||
if pad_f > 0 or pad_t > 0:
|
||||
spec = np.pad(spec, ((0, 0), (0, pad_f), (0, pad_t), (0, 0)))
|
||||
|
||||
# Reshape to batches
|
||||
spec = spec.transpose(0, 3, 1, 2) # (channels, 2, freq, time)
|
||||
|
||||
# Split into time chunks
|
||||
t_chunks = spec.shape[-1] // dim_t
|
||||
spec = spec.reshape(b, c, dim_f, t_chunks, dim_t)
|
||||
spec = spec.transpose(0, 3, 1, 2, 4) # (b, t_chunks, c, f, t)
|
||||
spec = spec.reshape(-1, c, dim_f, dim_t)
|
||||
|
||||
metadata = {
|
||||
'original_shape': (b, f, t, c),
|
||||
'pad_f': pad_f,
|
||||
'pad_t': pad_t,
|
||||
't_chunks': t_chunks,
|
||||
}
|
||||
|
||||
return spec, metadata
|
||||
|
||||
|
||||
def _batch_to_spec(batch: np.ndarray, metadata: dict) -> np.ndarray:
|
||||
"""Convert batch format back to spectrogram."""
|
||||
b, f, t, c = metadata['original_shape']
|
||||
pad_f = metadata['pad_f']
|
||||
pad_t = metadata['pad_t']
|
||||
t_chunks = metadata['t_chunks']
|
||||
|
||||
# Reshape back
|
||||
batch = batch.reshape(b, t_chunks, c, DIM_F, DIM_T)
|
||||
batch = batch.transpose(0, 2, 3, 1, 4) # (b, c, f, t_chunks, t)
|
||||
batch = batch.reshape(b, c, DIM_F, t_chunks * DIM_T)
|
||||
|
||||
# Remove padding
|
||||
spec = batch.transpose(0, 2, 3, 1) # (b, f, t, c)
|
||||
if pad_f > 0:
|
||||
spec = spec[:, :f, :, :]
|
||||
if pad_t > 0:
|
||||
spec = spec[:, :, :t, :]
|
||||
|
||||
return spec
|
||||
|
||||
|
||||
class MDXNetSeparator:
|
||||
"""
|
||||
MDX-Net separator using ONNX models from Ultimate Vocal Remover.
|
||||
Supports various models for vocals/instrumental/karaoke separation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_id: str,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
):
|
||||
if not ONNX_AVAILABLE:
|
||||
raise ImportError("onnxruntime is required for MDX-Net models. Install with: pip install onnxruntime")
|
||||
|
||||
if model_id not in MDXNET_MODELS:
|
||||
raise ValueError(f"Unknown MDX-Net model: {model_id}. Available: {list(MDXNET_MODELS.keys())}")
|
||||
|
||||
self.model_id = model_id
|
||||
self.model_config = MDXNET_MODELS[model_id]
|
||||
self.model_dir = Path(model_dir) if model_dir else Path('/opt/mdxnet_models')
|
||||
self.model_path = self.model_dir / self.model_config['path'].split('/')[-1]
|
||||
self.device = 'cpu' if cpu_separation else 'cuda'
|
||||
self.output_format = output_format.lower()
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self.stems = self.model_config['stems']
|
||||
self.target_stem = self.model_config['target_stem']
|
||||
self._session = None
|
||||
|
||||
def _ensure_model(self):
|
||||
"""Download model if not present."""
|
||||
if self.model_path.exists():
|
||||
return
|
||||
|
||||
self.model_dir.mkdir(parents=True, exist_ok=True)
|
||||
from huggingface_hub import hf_hub_download
|
||||
import shutil
|
||||
|
||||
# Download to HF cache then copy flat into model_dir so model_path is correct
|
||||
cached = hf_hub_download(
|
||||
repo_id=self.model_config['repo_id'],
|
||||
filename=self.model_config['path'],
|
||||
)
|
||||
shutil.copy2(cached, str(self.model_path))
|
||||
|
||||
def _get_session(self):
|
||||
"""Get or create ONNX runtime session."""
|
||||
if self._session is None:
|
||||
self._ensure_model()
|
||||
|
||||
providers = ['CPUExecutionProvider']
|
||||
if not self.device == 'cpu' and 'CUDAExecutionProvider' in ort.get_available_providers():
|
||||
providers = ['CUDAExecutionProvider', 'CPUExecutionProvider']
|
||||
|
||||
self._session = ort.InferenceSession(
|
||||
str(self.model_path),
|
||||
providers=providers
|
||||
)
|
||||
return self._session
|
||||
|
||||
def _demix(self, mix: np.ndarray) -> Dict[str, np.ndarray]:
|
||||
"""Separate audio into stems using MDX-Net."""
|
||||
session = self._get_session()
|
||||
input_name = session.get_inputs()[0].name
|
||||
|
||||
# Compute STFT
|
||||
mix_spec = _stft(mix) # (channels, freq, time, 2)
|
||||
|
||||
# Convert to batch format
|
||||
mix_batch, metadata = _spec_to_batch(mix_spec)
|
||||
|
||||
# Run inference in chunks
|
||||
output_batches = []
|
||||
batch_size = 1 # Process one chunk at a time for memory efficiency
|
||||
|
||||
with tqdm(total=len(mix_batch), desc='Separating', unit='chunk', ncols=100) as pbar:
|
||||
for i in range(0, len(mix_batch), batch_size):
|
||||
batch = mix_batch[i:i + batch_size]
|
||||
# ONNX expects (batch, channels, freq, time)
|
||||
outputs = session.run(None, {input_name: batch.astype(np.float32)})
|
||||
output_batches.append(outputs[0])
|
||||
pbar.update(len(batch))
|
||||
|
||||
# Concatenate outputs
|
||||
target_spec = np.concatenate(output_batches, axis=0)
|
||||
|
||||
# Convert back to spectrogram format
|
||||
target_spec = _batch_to_spec(target_spec, metadata)
|
||||
|
||||
# Inverse STFT to get audio
|
||||
target_audio = _istft(target_spec)
|
||||
|
||||
# Calculate the other stem (residual)
|
||||
other_audio = mix - target_audio
|
||||
|
||||
# Create output dict based on stems configuration
|
||||
sources = {}
|
||||
if self.target_stem == 'vocals':
|
||||
sources['vocals'] = target_audio
|
||||
sources['instrumental'] = other_audio
|
||||
elif self.target_stem == 'instrumental':
|
||||
sources['instrumental'] = target_audio
|
||||
sources['vocals'] = other_audio
|
||||
elif self.target_stem == 'lead':
|
||||
sources['lead'] = target_audio
|
||||
sources['backing_instrumental'] = other_audio
|
||||
else:
|
||||
sources[self.target_stem] = target_audio
|
||||
sources['other'] = other_audio
|
||||
|
||||
return sources
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
||||
"""Separate audio and write stems."""
|
||||
output_dir = Path(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Load audio
|
||||
audio = _load_audio(input_path)
|
||||
|
||||
# Separate
|
||||
sources = self._demix(audio)
|
||||
|
||||
# Cleanup
|
||||
if self._session is not None:
|
||||
del self._session
|
||||
self._session = None
|
||||
gc.collect()
|
||||
|
||||
# Save outputs
|
||||
ext = self.output_format
|
||||
result = {}
|
||||
for name, data in sources.items():
|
||||
dest = str(output_dir / f'{name}.{ext}')
|
||||
_save_audio(dest, data.T, ext, self.mp3_bitrate)
|
||||
result[name] = dest
|
||||
|
||||
return result
|
||||
Executable
+444
@@ -0,0 +1,444 @@
|
||||
"""
|
||||
MedleyVox separator - Multi-singer vocal separation
|
||||
HuggingFace model: Cyru5/MedleyVox
|
||||
Backup: miercolesv/MVSEP-Central-Backup
|
||||
|
||||
The MVSEP checkpoints are flat EMA state-dicts (OrderedDict with an
|
||||
'ema_model.module.' key prefix). This file contains an inline Conv-TasNet
|
||||
implementation whose architecture was reverse-engineered from those weights:
|
||||
N=2050 filters, L=2048 kernel, S=1024 stride,
|
||||
B=256 bottleneck, H=1024 hidden, P=3 TCN kernel,
|
||||
X=8 blocks/repeat, R=3 repeats, n_src=2 (vocals + instrumental).
|
||||
"""
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import torchaudio
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
from ..config import MEDLEYVOX_MODEL_DIR, CPU_SEPARATION
|
||||
|
||||
|
||||
# ── Model configurations ───────────────────────────────────────────────────────
|
||||
|
||||
MEDLEYVOX_MODELS = {
|
||||
"medleyvox": {
|
||||
"model_path": "vocals 100.pth",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "MedleyVox-MultiSinger",
|
||||
"stems": ["vocals", "instrumental"],
|
||||
},
|
||||
"medleyvox_55ep": {
|
||||
"model_path": "vocals 55 epochs.pth",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "MedleyVox-MultiSinger",
|
||||
"stems": ["vocals", "instrumental"],
|
||||
},
|
||||
"medleyvox_66ep": {
|
||||
"model_path": "vocals 66.pth",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "MedleyVox-MultiSinger",
|
||||
"stems": ["vocals", "instrumental"],
|
||||
},
|
||||
"medleyvox_88ep": {
|
||||
"model_path": "vocals 88.pth",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "MedleyVox-MultiSinger",
|
||||
"stems": ["vocals", "instrumental"],
|
||||
},
|
||||
"medleyvox_100ep": {
|
||||
"model_path": "vocals 100.pth",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "MedleyVox-MultiSinger",
|
||||
"stems": ["vocals", "instrumental"],
|
||||
},
|
||||
}
|
||||
|
||||
# Sample rate expected by all MedleyVox checkpoints
|
||||
_TARGET_SR = 44100
|
||||
# Chunk length for streaming inference (seconds) — keeps VRAM bounded
|
||||
_CHUNK_SECS = 30
|
||||
|
||||
|
||||
# ── Inline Conv-TasNet (architecture reverse-engineered from checkpoint) ────────
|
||||
|
||||
class _GlobalLayerNorm(nn.Module):
|
||||
"""Global Layer Norm over (channel × time) with learned gamma/beta."""
|
||||
|
||||
def __init__(self, n_feats: int, eps: float = 1e-8):
|
||||
super().__init__()
|
||||
self.gamma = nn.Parameter(torch.ones(n_feats))
|
||||
self.beta = nn.Parameter(torch.zeros(n_feats))
|
||||
self.eps = eps
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
# x: (B, C, T) — normalise over C and T jointly
|
||||
mean = x.mean(dim=(1, 2), keepdim=True)
|
||||
var = ((x - mean) ** 2).mean(dim=(1, 2), keepdim=True)
|
||||
return (
|
||||
self.gamma[None, :, None] * (x - mean) / (var + self.eps).sqrt()
|
||||
+ self.beta[None, :, None]
|
||||
)
|
||||
|
||||
|
||||
class _Conv1DBlock(nn.Module):
|
||||
"""One TCN block: pointwise expand → depthwise → residual + skip."""
|
||||
|
||||
def __init__(self, in_chan: int, hid_chan: int,
|
||||
kernel_size: int, dilation: int):
|
||||
super().__init__()
|
||||
pad = dilation * (kernel_size - 1) // 2
|
||||
self.shared_block = nn.Sequential(
|
||||
nn.Conv1d(in_chan, hid_chan, 1),
|
||||
nn.PReLU(),
|
||||
_GlobalLayerNorm(hid_chan),
|
||||
nn.Conv1d(hid_chan, hid_chan, kernel_size,
|
||||
dilation=dilation, groups=hid_chan, padding=pad),
|
||||
nn.PReLU(),
|
||||
_GlobalLayerNorm(hid_chan),
|
||||
)
|
||||
self.res_conv = nn.Conv1d(hid_chan, in_chan, 1)
|
||||
self.skip_conv = nn.Conv1d(hid_chan, in_chan, 1)
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
y = self.shared_block(x)
|
||||
return x + self.res_conv(y), self.skip_conv(y)
|
||||
|
||||
|
||||
class _Filterbank(nn.Module):
|
||||
"""Learnable windowed filterbank stored as _filters + torch_window buffer."""
|
||||
|
||||
def __init__(self, n_filters: int, kernel_size: int):
|
||||
super().__init__()
|
||||
self._filters = nn.Parameter(torch.empty(n_filters, 1, kernel_size))
|
||||
self.register_buffer("torch_window", torch.hann_window(kernel_size))
|
||||
|
||||
@property
|
||||
def filters(self) -> torch.Tensor:
|
||||
"""Windowed filters ready for conv1d / conv_transpose1d."""
|
||||
return self._filters * self.torch_window
|
||||
|
||||
|
||||
class _Encoder(nn.Module):
|
||||
def __init__(self, n_filters: int, kernel_size: int, stride: int):
|
||||
super().__init__()
|
||||
self.filterbank = _Filterbank(n_filters, kernel_size)
|
||||
self._stride = stride
|
||||
self._pad = (kernel_size - stride) // 2
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
# x: (B, 1, T) → (B, N, T')
|
||||
out = F.conv1d(x, self.filterbank.filters,
|
||||
stride=self._stride, padding=self._pad)
|
||||
return F.relu(out)
|
||||
|
||||
|
||||
class _Masker(nn.Module):
|
||||
def __init__(self, n_filters: int, n_feats: int, n_hidden: int,
|
||||
tcn_kernel: int, n_blocks: int, n_repeats: int, n_src: int):
|
||||
super().__init__()
|
||||
self.bottleneck = nn.Sequential(
|
||||
_GlobalLayerNorm(n_filters),
|
||||
nn.Conv1d(n_filters, n_feats, 1),
|
||||
)
|
||||
blocks = []
|
||||
for _ in range(n_repeats):
|
||||
for b in range(n_blocks):
|
||||
blocks.append(
|
||||
_Conv1DBlock(n_feats, n_hidden, tcn_kernel, dilation=2 ** b)
|
||||
)
|
||||
self.TCN = nn.ModuleList(blocks)
|
||||
self.mask_net = nn.Sequential(
|
||||
nn.PReLU(),
|
||||
nn.Conv1d(n_feats, n_filters * n_src, 1),
|
||||
)
|
||||
self._n_src = n_src
|
||||
self._n_filters = n_filters
|
||||
|
||||
def forward(self, enc: torch.Tensor) -> torch.Tensor:
|
||||
# enc: (B, N, T') → masks: (B, n_src, N, T')
|
||||
feat = self.bottleneck(enc)
|
||||
skip = torch.zeros_like(feat)
|
||||
for block in self.TCN:
|
||||
feat, s = block(feat)
|
||||
skip = skip + s
|
||||
masks = self.mask_net(skip) # (B, N*n_src, T')
|
||||
B = enc.shape[0]
|
||||
masks = masks.view(B, self._n_src, self._n_filters, -1)
|
||||
return torch.sigmoid(masks)
|
||||
|
||||
|
||||
class _Decoder(nn.Module):
|
||||
def __init__(self, n_filters: int, kernel_size: int, stride: int):
|
||||
super().__init__()
|
||||
self.filterbank = _Filterbank(n_filters, kernel_size)
|
||||
self._stride = stride
|
||||
self._pad = (kernel_size - stride) // 2
|
||||
|
||||
def forward(self, x: torch.Tensor, orig_len: int) -> torch.Tensor:
|
||||
# x: (B*n_src, N, T') → (B*n_src, orig_len)
|
||||
out = F.conv_transpose1d(x, self.filterbank.filters,
|
||||
stride=self._stride, padding=self._pad)
|
||||
return out.squeeze(1)[..., :orig_len]
|
||||
|
||||
|
||||
class _ConvTasNet(nn.Module):
|
||||
"""
|
||||
Conv-TasNet for MedleyVox (2-source: vocals + instrumental).
|
||||
Hyperparameters inferred from 'vocals 100.pth' weight shapes:
|
||||
N=2050, L=2048, S=1024, B=256, H=1024, P=3, X=8, R=3, n_src=2
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
N, L, S = 2050, 2048, 1024 # encoder/decoder params
|
||||
B, H, P, X, R, Ns = 256, 1024, 3, 8, 3, 2
|
||||
|
||||
self.encoder = _Encoder(N, L, S)
|
||||
self.masker = _Masker(N, B, H, P, X, R, Ns)
|
||||
self.decoder = _Decoder(N, L, S)
|
||||
self._n_src = Ns
|
||||
self._n_filters = N
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
# x: (B, 1, T) or (B, T) → (B, n_src, T)
|
||||
if x.dim() == 2:
|
||||
x = x.unsqueeze(1)
|
||||
orig_len = x.shape[-1]
|
||||
B = x.shape[0]
|
||||
|
||||
enc = self.encoder(x) # (B, N, T')
|
||||
masks = self.masker(enc) # (B, n_src, N, T')
|
||||
|
||||
masked = (enc.unsqueeze(1) * masks).view(
|
||||
B * self._n_src, self._n_filters, -1
|
||||
) # (B*n_src, N, T')
|
||||
|
||||
sources = self.decoder(masked, orig_len) # (B*n_src, T)
|
||||
return sources.view(B, self._n_src, orig_len) # (B, n_src, T)
|
||||
|
||||
|
||||
# ── Separator ──────────────────────────────────────────────────────────────────
|
||||
|
||||
class MedleyVoxSeparator:
|
||||
"""
|
||||
MedleyVox separator for multi-singer vocal separation.
|
||||
Specialized for separating vocals when multiple singers are present.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_id: str = "medleyvox",
|
||||
cpu_separation: bool = CPU_SEPARATION,
|
||||
output_format: str = "mp3",
|
||||
mp3_bitrate: str = "256k",
|
||||
):
|
||||
if model_id not in MEDLEYVOX_MODELS:
|
||||
raise ValueError(f"Unknown MedleyVox model: {model_id}")
|
||||
|
||||
self.model_id = model_id
|
||||
self.model_config = MEDLEYVOX_MODELS[model_id]
|
||||
self.cpu_separation = cpu_separation
|
||||
self.output_format = output_format
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self._model = None
|
||||
self._device = None
|
||||
|
||||
# ── Model download ────────────────────────────────────────────────────────
|
||||
|
||||
def _ensure_model(self) -> Path:
|
||||
"""Download checkpoint from HuggingFace if not already present."""
|
||||
model_dir = MEDLEYVOX_MODEL_DIR / self.model_id
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
model_path = model_dir / self.model_config["model_path"]
|
||||
|
||||
if not model_path.exists():
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
print(f"Downloading MedleyVox model: {self.model_id}")
|
||||
hf_hub_download(
|
||||
repo_id=self.model_config["hf_repo"],
|
||||
filename=f"{self.model_config['hf_subfolder']}/{self.model_config['model_path']}",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to download MedleyVox model: {e}")
|
||||
|
||||
return model_path
|
||||
|
||||
# ── Model loading ─────────────────────────────────────────────────────────
|
||||
|
||||
def _load_model(self):
|
||||
"""
|
||||
Load the checkpoint.
|
||||
|
||||
Tries four strategies in order:
|
||||
1. Checkpoint IS an nn.Module (torch.save(model, path))
|
||||
2. Dict with a model/net/network/module key holding an nn.Module
|
||||
3. Dict with an EMA online/shadow key holding an nn.Module
|
||||
4. Flat state dict (possibly prefixed with 'ema_model.module.' or
|
||||
'module.') — loaded into an inline _ConvTasNet instance.
|
||||
"""
|
||||
if self._model is not None:
|
||||
return self._model
|
||||
|
||||
model_path = self._ensure_model()
|
||||
self._device = (
|
||||
"cpu" if self.cpu_separation
|
||||
else ("cuda" if torch.cuda.is_available() else "cpu")
|
||||
)
|
||||
|
||||
# weights_only=False is required to deserialise a full model object.
|
||||
# These checkpoints come from a trusted HuggingFace repository.
|
||||
checkpoint = torch.load(str(model_path), map_location="cpu", weights_only=False)
|
||||
|
||||
# Case 1: checkpoint IS the model (torch.save(model, path))
|
||||
if callable(getattr(checkpoint, "forward", None)):
|
||||
self._model = checkpoint.to(self._device).eval()
|
||||
return self._model
|
||||
|
||||
if isinstance(checkpoint, dict):
|
||||
# Case 2: simple dict wrapper {"model": <nn.Module>, ...}
|
||||
for key in ("model", "net", "network", "module"):
|
||||
inner = checkpoint.get(key)
|
||||
if inner is not None and callable(getattr(inner, "forward", None)):
|
||||
self._model = inner.to(self._device).eval()
|
||||
return self._model
|
||||
|
||||
# Case 3: EMA training wrapper — model stored under the online/shadow key
|
||||
for key in ("online_model", "online", "ema_model", "ema",
|
||||
"shadow_model", "shadow", "averaged_model"):
|
||||
inner = checkpoint.get(key)
|
||||
if inner is not None and callable(getattr(inner, "forward", None)):
|
||||
self._model = inner.to(self._device).eval()
|
||||
return self._model
|
||||
|
||||
# Case 4: flat EMA state dict — strip common prefix, load into _ConvTasNet.
|
||||
# MVSEP checkpoints use ema-pytorch which saves a flat OrderedDict
|
||||
# with keys prefixed by 'ema_model.module.'.
|
||||
for prefix in ("ema_model.module.", "module.", ""):
|
||||
if prefix and not any(
|
||||
k.startswith(prefix) for k in checkpoint.keys()
|
||||
):
|
||||
continue
|
||||
stripped = {
|
||||
(k[len(prefix):] if k.startswith(prefix) else k): v
|
||||
for k, v in checkpoint.items()
|
||||
}
|
||||
net = _ConvTasNet()
|
||||
result = net.load_state_dict(stripped, strict=False)
|
||||
if not result.missing_keys:
|
||||
if result.unexpected_keys:
|
||||
print(
|
||||
f"[MedleyVox] Ignoring {len(result.unexpected_keys)} "
|
||||
"unexpected checkpoint keys."
|
||||
)
|
||||
self._model = net.to(self._device).eval()
|
||||
return self._model
|
||||
|
||||
keys = list(checkpoint.keys()) if isinstance(checkpoint, dict) else "N/A"
|
||||
raise RuntimeError(
|
||||
f"Cannot load MedleyVox checkpoint '{model_path.name}'. "
|
||||
f"Expected a full torch.nn.Module or compatible state dict but got "
|
||||
f"{type(checkpoint).__name__} (keys: {str(keys)[:300]}). "
|
||||
"The checkpoint format may have changed — check the HuggingFace repo."
|
||||
)
|
||||
|
||||
# ── Inference ─────────────────────────────────────────────────────────────
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
||||
"""
|
||||
Separate audio into stems (vocals / instrumental).
|
||||
|
||||
Returns:
|
||||
Dict mapping stem name → output file path.
|
||||
"""
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
model = self._load_model()
|
||||
|
||||
# Load and normalise to mono at the model's expected sample rate
|
||||
audio, sr = torchaudio.load(input_path)
|
||||
if sr != _TARGET_SR:
|
||||
audio = torchaudio.functional.resample(audio, sr, _TARGET_SR)
|
||||
sr = _TARGET_SR
|
||||
if audio.shape[0] > 1:
|
||||
mix = audio.mean(dim=0, keepdim=True) # (1, T)
|
||||
else:
|
||||
mix = audio
|
||||
|
||||
mix = mix.to(self._device)
|
||||
chunk_samples = sr * _CHUNK_SECS
|
||||
|
||||
# Process in chunks so GPU memory stays bounded for long tracks
|
||||
source_chunks: list[torch.Tensor] = []
|
||||
for start in range(0, mix.shape[-1], chunk_samples):
|
||||
chunk = mix[:, start : start + chunk_samples] # (1, T')
|
||||
with torch.no_grad():
|
||||
out = model(chunk.unsqueeze(0)) # (1, n_src, T')
|
||||
out = out.squeeze(0).cpu() # (n_src, T')
|
||||
source_chunks.append(out)
|
||||
|
||||
# Concatenate along the time axis (last dim)
|
||||
sources = torch.cat(source_chunks, dim=-1) # (n_src, T)
|
||||
|
||||
# Save each stem to the target format
|
||||
stems = self.model_config.get("stems", ["vocals", "instrumental"])
|
||||
output_paths: Dict[str, str] = {}
|
||||
|
||||
for i, stem_name in enumerate(stems):
|
||||
if i >= sources.shape[0]:
|
||||
break
|
||||
stem_audio = sources[i]
|
||||
# Ensure shape is (channels, T)
|
||||
if stem_audio.dim() == 1:
|
||||
stem_audio = stem_audio.unsqueeze(0)
|
||||
elif stem_audio.dim() == 3:
|
||||
# (channels, 1, T) → squeeze middle dim
|
||||
stem_audio = stem_audio.squeeze(1)
|
||||
|
||||
self._save_stem(stem_audio, sr, stem_name, output_path, output_paths)
|
||||
|
||||
return output_paths
|
||||
|
||||
# ── Output helpers ────────────────────────────────────────────────────────
|
||||
|
||||
def _save_stem(
|
||||
self,
|
||||
audio: torch.Tensor,
|
||||
sample_rate: int,
|
||||
stem_name: str,
|
||||
output_dir: Path,
|
||||
output_paths: Dict[str, str],
|
||||
) -> None:
|
||||
"""Save a stem tensor in the configured output format."""
|
||||
fmt = self.output_format.lower()
|
||||
|
||||
if fmt in ("mp3", "mp3_320"):
|
||||
# Write to WAV first, then encode with pydub
|
||||
wav_path = output_dir / f"{stem_name}_tmp.wav"
|
||||
torchaudio.save(str(wav_path), audio, sample_rate)
|
||||
final_path = output_dir / f"{stem_name}.mp3"
|
||||
bitrate = self.mp3_bitrate or ("320k" if fmt == "mp3_320" else "256k")
|
||||
from pydub import AudioSegment
|
||||
AudioSegment.from_wav(str(wav_path)).export(
|
||||
str(final_path), format="mp3", bitrate=bitrate
|
||||
)
|
||||
wav_path.unlink(missing_ok=True)
|
||||
elif fmt == "flac":
|
||||
wav_path = output_dir / f"{stem_name}_tmp.wav"
|
||||
torchaudio.save(str(wav_path), audio, sample_rate)
|
||||
final_path = output_dir / f"{stem_name}.flac"
|
||||
from pydub import AudioSegment
|
||||
AudioSegment.from_wav(str(wav_path)).export(str(final_path), format="flac")
|
||||
wav_path.unlink(missing_ok=True)
|
||||
else:
|
||||
# Default: WAV
|
||||
final_path = output_dir / f"{stem_name}.wav"
|
||||
torchaudio.save(str(final_path), audio, sample_rate)
|
||||
|
||||
output_paths[stem_name] = str(final_path)
|
||||
+336
@@ -0,0 +1,336 @@
|
||||
"""
|
||||
MelBand RoFormer separator — standalone, no Django/Spleeter/TensorFlow dependency.
|
||||
|
||||
Audio I/O uses soundfile (WAV/FLAC) + subprocess ffmpeg (MP3 encoding).
|
||||
Model architecture is from https://github.com/lucidrains/BS-RoFormer
|
||||
|
||||
This module patches sys.modules to fix the import path for models.bs_roformer.attend
|
||||
which the melband files expect but doesn't exist in this project structure.
|
||||
"""
|
||||
import gc
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import yaml
|
||||
from omegaconf import OmegaConf
|
||||
from tqdm import tqdm
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Import path patching
|
||||
# The melband files import from models.bs_roformer.attend which doesn't exist
|
||||
# in this project structure. We patch sys.modules to redirect to our bs_roformer.
|
||||
# ---------------------------------------------------------------------------
|
||||
if 'models' not in sys.modules:
|
||||
sys.modules['models'] = type(sys)('models')
|
||||
if 'models.bs_roformer' not in sys.modules:
|
||||
import backend.services.bs_roformer as _bs_roformer
|
||||
sys.modules['models.bs_roformer'] = _bs_roformer
|
||||
if 'models.bs_roformer.attend' not in sys.modules:
|
||||
from backend.services.bs_roformer import attend as _attend
|
||||
sys.modules['models.bs_roformer.attend'] = _attend
|
||||
|
||||
# Import MelBandRoformer from the bs_roformer pip package (lucidrains/BS-RoFormer)
|
||||
from bs_roformer.mel_band_roformer import MelBandRoformer
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HuggingFace model coordinates
|
||||
# ---------------------------------------------------------------------------
|
||||
HF_REPO_ID = 'fabiankruger/mel-band-roformer-4-stem'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Default model paths (overridable via env var MEL_BAND_MODEL_DIR)
|
||||
# ---------------------------------------------------------------------------
|
||||
_MODEL_DIR_ENV = os.environ.get('MEL_BAND_MODEL_DIR', '/opt/mel_band_models')
|
||||
DEFAULT_MODEL_DIR = Path(_MODEL_DIR_ENV)
|
||||
DEFAULT_MODEL_PATH = DEFAULT_MODEL_DIR / 'mel_band_roformer_4_stem.ckpt'
|
||||
DEFAULT_CONFIG_PATH = DEFAULT_MODEL_DIR / 'config.yaml'
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _try_download_model(
|
||||
model_dir: Path = DEFAULT_MODEL_DIR,
|
||||
model_path: Path = DEFAULT_MODEL_PATH,
|
||||
config_path: Path = DEFAULT_CONFIG_PATH,
|
||||
) -> None:
|
||||
"""Download MelBand RoFormer model from HuggingFace if not already present."""
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
from huggingface_hub import snapshot_download
|
||||
snapshot_download(
|
||||
repo_id=HF_REPO_ID,
|
||||
revision=HF_REVISION,
|
||||
local_dir=str(model_dir),
|
||||
)
|
||||
|
||||
|
||||
def _load_config(config_path: Path):
|
||||
"""Load YAML config, supporting !!python/tuple tags."""
|
||||
with open(config_path, 'r') as f:
|
||||
config = yaml.load(f, Loader=yaml.UnsafeLoader)
|
||||
return OmegaConf.create(config)
|
||||
|
||||
|
||||
def _load_audio(path: str) -> np.ndarray:
|
||||
"""
|
||||
Load audio file and return as float32 numpy array of shape (channels, samples).
|
||||
Uses soundfile for WAV/FLAC; falls back to ffmpeg decode for MP3 and other formats.
|
||||
"""
|
||||
import soundfile as sf
|
||||
|
||||
try:
|
||||
data, sr = sf.read(path, dtype='float32', always_2d=True)
|
||||
# soundfile returns (samples, channels); transpose to (channels, samples)
|
||||
data = data.T
|
||||
except Exception:
|
||||
# Fallback: decode via ffmpeg to a temporary WAV
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', path, '-ar', str(SAMPLE_RATE),
|
||||
'-ac', '2', '-f', 'wav', tmp_path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
data, sr = sf.read(tmp_path, dtype='float32', always_2d=True)
|
||||
data = data.T
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
# Resample if needed
|
||||
if sr != SAMPLE_RATE:
|
||||
import librosa
|
||||
data = np.array([
|
||||
librosa.resample(ch, orig_sr=sr, target_sr=SAMPLE_RATE)
|
||||
for ch in data
|
||||
], dtype=np.float32)
|
||||
|
||||
# Ensure stereo
|
||||
if data.shape[0] == 1:
|
||||
data = np.vstack([data, data])
|
||||
|
||||
return data # (channels, samples)
|
||||
|
||||
|
||||
def _save_audio(path: str, data: np.ndarray, output_format: str, bitrate: str = '256k') -> None:
|
||||
"""
|
||||
Save (samples, channels) float32 array as audio file.
|
||||
WAV and FLAC use soundfile directly; MP3 encodes via ffmpeg.
|
||||
"""
|
||||
import soundfile as sf
|
||||
|
||||
ext = output_format.lower()
|
||||
|
||||
if ext in ('wav', 'flac'):
|
||||
sf.write(path, data, SAMPLE_RATE, format=ext.upper(),
|
||||
subtype='PCM_24' if ext == 'wav' else 'PCM_16')
|
||||
else:
|
||||
# Encode to MP3 via ffmpeg
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
sf.write(tmp_path, data, SAMPLE_RATE, format='WAV', subtype='PCM_16')
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', tmp_path,
|
||||
'-b:a', bitrate, path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Separator class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class MelBandRoformerSeparator:
|
||||
"""
|
||||
Performs source separation using the MelBand RoFormer model.
|
||||
Pure PyTorch — no TensorFlow / Spleeter dependency.
|
||||
"""
|
||||
|
||||
# MelBand RoFormer outputs 4 stems by default
|
||||
DEFAULT_STEMS = ['vocals', 'drums', 'bass', 'other']
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_path: Optional[str] = None,
|
||||
config_path: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3', # 'mp3' | 'flac' | 'wav'
|
||||
mp3_bitrate: str = '256k',
|
||||
):
|
||||
self.model_path = Path(model_path) if model_path else DEFAULT_MODEL_PATH
|
||||
self.config_path = Path(config_path) if config_path else DEFAULT_CONFIG_PATH
|
||||
self.device = 'cpu' if cpu_separation else 'cuda'
|
||||
self.output_format = output_format.lower()
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self._config = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Model loading
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _ensure_model(self):
|
||||
_try_download_model(DEFAULT_MODEL_DIR, self.model_path, self.config_path)
|
||||
|
||||
def _load_config(self):
|
||||
if self._config is None:
|
||||
self._config = _load_config(self.config_path)
|
||||
return self._config
|
||||
|
||||
def get_model(self) -> MelBandRoformer:
|
||||
self._ensure_model()
|
||||
config = self._load_config()
|
||||
model_cfg = OmegaConf.to_container(config.model, resolve=True)
|
||||
|
||||
# OmegaConf loads tuples as lists; the model requires tuples for certain fields
|
||||
if 'freqs_per_bands' in model_cfg and isinstance(model_cfg['freqs_per_bands'], list):
|
||||
model_cfg['freqs_per_bands'] = tuple(model_cfg['freqs_per_bands'])
|
||||
if 'multi_stft_resolutions_window_sizes' in model_cfg and isinstance(model_cfg['multi_stft_resolutions_window_sizes'], list):
|
||||
model_cfg['multi_stft_resolutions_window_sizes'] = tuple(model_cfg['multi_stft_resolutions_window_sizes'])
|
||||
|
||||
model = MelBandRoformer(**model_cfg)
|
||||
checkpoint = torch.load(str(self.model_path), map_location='cpu', weights_only=False)
|
||||
|
||||
# Handle different checkpoint formats
|
||||
if 'state_dict' in checkpoint:
|
||||
state_dict = checkpoint['state_dict']
|
||||
elif 'model' in checkpoint:
|
||||
state_dict = checkpoint['model']
|
||||
else:
|
||||
state_dict = checkpoint
|
||||
|
||||
model.load_state_dict(state_dict)
|
||||
model.to(self.device).eval()
|
||||
return model
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Inference
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _demix(self, mix: np.ndarray, model: MelBandRoformer,
|
||||
batch_size_override: Optional[int] = None,
|
||||
num_overlap_override: Optional[int] = None) -> Dict[str, np.ndarray]:
|
||||
"""Separate (channels, samples) array into a dict of stem arrays."""
|
||||
config = self._load_config()
|
||||
hop_length = config.model.stft_hop_length
|
||||
segment_size = getattr(config.inference, 'dim_t', 2000)
|
||||
C = hop_length * (segment_size - 1)
|
||||
N = num_overlap_override if num_overlap_override is not None else getattr(config.inference, 'num_overlap', 4)
|
||||
step = int(C // N)
|
||||
fade_size = C // 10
|
||||
batch_size = batch_size_override if batch_size_override is not None else getattr(config.inference, 'batch_size', 1)
|
||||
device = self.device
|
||||
|
||||
mix_t = torch.tensor(mix, dtype=torch.float32)
|
||||
length_init = mix_t.shape[-1]
|
||||
|
||||
if length_init > 2 * (C - step) and (C - step > 0):
|
||||
mix_t = nn.functional.pad(mix_t, (C - step, C - step), mode='reflect')
|
||||
|
||||
fadein = torch.linspace(0, 1, fade_size).to(device)
|
||||
fadeout = torch.linspace(1, 0, fade_size).to(device)
|
||||
window_start = torch.ones(C).to(device)
|
||||
window_start[-fade_size:] *= fadeout
|
||||
window_finish = torch.ones(C).to(device)
|
||||
window_finish[:fade_size] *= fadein
|
||||
window_middle = torch.ones(C).to(device)
|
||||
window_middle[:fade_size] *= fadein
|
||||
window_middle[-fade_size:] *= fadeout
|
||||
|
||||
# Get stems from config or use default
|
||||
stems = getattr(config.training, 'instruments', self.DEFAULT_STEMS)
|
||||
num_stems = len(stems)
|
||||
req_shape = (num_stems,) + tuple(mix_t.shape)
|
||||
|
||||
with torch.inference_mode():
|
||||
result = torch.zeros(req_shape, dtype=torch.float32, device=device)
|
||||
counter = torch.zeros(req_shape, dtype=torch.float32, device=device)
|
||||
batch_data, batch_locs = [], []
|
||||
i = 0
|
||||
|
||||
# Count total segments for tqdm
|
||||
total_seg = sum(1 for _ in range(0, mix_t.shape[1], step))
|
||||
|
||||
with tqdm(total=total_seg, desc='Separating', unit='segment', ncols=100) as pbar:
|
||||
while i < mix_t.shape[1]:
|
||||
part = mix_t[:, i:i + C].to(device)
|
||||
l = part.shape[-1]
|
||||
if l < C:
|
||||
part = nn.functional.pad(
|
||||
part,
|
||||
(0, C - l) if l > C // 2 + 1 else (0, C - l, 0, 0),
|
||||
mode='reflect' if l > C // 2 + 1 else 'constant',
|
||||
)
|
||||
batch_data.append(part)
|
||||
batch_locs.append((i, l))
|
||||
i += step
|
||||
|
||||
if len(batch_data) >= batch_size or i >= mix_t.shape[1]:
|
||||
arr = torch.stack(batch_data, dim=0)
|
||||
x = model(arr)
|
||||
for j, (start, ln) in enumerate(batch_locs):
|
||||
window = (window_start if start == 0
|
||||
else window_finish if i >= mix_t.shape[1]
|
||||
else window_middle)
|
||||
result[..., start:start + ln] += x[j][..., :ln] * window[..., :ln]
|
||||
counter[..., start:start + ln] += window[..., :ln]
|
||||
pbar.update(len(batch_data))
|
||||
batch_data, batch_locs = [], []
|
||||
|
||||
estimated = result / counter.clamp(min=1e-10)
|
||||
if length_init > 2 * (C - step) and (C - step > 0):
|
||||
estimated = estimated[..., (C - step):-(C - step)]
|
||||
|
||||
sources = {}
|
||||
for idx, name in enumerate(stems):
|
||||
sources[name] = estimated[idx].cpu().numpy()
|
||||
return sources
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str,
|
||||
batch_size_override: Optional[int] = None,
|
||||
num_overlap_override: Optional[int] = None) -> Dict[str, str]:
|
||||
"""
|
||||
Separate audio at input_path and write one file per stem into output_dir.
|
||||
Returns {stem_name: output_file_path}.
|
||||
"""
|
||||
output_dir = Path(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
audio = _load_audio(input_path) # (channels, samples)
|
||||
model = self.get_model()
|
||||
sources = self._demix(audio, model,
|
||||
batch_size_override=batch_size_override,
|
||||
num_overlap_override=num_overlap_override)
|
||||
|
||||
del model
|
||||
torch.cuda.empty_cache()
|
||||
gc.collect()
|
||||
|
||||
ext = self.output_format
|
||||
result = {}
|
||||
for name, data in sources.items():
|
||||
dest = str(output_dir / f'{name}.{ext}')
|
||||
_save_audio(dest, data.T, ext, self.mp3_bitrate) # T: (channels,samples)→(samples,channels)
|
||||
result[name] = dest
|
||||
return result
|
||||
+669
@@ -0,0 +1,669 @@
|
||||
"""
|
||||
Specialized MelBand RoFormer separators for specific use cases:
|
||||
- Vocals/Instrumental separation (2-stem)
|
||||
- Dereverb (dry vocals vs reverb)
|
||||
- Karaoke (lead vocals vs backing+instrumental)
|
||||
|
||||
Uses sys.modules patching for import path fixes.
|
||||
"""
|
||||
import gc
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional, List
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import yaml
|
||||
from omegaconf import OmegaConf
|
||||
from tqdm import tqdm
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Import path patching
|
||||
# ---------------------------------------------------------------------------
|
||||
if 'models' not in sys.modules:
|
||||
sys.modules['models'] = type(sys)('models')
|
||||
if 'models.bs_roformer' not in sys.modules:
|
||||
import backend.services.bs_roformer as _bs_roformer
|
||||
sys.modules['models.bs_roformer'] = _bs_roformer
|
||||
if 'models.bs_roformer.attend' not in sys.modules:
|
||||
from backend.services.bs_roformer import attend as _attend
|
||||
sys.modules['models.bs_roformer.attend'] = _attend
|
||||
|
||||
from bs_roformer.mel_band_roformer import MelBandRoformer
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
|
||||
def _load_audio(path: str) -> np.ndarray:
|
||||
"""Load audio as (channels, samples) float32 array."""
|
||||
import soundfile as sf
|
||||
|
||||
try:
|
||||
data, sr = sf.read(path, dtype='float32', always_2d=True)
|
||||
data = data.T
|
||||
except Exception:
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', path, '-ar', str(SAMPLE_RATE),
|
||||
'-ac', '2', '-f', 'wav', tmp_path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
data, sr = sf.read(tmp_path, dtype='float32', always_2d=True)
|
||||
data = data.T
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
if sr != SAMPLE_RATE:
|
||||
import librosa
|
||||
data = np.array([
|
||||
librosa.resample(ch, orig_sr=sr, target_sr=SAMPLE_RATE)
|
||||
for ch in data
|
||||
], dtype=np.float32)
|
||||
|
||||
if data.shape[0] == 1:
|
||||
data = np.vstack([data, data])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def _save_audio(path: str, data: np.ndarray, output_format: str, bitrate: str = '256k') -> None:
|
||||
"""Save (samples, channels) array to audio file."""
|
||||
import soundfile as sf
|
||||
|
||||
ext = output_format.lower()
|
||||
|
||||
if ext in ('wav', 'flac'):
|
||||
sf.write(path, data, SAMPLE_RATE, format=ext.upper(),
|
||||
subtype='PCM_24' if ext == 'wav' else 'PCM_16')
|
||||
else:
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
sf.write(tmp_path, data, SAMPLE_RATE, format='WAV', subtype='PCM_16')
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', tmp_path, '-b:a', bitrate, path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
|
||||
class MelBandSeparatorBase:
|
||||
"""Base class for MelBand RoFormer separators."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_path: Path,
|
||||
config_path: Path,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
stems: List[str] = None,
|
||||
):
|
||||
self.model_path = model_path
|
||||
self.config_path = config_path
|
||||
self.device = 'cpu' if cpu_separation else 'cuda'
|
||||
self.output_format = output_format.lower()
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self.stems = stems or ['vocals', 'instrumental']
|
||||
self._config = None
|
||||
|
||||
def _load_config(self):
|
||||
if self._config is None:
|
||||
with open(self.config_path, 'r') as f:
|
||||
config = yaml.load(f, Loader=yaml.UnsafeLoader)
|
||||
self._config = OmegaConf.create(config)
|
||||
return self._config
|
||||
|
||||
def get_model(self) -> MelBandRoformer:
|
||||
config = self._load_config()
|
||||
model_cfg = OmegaConf.to_container(config.model, resolve=True)
|
||||
|
||||
# Convert lists to tuples where needed
|
||||
if 'freqs_per_bands' in model_cfg and isinstance(model_cfg['freqs_per_bands'], list):
|
||||
model_cfg['freqs_per_bands'] = tuple(model_cfg['freqs_per_bands'])
|
||||
if 'multi_stft_resolutions_window_sizes' in model_cfg and isinstance(model_cfg['multi_stft_resolutions_window_sizes'], list):
|
||||
model_cfg['multi_stft_resolutions_window_sizes'] = tuple(model_cfg['multi_stft_resolutions_window_sizes'])
|
||||
|
||||
model = MelBandRoformer(**model_cfg)
|
||||
checkpoint = torch.load(str(self.model_path), map_location='cpu', weights_only=False)
|
||||
|
||||
if 'state_dict' in checkpoint:
|
||||
state_dict = checkpoint['state_dict']
|
||||
elif 'model' in checkpoint:
|
||||
state_dict = checkpoint['model']
|
||||
else:
|
||||
state_dict = checkpoint
|
||||
|
||||
model.load_state_dict(state_dict)
|
||||
model.to(self.device).eval()
|
||||
return model
|
||||
|
||||
def _demix(self, mix: np.ndarray, model: MelBandRoformer,
|
||||
batch_size_override: Optional[int] = None,
|
||||
num_overlap_override: Optional[int] = None) -> Dict[str, np.ndarray]:
|
||||
"""Separate audio into stems."""
|
||||
config = self._load_config()
|
||||
hop_length = config.model.stft_hop_length
|
||||
segment_size = getattr(config.inference, 'dim_t', 2000)
|
||||
C = hop_length * (segment_size - 1)
|
||||
N = num_overlap_override if num_overlap_override is not None else getattr(config.inference, 'num_overlap', 4)
|
||||
step = int(C // N)
|
||||
fade_size = C // 10
|
||||
batch_size = batch_size_override if batch_size_override is not None else getattr(config.inference, 'batch_size', 1)
|
||||
device = self.device
|
||||
|
||||
mix_t = torch.tensor(mix, dtype=torch.float32)
|
||||
length_init = mix_t.shape[-1]
|
||||
|
||||
if length_init > 2 * (C - step) and (C - step > 0):
|
||||
mix_t = nn.functional.pad(mix_t, (C - step, C - step), mode='reflect')
|
||||
|
||||
fadein = torch.linspace(0, 1, fade_size).to(device)
|
||||
fadeout = torch.linspace(1, 0, fade_size).to(device)
|
||||
window_start = torch.ones(C).to(device)
|
||||
window_start[-fade_size:] *= fadeout
|
||||
window_finish = torch.ones(C).to(device)
|
||||
window_finish[:fade_size] *= fadein
|
||||
window_middle = torch.ones(C).to(device)
|
||||
window_middle[:fade_size] *= fadein
|
||||
window_middle[-fade_size:] *= fadeout
|
||||
|
||||
num_stems = len(self.stems)
|
||||
req_shape = (num_stems,) + tuple(mix_t.shape)
|
||||
|
||||
with torch.inference_mode():
|
||||
result = torch.zeros(req_shape, dtype=torch.float32, device=device)
|
||||
counter = torch.zeros(req_shape, dtype=torch.float32, device=device)
|
||||
batch_data, batch_locs = [], []
|
||||
i = 0
|
||||
|
||||
total_seg = sum(1 for _ in range(0, mix_t.shape[1], step))
|
||||
|
||||
with tqdm(total=total_seg, desc='Separating', unit='segment', ncols=100) as pbar:
|
||||
while i < mix_t.shape[1]:
|
||||
part = mix_t[:, i:i + C].to(device)
|
||||
l = part.shape[-1]
|
||||
if l < C:
|
||||
part = nn.functional.pad(
|
||||
part,
|
||||
(0, C - l) if l > C // 2 + 1 else (0, C - l, 0, 0),
|
||||
mode='reflect' if l > C // 2 + 1 else 'constant',
|
||||
)
|
||||
batch_data.append(part)
|
||||
batch_locs.append((i, l))
|
||||
i += step
|
||||
|
||||
if len(batch_data) >= batch_size or i >= mix_t.shape[1]:
|
||||
arr = torch.stack(batch_data, dim=0)
|
||||
x = model(arr)
|
||||
for j, (start, ln) in enumerate(batch_locs):
|
||||
window = (window_start if start == 0
|
||||
else window_finish if i >= mix_t.shape[1]
|
||||
else window_middle)
|
||||
result[..., start:start + ln] += x[j][..., :ln] * window[..., :ln]
|
||||
counter[..., start:start + ln] += window[..., :ln]
|
||||
pbar.update(len(batch_data))
|
||||
batch_data, batch_locs = [], []
|
||||
|
||||
estimated = result / counter.clamp(min=1e-10)
|
||||
if length_init > 2 * (C - step) and (C - step > 0):
|
||||
estimated = estimated[..., (C - step):-(C - step)]
|
||||
|
||||
sources = {}
|
||||
for idx, name in enumerate(self.stems):
|
||||
sources[name] = estimated[idx].cpu().numpy()
|
||||
return sources
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str,
|
||||
batch_size_override: Optional[int] = None,
|
||||
num_overlap_override: Optional[int] = None) -> Dict[str, str]:
|
||||
"""Separate audio and write stems to output_dir."""
|
||||
output_dir = Path(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
audio = _load_audio(input_path)
|
||||
model = self.get_model()
|
||||
sources = self._demix(audio, model,
|
||||
batch_size_override=batch_size_override,
|
||||
num_overlap_override=num_overlap_override)
|
||||
|
||||
del model
|
||||
torch.cuda.empty_cache()
|
||||
gc.collect()
|
||||
|
||||
ext = self.output_format
|
||||
result = {}
|
||||
for name, data in sources.items():
|
||||
dest = str(output_dir / f'{name}.{ext}')
|
||||
_save_audio(dest, data.T, ext, self.mp3_bitrate)
|
||||
result[name] = dest
|
||||
return result
|
||||
|
||||
|
||||
class MelBandVocalsSeparator(MelBandSeparatorBase):
|
||||
"""MelBand RoFormer optimized for vocals/instrumental separation."""
|
||||
|
||||
HF_REPO_ID = 'becruily/mel-band-roformer-vocals'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
):
|
||||
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/vocals')
|
||||
model_path = model_dir / 'model.ckpt'
|
||||
config_path = model_dir / 'config.yaml'
|
||||
|
||||
# Ensure model is downloaded
|
||||
self._ensure_model(model_dir, model_path, config_path)
|
||||
|
||||
super().__init__(
|
||||
model_path=model_path,
|
||||
config_path=config_path,
|
||||
cpu_separation=cpu_separation,
|
||||
output_format=output_format,
|
||||
mp3_bitrate=mp3_bitrate,
|
||||
stems=['vocals', 'instrumental'],
|
||||
)
|
||||
|
||||
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
from huggingface_hub import snapshot_download
|
||||
snapshot_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
revision=self.HF_REVISION,
|
||||
local_dir=str(model_dir),
|
||||
)
|
||||
|
||||
|
||||
class MelBandDereverbSeparator(MelBandSeparatorBase):
|
||||
"""MelBand RoFormer for dereverb - separates dry vocals from reverb."""
|
||||
|
||||
HF_REPO_ID = 'anvuew/dereverb_mel_band_roformer'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
use_less_aggressive: bool = False,
|
||||
):
|
||||
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/dereverb')
|
||||
|
||||
# The dereverb repo has multiple checkpoints
|
||||
if use_less_aggressive:
|
||||
model_path = model_dir / 'dereverb_mel_band_roformer_less_aggressive_anvuew_sdr_18.8050.ckpt'
|
||||
else:
|
||||
model_path = model_dir / 'dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt'
|
||||
config_path = model_dir / 'config.yaml'
|
||||
|
||||
self._ensure_model(model_dir, model_path, config_path)
|
||||
|
||||
super().__init__(
|
||||
model_path=model_path,
|
||||
config_path=config_path,
|
||||
cpu_separation=cpu_separation,
|
||||
output_format=output_format,
|
||||
mp3_bitrate=mp3_bitrate,
|
||||
stems=['dry_vocals', 'reverb'],
|
||||
)
|
||||
|
||||
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
from huggingface_hub import snapshot_download
|
||||
snapshot_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
revision=self.HF_REVISION,
|
||||
local_dir=str(model_dir),
|
||||
)
|
||||
|
||||
|
||||
class MelBandKaraokeSeparator(MelBandSeparatorBase):
|
||||
"""MelBand RoFormer for karaoke - separates lead vocals from backing+instrumental."""
|
||||
|
||||
HF_REPO_ID = 'becruily/mel-band-roformer-karaoke'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
):
|
||||
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/karaoke')
|
||||
model_path = model_dir / 'model.ckpt'
|
||||
config_path = model_dir / 'config.yaml'
|
||||
|
||||
self._ensure_model(model_dir, model_path, config_path)
|
||||
|
||||
super().__init__(
|
||||
model_path=model_path,
|
||||
config_path=config_path,
|
||||
cpu_separation=cpu_separation,
|
||||
output_format=output_format,
|
||||
mp3_bitrate=mp3_bitrate,
|
||||
stems=['lead', 'backing_instrumental'],
|
||||
)
|
||||
|
||||
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
from huggingface_hub import snapshot_download
|
||||
snapshot_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
revision=self.HF_REVISION,
|
||||
local_dir=str(model_dir),
|
||||
)
|
||||
|
||||
|
||||
class MelBandDereverbRoomSeparator(MelBandSeparatorBase):
|
||||
"""MelBand RoFormer for room dereverb from MVSEP-Central-Backup."""
|
||||
|
||||
HF_REPO_ID = 'miercolesv/MVSEP-Central-Backup'
|
||||
HF_SUBFOLDER = 'Dereverb-Room-Anvuew'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
):
|
||||
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/dereverb_room')
|
||||
model_path = model_dir / 'dereverb_room_anvuew_sdr_13.7432.ckpt'
|
||||
config_path = model_dir / 'dereverb_room_anvuew.yaml'
|
||||
|
||||
self._ensure_model(model_dir, model_path, config_path)
|
||||
|
||||
super().__init__(
|
||||
model_path=model_path,
|
||||
config_path=config_path,
|
||||
cpu_separation=cpu_separation,
|
||||
output_format=output_format,
|
||||
mp3_bitrate=mp3_bitrate,
|
||||
stems=['dry_vocals', 'room_reverb'],
|
||||
)
|
||||
|
||||
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
if not model_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/dereverb_room_anvuew_sdr_13.7432.ckpt",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
if not config_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/dereverb_room_anvuew.yaml",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to download Room Dereverb model: {e}")
|
||||
|
||||
|
||||
class MelBandDereverbAnvuewSeparator(MelBandSeparatorBase):
|
||||
"""MelBand RoFormer Anvuew dereverb from MVSEP-Central-Backup."""
|
||||
|
||||
HF_REPO_ID = 'miercolesv/MVSEP-Central-Backup'
|
||||
HF_SUBFOLDER = 'Dereverb-MelBand-Anvuew'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
use_less_aggressive: bool = False,
|
||||
):
|
||||
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/dereverb_anvuew')
|
||||
|
||||
if use_less_aggressive:
|
||||
model_path = model_dir / 'dereverb_mel_band_roformer_less_aggressive_anvuew_sdr_18.8050.ckpt'
|
||||
else:
|
||||
model_path = model_dir / 'dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt'
|
||||
|
||||
config_path = model_dir / 'dereverb_mel_band_roformer_anvuew.yaml'
|
||||
|
||||
self._ensure_model(model_dir, model_path, config_path)
|
||||
|
||||
super().__init__(
|
||||
model_path=model_path,
|
||||
config_path=config_path,
|
||||
cpu_separation=cpu_separation,
|
||||
output_format=output_format,
|
||||
mp3_bitrate=mp3_bitrate,
|
||||
stems=['dry_vocals', 'reverb'],
|
||||
)
|
||||
|
||||
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
# Download the specific model variant
|
||||
model_filename = model_path.name
|
||||
if not model_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/{model_filename}",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
# Download config
|
||||
if not config_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/dereverb_mel_band_roformer_anvuew.yaml",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to download Anvuew Dereverb model: {e}")
|
||||
|
||||
|
||||
class MelBandVocalsAnvuewSeparator(MelBandSeparatorBase):
|
||||
"""MelBand RoFormer Anvuew vocals from MVSEP-Central-Backup."""
|
||||
|
||||
HF_REPO_ID = 'miercolesv/MVSEP-Central-Backup'
|
||||
HF_SUBFOLDER = 'BS-RoFormer-Anvuew'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
):
|
||||
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/vocals_anvuew')
|
||||
model_path = model_dir / 'bs_roformer_anvuew_sdr_12.45.ckpt'
|
||||
config_path = model_dir / 'config.yaml'
|
||||
|
||||
self._ensure_model(model_dir, model_path, config_path)
|
||||
|
||||
super().__init__(
|
||||
model_path=model_path,
|
||||
config_path=config_path,
|
||||
cpu_separation=cpu_separation,
|
||||
output_format=output_format,
|
||||
mp3_bitrate=mp3_bitrate,
|
||||
stems=['vocals', 'instrumental'],
|
||||
)
|
||||
|
||||
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
if not model_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/bs_roformer_anvuew_sdr_12.45.ckpt",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
if not config_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/config.yaml",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to download Anvuew Vocals model: {e}")
|
||||
|
||||
|
||||
class MelBandKaraokeGaboxSeparator(MelBandSeparatorBase):
|
||||
"""MelBand RoFormer Karaoke Gabox from MVSEP-Central-Backup."""
|
||||
|
||||
HF_REPO_ID = 'miercolesv/MVSEP-Central-Backup'
|
||||
HF_SUBFOLDER = 'MelBandRoformers/bsroformers'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
):
|
||||
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/karaoke_gabox')
|
||||
model_path = model_dir / 'bs_karaoke_gabox_IS.ckpt'
|
||||
config_path = model_dir / 'karaoke_bs_roformer.yaml'
|
||||
|
||||
self._ensure_model(model_dir, model_path, config_path)
|
||||
|
||||
super().__init__(
|
||||
model_path=model_path,
|
||||
config_path=config_path,
|
||||
cpu_separation=cpu_separation,
|
||||
output_format=output_format,
|
||||
mp3_bitrate=mp3_bitrate,
|
||||
stems=['vocals', 'karaoke'],
|
||||
)
|
||||
|
||||
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
if not model_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/bs_karaoke_gabox_IS.ckpt",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
if not config_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/karaoke_bs_roformer.yaml",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to download Karaoke Gabox model: {e}")
|
||||
|
||||
|
||||
class MelBandDeuxBecruilySeparator(MelBandSeparatorBase):
|
||||
"""MelBand RoFormer Deux Becruily - specialized 2-stem vocals model."""
|
||||
|
||||
HF_REPO_ID = 'miercolesv/MVSEP-Central-Backup'
|
||||
HF_SUBFOLDER = 'MelBand-Roformer-Deux-Becruily'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
):
|
||||
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/deux_becruily')
|
||||
model_path = model_dir / 'becruily_deux.ckpt'
|
||||
config_path = model_dir / 'config_deux_becruily.yaml'
|
||||
|
||||
self._ensure_model(model_dir, model_path, config_path)
|
||||
|
||||
super().__init__(
|
||||
model_path=model_path,
|
||||
config_path=config_path,
|
||||
cpu_separation=cpu_separation,
|
||||
output_format=output_format,
|
||||
mp3_bitrate=mp3_bitrate,
|
||||
stems=['vocals', 'instrumental'],
|
||||
)
|
||||
|
||||
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
print(f"Downloading Deux Becruily model from {self.HF_REPO_ID}")
|
||||
|
||||
# Download model
|
||||
if not model_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/becruily_deux.ckpt",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
# Download config
|
||||
if not config_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/config_deux_becruily.yaml",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to download Deux Becruily model: {e}")
|
||||
Executable
+576
@@ -0,0 +1,576 @@
|
||||
"""
|
||||
Audio metadata (ID3 / Vorbis / EXIF) utilities for reading and writing tags.
|
||||
Uses mutagen for format-agnostic read/write.
|
||||
"""
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Dict, Tuple
|
||||
import re
|
||||
|
||||
|
||||
def _read_cover_art(source_path: str):
|
||||
"""
|
||||
Read cover art from an audio file.
|
||||
Returns (data_bytes, mime_type) or (None, None) if not found.
|
||||
"""
|
||||
try:
|
||||
ext = Path(source_path).suffix.lower()
|
||||
if ext == '.mp3':
|
||||
from mutagen.id3 import ID3, ID3NoHeaderError
|
||||
try:
|
||||
tags = ID3(source_path)
|
||||
apic_keys = [k for k in tags.keys() if k.startswith('APIC')]
|
||||
if apic_keys:
|
||||
apic = tags[apic_keys[0]]
|
||||
return apic.data, apic.mime
|
||||
except ID3NoHeaderError:
|
||||
pass
|
||||
elif ext == '.flac':
|
||||
from mutagen.flac import FLAC
|
||||
audio = FLAC(source_path)
|
||||
if audio.pictures:
|
||||
pic = audio.pictures[0]
|
||||
return pic.data, pic.mime
|
||||
elif ext == '.wav':
|
||||
from mutagen.wave import WAVE
|
||||
try:
|
||||
audio = WAVE(source_path)
|
||||
if audio.tags:
|
||||
apic_keys = [k for k in audio.tags.keys() if k.startswith('APIC')]
|
||||
if apic_keys:
|
||||
apic = audio.tags[apic_keys[0]]
|
||||
return apic.data, apic.mime
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
return None, None
|
||||
|
||||
|
||||
def copy_metadata(source_path: str, dest_path: str, stem_label: Optional[str] = None,
|
||||
copy_lyrics: bool = True) -> None:
|
||||
"""
|
||||
Copy all common tags (title, artist, album, track, date, genre, cover art, lyrics)
|
||||
from source_path to dest_path. Optionally appends stem_label to the title.
|
||||
Silently ignores unsupported formats — metadata is cosmetic, never fatal.
|
||||
"""
|
||||
try:
|
||||
import mutagen
|
||||
|
||||
src = mutagen.File(source_path, easy=True)
|
||||
if src is None:
|
||||
return
|
||||
|
||||
title = src.get("title", [""])[0] if src.get("title") else ""
|
||||
artist = src.get("artist", [""])[0] if src.get("artist") else ""
|
||||
albumartist = src.get("albumartist", [""])[0] if src.get("albumartist") else ""
|
||||
album = src.get("album", [""])[0] if src.get("album") else ""
|
||||
tracknumber = src.get("tracknumber", [""])[0] if src.get("tracknumber") else ""
|
||||
discnumber = src.get("discnumber", [""])[0] if src.get("discnumber") else ""
|
||||
date = src.get("date", [""])[0] if src.get("date") else ""
|
||||
genre = src.get("genre", [""])[0] if src.get("genre") else ""
|
||||
composer = src.get("composer", [""])[0] if src.get("composer") else ""
|
||||
|
||||
if stem_label and title:
|
||||
title = f"{title} ({stem_label})"
|
||||
elif stem_label:
|
||||
title = stem_label
|
||||
|
||||
cover_data, cover_mime = _read_cover_art(source_path)
|
||||
|
||||
# Read lyrics from source
|
||||
lyrics = ""
|
||||
if copy_lyrics:
|
||||
lyrics = _read_lyrics_from_file(source_path)
|
||||
|
||||
ext = Path(dest_path).suffix.lower()
|
||||
if ext == ".mp3":
|
||||
_write_mp3_tags(dest_path, title, artist, albumartist, album, tracknumber,
|
||||
discnumber, date, genre, composer, cover_data, cover_mime, lyrics)
|
||||
elif ext == ".flac":
|
||||
_write_flac_tags(dest_path, title, artist, albumartist, album, tracknumber,
|
||||
discnumber, date, genre, composer, cover_data, cover_mime, lyrics)
|
||||
elif ext == ".wav":
|
||||
_write_wav_tags(dest_path, title, artist, albumartist, album, tracknumber,
|
||||
discnumber, date, genre, composer, cover_data, cover_mime)
|
||||
|
||||
except Exception:
|
||||
pass # Never fail over metadata — it's cosmetic
|
||||
|
||||
|
||||
def _read_lyrics_from_file(path: str) -> str:
|
||||
"""Read lyrics from a file using the same logic as read_metadata."""
|
||||
try:
|
||||
ext = Path(path).suffix.lower()
|
||||
if ext == '.mp3':
|
||||
return _read_mp3_lyrics(path)
|
||||
elif ext in ('.flac', '.ogg', '.opus'):
|
||||
return _read_vorbis_lyrics(path)
|
||||
except Exception:
|
||||
pass
|
||||
return ''
|
||||
|
||||
|
||||
def _write_mp3_tags(path, title, artist, albumartist, album, tracknumber,
|
||||
discnumber, date, genre, composer,
|
||||
cover_data=None, cover_mime=None, lyrics=None):
|
||||
from mutagen.id3 import (
|
||||
ID3, ID3NoHeaderError, TIT2, TPE1, TPE2, TALB, TRCK, TPOS,
|
||||
TDRC, TCON, TCOM, APIC, USLT,
|
||||
)
|
||||
try:
|
||||
tags = ID3(path)
|
||||
except ID3NoHeaderError:
|
||||
tags = ID3()
|
||||
if title: tags["TIT2"] = TIT2(encoding=3, text=title)
|
||||
if artist: tags["TPE1"] = TPE1(encoding=3, text=artist)
|
||||
if albumartist: tags["TPE2"] = TPE2(encoding=3, text=albumartist)
|
||||
if album: tags["TALB"] = TALB(encoding=3, text=album)
|
||||
if tracknumber: tags["TRCK"] = TRCK(encoding=3, text=tracknumber)
|
||||
if discnumber: tags["TPOS"] = TPOS(encoding=3, text=discnumber)
|
||||
if date: tags["TDRC"] = TDRC(encoding=3, text=date)
|
||||
if genre: tags["TCON"] = TCON(encoding=3, text=genre)
|
||||
if composer: tags["TCOM"] = TCOM(encoding=3, text=composer)
|
||||
if lyrics: tags["USLT::'eng'"] = USLT(encoding=3, lang='eng', desc='', text=lyrics)
|
||||
if cover_data:
|
||||
tags["APIC:"] = APIC(
|
||||
encoding=3, mime=cover_mime or "image/jpeg",
|
||||
type=3, desc="Cover", data=cover_data,
|
||||
)
|
||||
tags.save(path)
|
||||
|
||||
|
||||
def _write_flac_tags(path, title, artist, albumartist, album, tracknumber,
|
||||
discnumber, date, genre, composer,
|
||||
cover_data=None, cover_mime=None, lyrics=None):
|
||||
from mutagen.flac import FLAC, Picture
|
||||
audio = FLAC(path)
|
||||
if title: audio["title"] = title
|
||||
if artist: audio["artist"] = artist
|
||||
if albumartist: audio["albumartist"] = albumartist
|
||||
if album: audio["album"] = album
|
||||
if tracknumber: audio["tracknumber"] = tracknumber
|
||||
if discnumber: audio["discnumber"] = discnumber
|
||||
if date: audio["date"] = date
|
||||
if genre: audio["genre"] = genre
|
||||
if composer: audio["composer"] = composer
|
||||
if lyrics: audio["lyrics"] = lyrics
|
||||
if cover_data:
|
||||
pic = Picture()
|
||||
pic.type = 3 # front cover
|
||||
pic.mime = cover_mime or "image/jpeg"
|
||||
pic.desc = "Cover"
|
||||
pic.data = cover_data
|
||||
audio.clear_pictures()
|
||||
audio.add_picture(pic)
|
||||
audio.save()
|
||||
|
||||
|
||||
def _write_wav_tags(path, title, artist, albumartist, album, tracknumber,
|
||||
discnumber, date, genre, composer,
|
||||
cover_data=None, cover_mime=None):
|
||||
from mutagen.wave import WAVE
|
||||
from mutagen.id3 import TIT2, TPE1, TPE2, TALB, TRCK, TPOS, TDRC, TCON, TCOM, APIC
|
||||
try:
|
||||
audio = WAVE(path)
|
||||
if audio.tags is None:
|
||||
audio.add_tags()
|
||||
if title: audio.tags["TIT2"] = TIT2(encoding=3, text=title)
|
||||
if artist: audio.tags["TPE1"] = TPE1(encoding=3, text=artist)
|
||||
if albumartist: audio.tags["TPE2"] = TPE2(encoding=3, text=albumartist)
|
||||
if album: audio.tags["TALB"] = TALB(encoding=3, text=album)
|
||||
if tracknumber: audio.tags["TRCK"] = TRCK(encoding=3, text=tracknumber)
|
||||
if discnumber: audio.tags["TPOS"] = TPOS(encoding=3, text=discnumber)
|
||||
if date: audio.tags["TDRC"] = TDRC(encoding=3, text=date)
|
||||
if genre: audio.tags["TCON"] = TCON(encoding=3, text=genre)
|
||||
if composer: audio.tags["TCOM"] = TCOM(encoding=3, text=composer)
|
||||
if cover_data:
|
||||
audio.tags["APIC:"] = APIC(
|
||||
encoding=3, mime=cover_mime or "image/jpeg",
|
||||
type=3, desc="Cover", data=cover_data,
|
||||
)
|
||||
audio.save()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Metadata reading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def read_metadata(source_path: str) -> dict:
|
||||
"""
|
||||
Read basic metadata from an audio file.
|
||||
Returns dict with 'artist', 'albumartist', 'title', 'album', 'lyrics' keys (all str).
|
||||
Lyrics are read in order of priority:
|
||||
1. Synced lyrics tags (SYLT for MP3, syncedlyrics for FLAC/OGG)
|
||||
2. Unsynced lyrics tags (USLT for MP3, lyrics/unsyncedlyrics for FLAC/OGG)
|
||||
3. External .lrc file with same base name
|
||||
Silently returns empty strings on failure or unsupported format.
|
||||
"""
|
||||
result = {'artist': '', 'albumartist': '', 'title': '', 'album': '', 'lyrics': ''}
|
||||
try:
|
||||
import mutagen
|
||||
src = mutagen.File(source_path, easy=True)
|
||||
if src is None:
|
||||
return result
|
||||
result['artist'] = (src.get('artist') or [''])[0]
|
||||
result['albumartist'] = (src.get('albumartist') or [''])[0]
|
||||
result['title'] = (src.get('title') or [''])[0]
|
||||
result['album'] = (src.get('album') or [''])[0]
|
||||
|
||||
ext = Path(source_path).suffix.lower()
|
||||
lyrics = ''
|
||||
|
||||
# Try to read lyrics from tags first
|
||||
if ext == '.mp3':
|
||||
lyrics = _read_mp3_lyrics(source_path)
|
||||
elif ext in ('.flac', '.ogg', '.opus'):
|
||||
lyrics = _read_vorbis_lyrics(source_path)
|
||||
|
||||
# If no lyrics in tags, try external LRC file
|
||||
if not lyrics:
|
||||
lyrics = read_lrc_file(source_path)
|
||||
|
||||
result['lyrics'] = lyrics
|
||||
except Exception:
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def _read_mp3_lyrics(path: str) -> str:
|
||||
"""Read lyrics from ID3 tags: SYLT (synced) → USLT (unsynced) → TXXX:LYRICS.
|
||||
Also checks $-prefixed variants used by some tagging software.
|
||||
"""
|
||||
try:
|
||||
from mutagen.id3 import ID3, ID3NoHeaderError
|
||||
try:
|
||||
tags = ID3(path)
|
||||
except ID3NoHeaderError:
|
||||
return ''
|
||||
|
||||
# Synced lyrics: SYLT contains list of (text, time_ms) tuples
|
||||
sylt_keys = [k for k in tags.keys() if k.startswith('SYLT')]
|
||||
if sylt_keys:
|
||||
return '\n'.join(t for t, _ in tags[sylt_keys[0]].text)
|
||||
|
||||
# Unsynced lyrics
|
||||
uslt_keys = [k for k in tags.keys() if k.startswith('USLT')]
|
||||
if uslt_keys:
|
||||
return tags[uslt_keys[0]].text
|
||||
|
||||
# TXXX:LYRICS fallback (also check $-prefixed variants)
|
||||
for txxx_key in ('TXXX:LYRICS', 'TXXX:$LYRICS',
|
||||
'TXXX:SYNCEDLYRICS', 'TXXX:$SYNCEDLYRICS',
|
||||
'TXXX:UNSYNCEDLYRICS', 'TXXX:$UNSYNCEDLYRICS'):
|
||||
if txxx_key in tags:
|
||||
text = tags[txxx_key].text
|
||||
if text:
|
||||
return text[0] if isinstance(text, list) else str(text)
|
||||
except Exception:
|
||||
pass
|
||||
return ''
|
||||
|
||||
|
||||
def _read_vorbis_lyrics(path: str) -> str:
|
||||
"""Read lyrics from FLAC/OGG Vorbis comment tags.
|
||||
|
||||
Checks SYNCEDLYRICS, LYRICS, and UNSYNCEDLYRICS (case-insensitive).
|
||||
Also checks variants with $ prefix (e.g., $LYRICS) used by some tagging software.
|
||||
Returns the first non-empty value found, or empty string if all are empty.
|
||||
"""
|
||||
try:
|
||||
import mutagen
|
||||
f = mutagen.File(path)
|
||||
if f is None:
|
||||
return ''
|
||||
|
||||
# Build case-insensitive lookup: upper() -> original key
|
||||
key_map = {k.upper(): k for k in f.keys()}
|
||||
|
||||
# Check in priority order: SYNCEDLYRICS -> LYRICS -> UNSYNCEDLYRICS
|
||||
# Also check $-prefixed variants as fallback
|
||||
for key in ('SYNCEDLYRICS', '$SYNCEDLYRICS',
|
||||
'LYRICS', '$LYRICS',
|
||||
'UNSYNCEDLYRICS', '$UNSYNCEDLYRICS'):
|
||||
if key in key_map:
|
||||
val = f.get(key_map[key])
|
||||
if val:
|
||||
return val[0]
|
||||
except Exception:
|
||||
pass
|
||||
return ''
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LRC file utilities
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
LRC_LINE_REGEX = re.compile(
|
||||
r'^(\[\s*(\d+):(\d{1,2})(?:\.(\d{1,3}))?\s*\])' # timestamp [mm:ss.xx]
|
||||
r'\s*(.*?)$', # lyrics text
|
||||
re.MULTILINE
|
||||
)
|
||||
|
||||
ELRC_WORD_REGEX = re.compile(
|
||||
r'(<\s*(\d+):(\d{1,2})(?:\.(\d{1,3}))?\s*>)' # word timestamp <mm:ss.xx>
|
||||
r'\s*([^<\n]+)', # word text
|
||||
re.MULTILINE
|
||||
)
|
||||
|
||||
|
||||
def read_lrc_file(audio_path: str) -> str:
|
||||
"""
|
||||
Read lyrics from an external .lrc file with the same base name as the audio file.
|
||||
Returns the raw LRC content or empty string if not found.
|
||||
"""
|
||||
audio_path = Path(audio_path)
|
||||
lrc_path = audio_path.with_suffix('.lrc')
|
||||
|
||||
if not lrc_path.exists():
|
||||
# Try case-insensitive search (covers both .lrc and .LRC on Linux)
|
||||
for pattern in ('*.lrc', '*.LRC'):
|
||||
for f in audio_path.parent.glob(pattern):
|
||||
if f.stem.lower() == audio_path.stem.lower():
|
||||
lrc_path = f
|
||||
break
|
||||
if lrc_path.exists():
|
||||
break
|
||||
|
||||
if lrc_path.exists():
|
||||
try:
|
||||
return lrc_path.read_text(encoding='utf-8', errors='ignore')
|
||||
except Exception:
|
||||
pass
|
||||
return ''
|
||||
|
||||
|
||||
def parse_lrc(lrc_content: str) -> List[Tuple[float, str]]:
|
||||
"""
|
||||
Parse LRC content into list of (timestamp_seconds, line_text) tuples.
|
||||
|
||||
Args:
|
||||
lrc_content: Raw LRC file content
|
||||
|
||||
Returns:
|
||||
List of (timestamp, line) tuples sorted by timestamp
|
||||
"""
|
||||
lines = []
|
||||
|
||||
for match in LRC_LINE_REGEX.finditer(lrc_content):
|
||||
minutes = int(match.group(2))
|
||||
seconds = int(match.group(3))
|
||||
millis = match.group(4)
|
||||
text = match.group(5).strip()
|
||||
|
||||
if millis:
|
||||
# Handle different millisecond precisions
|
||||
millis = int(millis.ljust(3, '0')[:3])
|
||||
else:
|
||||
millis = 0
|
||||
|
||||
timestamp = minutes * 60 + seconds + millis / 1000.0
|
||||
|
||||
if text:
|
||||
lines.append((timestamp, text))
|
||||
|
||||
# Sort by timestamp and remove duplicates
|
||||
lines.sort(key=lambda x: x[0])
|
||||
|
||||
# Remove duplicate timestamps (keep first occurrence)
|
||||
seen = set()
|
||||
unique_lines = []
|
||||
for ts, text in lines:
|
||||
if ts not in seen:
|
||||
seen.add(ts)
|
||||
unique_lines.append((ts, text))
|
||||
|
||||
return unique_lines
|
||||
|
||||
|
||||
def parse_elrc(elrc_content: str) -> List[Tuple[float, str, List[Tuple[float, str]]]]:
|
||||
"""
|
||||
Parse Enhanced LRC (ELRC) content with word-level timestamps.
|
||||
|
||||
Args:
|
||||
elrc_content: Raw ELRC file content
|
||||
|
||||
Returns:
|
||||
List of (line_timestamp, line_text, [(word_timestamp, word), ...]) tuples
|
||||
"""
|
||||
lines = []
|
||||
|
||||
for match in LRC_LINE_REGEX.finditer(elrc_content):
|
||||
minutes = int(match.group(2))
|
||||
seconds = int(match.group(3))
|
||||
millis = match.group(4)
|
||||
full_line = match.group(5)
|
||||
|
||||
if millis:
|
||||
millis = int(millis.ljust(3, '0')[:3])
|
||||
else:
|
||||
millis = 0
|
||||
|
||||
line_timestamp = minutes * 60 + seconds + millis / 1000.0
|
||||
|
||||
# Parse word-level timestamps within this line
|
||||
words = []
|
||||
for word_match in ELRC_WORD_REGEX.finditer(full_line):
|
||||
w_minutes = int(word_match.group(2))
|
||||
w_seconds = int(word_match.group(3))
|
||||
w_millis = word_match.group(4)
|
||||
w_text = word_match.group(5).strip()
|
||||
|
||||
if w_millis:
|
||||
w_millis = int(w_millis.ljust(3, '0')[:3])
|
||||
else:
|
||||
w_millis = 0
|
||||
|
||||
word_timestamp = w_minutes * 60 + w_seconds + w_millis / 1000.0
|
||||
|
||||
if w_text:
|
||||
words.append((word_timestamp, w_text))
|
||||
|
||||
# Clean line text (remove word timestamps)
|
||||
line_text = ELRC_WORD_REGEX.sub(r'\5', full_line).strip()
|
||||
|
||||
if line_text or words:
|
||||
lines.append((line_timestamp, line_text, words))
|
||||
|
||||
lines.sort(key=lambda x: x[0])
|
||||
return lines
|
||||
|
||||
|
||||
def format_lrc(lines: List[Tuple[float, str]], include_empty: bool = False) -> str:
|
||||
"""
|
||||
Format lines back into LRC format.
|
||||
|
||||
Args:
|
||||
lines: List of (timestamp_seconds, line_text) tuples
|
||||
include_empty: Whether to include empty lines
|
||||
|
||||
Returns:
|
||||
LRC formatted string
|
||||
"""
|
||||
result = []
|
||||
for timestamp, text in lines:
|
||||
if not text and not include_empty:
|
||||
continue
|
||||
minutes = int(timestamp // 60)
|
||||
seconds = int(timestamp % 60)
|
||||
millis = int((timestamp % 1) * 100)
|
||||
result.append(f"[{minutes:02d}:{seconds:02d}.{millis:02d}]{text}")
|
||||
return '\n'.join(result)
|
||||
|
||||
|
||||
def format_elrc(lines: List[Tuple[float, str, List[Tuple[float, str]]]]) -> str:
|
||||
"""
|
||||
Format lines with word-level timestamps into ELRC format.
|
||||
|
||||
Args:
|
||||
lines: List of (line_timestamp, line_text, [(word_timestamp, word), ...]) tuples
|
||||
|
||||
Returns:
|
||||
ELRC formatted string
|
||||
"""
|
||||
result = []
|
||||
for line_ts, line_text, words in lines:
|
||||
minutes = int(line_ts // 60)
|
||||
seconds = int(line_ts % 60)
|
||||
millis = int((line_ts % 1) * 100)
|
||||
|
||||
if words:
|
||||
# Build line with word timestamps
|
||||
word_parts = []
|
||||
for word_ts, word in words:
|
||||
w_min = int(word_ts // 60)
|
||||
w_sec = int(word_ts % 60)
|
||||
w_mil = int((word_ts % 1) * 100)
|
||||
word_parts.append(f"<{w_min:02d}:{w_sec:02d}.{w_mil:02d}>{word}")
|
||||
line_content = ' '.join(word_parts)
|
||||
else:
|
||||
line_content = line_text
|
||||
|
||||
result.append(f"[{minutes:02d}:{seconds:02d}.{millis:02d}]{line_content}")
|
||||
return '\n'.join(result)
|
||||
|
||||
|
||||
def lyrics_to_plaintext(lyrics: str) -> str:
|
||||
"""
|
||||
Convert LRC/ELRC content to plain text (remove all timestamps).
|
||||
|
||||
Args:
|
||||
lyrics: LRC formatted content
|
||||
|
||||
Returns:
|
||||
Plain text lyrics
|
||||
"""
|
||||
# Remove line timestamps [mm:ss.xx]
|
||||
text = re.sub(r'\[\s*\d+:\d{2}(?:\.\d{1,3})?\s*\]', '', lyrics)
|
||||
# Remove word timestamps <mm:ss.xx>
|
||||
text = re.sub(r'<\s*\d+:\d{2}(?:\.\d{1,3})?\s*>', '', text)
|
||||
# Clean up whitespace
|
||||
lines = [line.strip() for line in text.split('\n') if line.strip()]
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
def is_lrc_synced(lyrics: str) -> bool:
|
||||
"""
|
||||
Check if lyrics content contains LRC timestamps (synced lyrics).
|
||||
|
||||
Args:
|
||||
lyrics: Lyrics content to check
|
||||
|
||||
Returns:
|
||||
True if content contains LRC timestamps
|
||||
"""
|
||||
return bool(LRC_LINE_REGEX.search(lyrics))
|
||||
|
||||
|
||||
def extract_lyrics_from_file(file_path: str) -> Dict[str, str]:
|
||||
"""
|
||||
Extract lyrics from an audio file or LRC file.
|
||||
|
||||
Args:
|
||||
file_path: Path to audio file or LRC file
|
||||
|
||||
Returns:
|
||||
Dict with 'source', 'raw', 'synced', 'plain' keys
|
||||
- source: 'tag', 'lrc_file', or 'none'
|
||||
- raw: Original lyrics content with timestamps if present
|
||||
- synced: True if lyrics are timestamped
|
||||
- plain: Plain text version without timestamps
|
||||
"""
|
||||
path = Path(file_path)
|
||||
result = {'source': 'none', 'raw': '', 'synced': False, 'plain': ''}
|
||||
|
||||
# If it's an LRC file, read it directly
|
||||
if path.suffix.lower() == '.lrc':
|
||||
try:
|
||||
content = path.read_text(encoding='utf-8', errors='ignore')
|
||||
result['source'] = 'lrc_file'
|
||||
result['raw'] = content
|
||||
result['synced'] = is_lrc_synced(content)
|
||||
result['plain'] = lyrics_to_plaintext(content)
|
||||
except Exception:
|
||||
pass
|
||||
return result
|
||||
|
||||
# Otherwise, try to read from audio file metadata
|
||||
metadata = read_metadata(file_path)
|
||||
lyrics = metadata.get('lyrics', '')
|
||||
|
||||
if lyrics:
|
||||
result['source'] = 'tag'
|
||||
result['raw'] = lyrics
|
||||
result['synced'] = is_lrc_synced(lyrics)
|
||||
result['plain'] = lyrics_to_plaintext(lyrics)
|
||||
else:
|
||||
# Try external LRC file as last resort
|
||||
lrc_content = read_lrc_file(file_path)
|
||||
if lrc_content:
|
||||
result['source'] = 'lrc_file'
|
||||
result['raw'] = lrc_content
|
||||
result['synced'] = is_lrc_synced(lrc_content)
|
||||
result['plain'] = lyrics_to_plaintext(lrc_content)
|
||||
|
||||
return result
|
||||
Executable
+56
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
Mix service: create static or dynamic mixes from separated stems.
|
||||
CPU-bound (pydub) — uses the general executor, not the GPU queue.
|
||||
"""
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from .. import job_manager as jm
|
||||
from .preset_service import make_static, make_dynamic
|
||||
|
||||
|
||||
def run_mix(
|
||||
job_id: str,
|
||||
stem_files: Dict[str, str], # stem_name (lowercase) → abs file path
|
||||
preset_type: str, # 'static' | 'dynamic'
|
||||
output_path: str,
|
||||
output_format: str,
|
||||
static_option: Optional[str] = None,
|
||||
dynamic_levels: Optional[Dict[str, float]] = None,
|
||||
):
|
||||
"""Background task: create a mix from separated stems."""
|
||||
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Creating mix...")
|
||||
try:
|
||||
out = Path(output_path)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if preset_type == "static":
|
||||
muted = _get_muted_stems(static_option)
|
||||
result_path = make_static(stem_files, muted, out, output_format)
|
||||
elif preset_type == "dynamic":
|
||||
volumes = dynamic_levels or {}
|
||||
result_path = make_dynamic(stem_files, volumes, out, output_format)
|
||||
else:
|
||||
raise ValueError(f"Unknown preset type: {preset_type}")
|
||||
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.DONE,
|
||||
progress="Mix complete.",
|
||||
files=[{"name": out.name, "path": result_path, "type": preset_type}],
|
||||
)
|
||||
except Exception as exc:
|
||||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
|
||||
|
||||
|
||||
def _get_muted_stems(static_option: Optional[str]) -> List[str]:
|
||||
if not static_option:
|
||||
return []
|
||||
option_map = {
|
||||
"no_vocals": ["vocals"],
|
||||
"no_drums": ["drums"],
|
||||
"no_bass": ["bass"],
|
||||
"no_vocals_no_bass": ["vocals", "bass"],
|
||||
"no_drums_no_bass": ["drums", "bass"],
|
||||
}
|
||||
return option_map.get(static_option, [])
|
||||
Executable
+310
@@ -0,0 +1,310 @@
|
||||
"""
|
||||
Full pipeline: separate → transcribe → abc-convert.
|
||||
Supports both Demucs (CLI) and BS-RoFormer (Python API) for the separation stage.
|
||||
Uses project-based folder structure for organized output.
|
||||
"""
|
||||
import sys
|
||||
import subprocess
|
||||
import time
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import PROJECTS_DIR, JOBS_DIR, CPU_SEPARATION, TRANSCRIBE_PRESETS
|
||||
from .metadata_util import copy_metadata, read_metadata
|
||||
from .project_namer import project_name, make_project_dirs
|
||||
from .separator_service import BSR_PREFIX, BSR_STEM_MODE
|
||||
from . import abc_generator
|
||||
|
||||
|
||||
def run_pipeline(
|
||||
job_id: str,
|
||||
input_path: str,
|
||||
sep_model: str,
|
||||
sep_format: str,
|
||||
transcribe_mode: str,
|
||||
song_title: str,
|
||||
project_name_override: str = None,
|
||||
):
|
||||
"""
|
||||
Full pipeline:
|
||||
1. Stem separation (Demucs CLI or BS-RoFormer Python API)
|
||||
2. MIDI transcription (basic-pitch) for each non-drum stem
|
||||
3. ABC notation conversion
|
||||
|
||||
Creates organized project folder structure under MEDIA_DIR.
|
||||
"""
|
||||
# Read metadata for project naming
|
||||
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Reading metadata...")
|
||||
metadata = read_metadata(input_path)
|
||||
|
||||
# Determine project name
|
||||
if project_name_override:
|
||||
proj_name = project_name_override
|
||||
else:
|
||||
filename_stem = Path(input_path).stem
|
||||
proj_name = project_name(metadata.get('artist', ''), metadata.get('title', ''), filename_stem)
|
||||
|
||||
# Create project directories
|
||||
jm.update_job(job_id, progress=f"Creating project: {proj_name}...")
|
||||
dirs = make_project_dirs(
|
||||
proj_name, PROJECTS_DIR,
|
||||
{'instrumental': False, 'static': False, 'dynamic': False}
|
||||
)
|
||||
|
||||
project_dir = dirs['project']
|
||||
stems_dir = dirs['stems']
|
||||
transcriptions_dir = project_dir / "Transcriptions"
|
||||
transcriptions_dir.mkdir(exist_ok=True)
|
||||
|
||||
all_files = []
|
||||
|
||||
# Normalise mp3_320 → mp3 with explicit bitrate
|
||||
mp3_bitrate = '320k' if sep_format == 'mp3_320' else '256k'
|
||||
normalised_format = 'mp3' if sep_format == 'mp3_320' else sep_format
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Stage 1: Separation
|
||||
# ------------------------------------------------------------------
|
||||
jm.update_job(job_id, progress="[1/3] Separating stems...")
|
||||
|
||||
try:
|
||||
if sep_model.startswith(BSR_PREFIX):
|
||||
stem_files = _separate_bsroformer(
|
||||
job_id, input_path, sep_model, normalised_format, stems_dir, mp3_bitrate
|
||||
)
|
||||
else:
|
||||
stem_files = _separate_demucs(
|
||||
job_id, input_path, sep_model, sep_format, stems_dir
|
||||
)
|
||||
except Exception as exc:
|
||||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Separation failed.")
|
||||
return
|
||||
|
||||
if stem_files is None:
|
||||
return # cancelled inside sub-function
|
||||
|
||||
# Record stems with proper metadata
|
||||
for stem_name, path in sorted(stem_files.items()):
|
||||
copy_metadata(input_path, path, stem_label=stem_name.capitalize())
|
||||
all_files.append({
|
||||
"name": Path(path).name,
|
||||
"path": path,
|
||||
"stage": "separation",
|
||||
"stem": stem_name
|
||||
})
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Stage 2: Transcription (skip drum stems)
|
||||
# ------------------------------------------------------------------
|
||||
from basic_pitch.inference import predict
|
||||
from basic_pitch import ICASSP_2022_MODEL_PATH
|
||||
|
||||
jm.update_job(job_id, progress="[2/3] Transcribing stems to MIDI...")
|
||||
|
||||
midi_files = []
|
||||
transcribe_paths = [
|
||||
(name, path) for name, path in stem_files.items()
|
||||
if 'drums' not in name.lower()
|
||||
]
|
||||
params = TRANSCRIBE_PRESETS.get(transcribe_mode, TRANSCRIBE_PRESETS["polyphonic"])
|
||||
|
||||
for stem_name, stem_path in transcribe_paths:
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
jm.update_job(job_id, progress=f"[2/3] Transcribing: {stem_name}...")
|
||||
|
||||
try:
|
||||
_, midi_data, _ = predict(stem_path, ICASSP_2022_MODEL_PATH, **params)
|
||||
|
||||
output_name = f"{proj_name} - {stem_name.capitalize()} ({transcribe_mode}).mid"
|
||||
dest = transcriptions_dir / output_name
|
||||
midi_data.write(str(dest))
|
||||
all_files.append({
|
||||
"name": dest.name,
|
||||
"path": str(dest),
|
||||
"stage": "transcription"
|
||||
})
|
||||
midi_files.append(str(dest))
|
||||
|
||||
except Exception as exc:
|
||||
jm.update_job(job_id, progress=f"[2/3] Error on {stem_name}: {exc}")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Stage 3: ABC conversion
|
||||
# ------------------------------------------------------------------
|
||||
if midi_files:
|
||||
jm.update_job(job_id, progress="[3/3] Converting MIDI to ABC notation...")
|
||||
try:
|
||||
abc_results = abc_generator.convert_midi_to_abc(midi_files, song_title or proj_name)
|
||||
if abc_results:
|
||||
for part_name, abc_text in abc_results.items():
|
||||
safe = part_name.replace("/", "_").replace("\\", "_")
|
||||
dest = transcriptions_dir / f"{safe}.abc"
|
||||
dest.write_text(abc_text, encoding="utf-8")
|
||||
all_files.append({
|
||||
"name": dest.name,
|
||||
"path": str(dest),
|
||||
"stage": "abc"
|
||||
})
|
||||
except Exception as exc:
|
||||
jm.update_job(job_id, progress=f"[3/3] ABC conversion error: {exc}")
|
||||
|
||||
jm.update_job(
|
||||
job_id, status=jm.JobStatus.DONE,
|
||||
progress="Pipeline complete.",
|
||||
files=all_files,
|
||||
result={
|
||||
"project_name": proj_name,
|
||||
"project_path": str(project_dir),
|
||||
"stems_dir": str(stems_dir),
|
||||
"transcriptions_dir": str(transcriptions_dir),
|
||||
}
|
||||
)
|
||||
jm.find_or_create_project(proj_name, job_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Separation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _separate_demucs(
|
||||
job_id: str, input_path: str, model: str, output_format: str, stems_dir: Path
|
||||
) -> dict | None:
|
||||
"""
|
||||
Run Demucs CLI; returns dict mapping stem names to file paths, or None if cancelled.
|
||||
"""
|
||||
ext = output_format.lower()
|
||||
fmt_flag = []
|
||||
if ext == "mp3_320":
|
||||
fmt_flag = ["--mp3", "--mp3-bitrate", "320"]
|
||||
ext = "mp3"
|
||||
elif ext == "mp3":
|
||||
fmt_flag = ["--mp3", "--mp3-bitrate", "256"]
|
||||
elif ext == "flac":
|
||||
fmt_flag = ["--flac"]
|
||||
|
||||
# Use temp directory for demucs output
|
||||
temp_out = JOBS_DIR / job_id / "demucs_out"
|
||||
temp_out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
cmd = [
|
||||
sys.executable, "-m", "demucs",
|
||||
"--name", model,
|
||||
"--out", str(temp_out),
|
||||
*fmt_flag,
|
||||
input_path,
|
||||
]
|
||||
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
|
||||
)
|
||||
while True:
|
||||
if jm.should_cancel(job_id):
|
||||
process.terminate()
|
||||
process.wait()
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return None
|
||||
line = process.stdout.readline()
|
||||
if line:
|
||||
jm.update_job(job_id, progress=f"[1/3] {line.rstrip()}")
|
||||
elif process.poll() is not None:
|
||||
break
|
||||
else:
|
||||
time.sleep(0.1)
|
||||
|
||||
if process.returncode != 0:
|
||||
raise RuntimeError(f"Demucs failed with code {process.returncode}")
|
||||
|
||||
input_stem = Path(input_path).stem
|
||||
stem_src_dir = temp_out / model / input_stem
|
||||
|
||||
stem_files = {}
|
||||
if stem_src_dir.exists():
|
||||
for f in sorted(stem_src_dir.iterdir()):
|
||||
if f.is_file():
|
||||
stem_name = f.stem.lower()
|
||||
for prefix in [input_stem.lower(), 'htdemucs', 'mdx']:
|
||||
stem_name = stem_name.replace(prefix, '').strip('_- ')
|
||||
if not stem_name:
|
||||
stem_name = f.stem
|
||||
|
||||
dest = stems_dir / f"{input_stem} - {stem_name.capitalize()}{f.suffix}"
|
||||
shutil.move(str(f), str(dest))
|
||||
stem_files[stem_name] = str(dest)
|
||||
|
||||
try:
|
||||
shutil.rmtree(temp_out, ignore_errors=True)
|
||||
except:
|
||||
pass
|
||||
|
||||
return stem_files
|
||||
|
||||
except Exception:
|
||||
try:
|
||||
shutil.rmtree(temp_out, ignore_errors=True)
|
||||
except:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def _separate_bsroformer(
|
||||
job_id: str, input_path: str, model: str, output_format: str, stems_dir: Path,
|
||||
mp3_bitrate: str = '256k'
|
||||
) -> dict | None:
|
||||
"""
|
||||
Run BS-RoFormer in-process; returns dict mapping stem names to file paths, or None if cancelled.
|
||||
"""
|
||||
from .bs_roformer_separator import BSRoformerSeparator
|
||||
|
||||
stem_mode = BSR_STEM_MODE.get(model, "4stem")
|
||||
jm.update_job(job_id, progress=f"[1/3] Loading BS-RoFormer ({stem_mode})...")
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return None
|
||||
|
||||
sep = BSRoformerSeparator(
|
||||
cpu_separation=CPU_SEPARATION,
|
||||
output_format=output_format,
|
||||
mp3_bitrate=mp3_bitrate,
|
||||
stem_mode=stem_mode,
|
||||
)
|
||||
|
||||
jm.update_job(job_id, progress="[1/3] Separating with BS-RoFormer...")
|
||||
|
||||
temp_out = JOBS_DIR / job_id / "bsr_out"
|
||||
temp_out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return None
|
||||
|
||||
input_stem = Path(input_path).stem
|
||||
stem_files = {}
|
||||
|
||||
for stem_name, path in sorted(stem_paths.items()):
|
||||
src = Path(path)
|
||||
dest = stems_dir / f"{input_stem} - {stem_name.capitalize()}{src.suffix}"
|
||||
shutil.move(str(src), str(dest))
|
||||
stem_files[stem_name.lower()] = str(dest)
|
||||
|
||||
try:
|
||||
shutil.rmtree(temp_out, ignore_errors=True)
|
||||
except:
|
||||
pass
|
||||
|
||||
return stem_files
|
||||
|
||||
except Exception:
|
||||
try:
|
||||
shutil.rmtree(temp_out, ignore_errors=True)
|
||||
except:
|
||||
pass
|
||||
raise
|
||||
Executable
+96
@@ -0,0 +1,96 @@
|
||||
"""
|
||||
Preset mixing: combine separated stems into a single output file.
|
||||
|
||||
- Instrumental: all non-vocal stems mixed
|
||||
- Static: all stems except user-selected ones mixed
|
||||
- Dynamic: all stems mixed with per-stem volume multipliers (0.0–2.0)
|
||||
"""
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
from pydub import AudioSegment
|
||||
|
||||
|
||||
def _load(path: str) -> AudioSegment:
|
||||
return AudioSegment.from_file(path)
|
||||
|
||||
|
||||
def _apply_gain(audio: AudioSegment, multiplier: float) -> AudioSegment:
|
||||
"""Convert a linear multiplier to dB and apply as gain."""
|
||||
if multiplier <= 0.0:
|
||||
return AudioSegment.silent(duration=len(audio), frame_rate=audio.frame_rate)
|
||||
gain_db = 20 * math.log10(multiplier)
|
||||
return audio + gain_db
|
||||
|
||||
|
||||
def _export(audio: AudioSegment, dest: Path, output_format: str) -> None:
|
||||
if output_format == 'mp3_320':
|
||||
audio.export(str(dest), format='mp3', bitrate='320k')
|
||||
elif output_format == 'mp3':
|
||||
audio.export(str(dest), format='mp3', bitrate='256k')
|
||||
elif output_format == 'flac':
|
||||
audio.export(str(dest), format='flac')
|
||||
else:
|
||||
audio.export(str(dest), format='wav')
|
||||
|
||||
|
||||
def _mix(paths: List[str]) -> AudioSegment:
|
||||
"""Overlay all audio files into one mix."""
|
||||
if not paths:
|
||||
raise ValueError("No stems to mix")
|
||||
result = _load(paths[0])
|
||||
for p in paths[1:]:
|
||||
result = result.overlay(_load(p))
|
||||
return result
|
||||
|
||||
|
||||
def make_instrumental(
|
||||
stem_files: Dict[str, str],
|
||||
output_path: Path,
|
||||
output_format: str,
|
||||
) -> str:
|
||||
"""Mix all stems except those with 'vocal' in the stem name."""
|
||||
keep = [p for name, p in stem_files.items() if 'vocal' not in name.lower()]
|
||||
if not keep:
|
||||
raise RuntimeError("No non-vocal stems available for Instrumental preset")
|
||||
_export(_mix(keep), output_path, output_format)
|
||||
return str(output_path)
|
||||
|
||||
|
||||
def make_static(
|
||||
stem_files: Dict[str, str],
|
||||
muted_stems: List[str],
|
||||
output_path: Path,
|
||||
output_format: str,
|
||||
) -> str:
|
||||
"""Mix all stems except those listed in muted_stems."""
|
||||
muted_lower = {s.lower() for s in muted_stems}
|
||||
keep = [p for name, p in stem_files.items() if name.lower() not in muted_lower]
|
||||
if not keep:
|
||||
raise RuntimeError("All stems are muted — nothing to mix for Static preset")
|
||||
_export(_mix(keep), output_path, output_format)
|
||||
return str(output_path)
|
||||
|
||||
|
||||
def make_dynamic(
|
||||
stem_files: Dict[str, str],
|
||||
volumes: Dict[str, float],
|
||||
output_path: Path,
|
||||
output_format: str,
|
||||
) -> str:
|
||||
"""
|
||||
Mix stems with per-stem linear volume multipliers.
|
||||
volumes keys are stem names (lowercase); missing stems default to 1.0.
|
||||
"""
|
||||
adjusted: List[AudioSegment] = []
|
||||
for name, path in stem_files.items():
|
||||
vol = volumes.get(name.lower(), 1.0)
|
||||
adjusted.append(_apply_gain(_load(path), vol))
|
||||
if not adjusted:
|
||||
raise RuntimeError("No stems available for Dynamic preset")
|
||||
result = adjusted[0]
|
||||
for a in adjusted[1:]:
|
||||
result = result.overlay(a)
|
||||
_export(result, output_path, output_format)
|
||||
return str(output_path)
|
||||
Executable
+93
@@ -0,0 +1,93 @@
|
||||
"""
|
||||
Project metadata file service.
|
||||
|
||||
Writes/reads a project.json snapshot in each project root folder.
|
||||
This file mirrors the DB state and can be used for browsing, repair,
|
||||
and offline project identification without a database query.
|
||||
"""
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
|
||||
_METADATA_FILENAME = "project.json"
|
||||
|
||||
|
||||
def write_project_metadata(
|
||||
project_dir: Path,
|
||||
*,
|
||||
project_id: str,
|
||||
name: str,
|
||||
artist: str = "",
|
||||
title: str = "",
|
||||
model: str = "",
|
||||
model_label: str = "",
|
||||
format: str = "",
|
||||
format_label: str = "",
|
||||
source_file: str = "",
|
||||
stems: dict,
|
||||
sep_job_id: str = "",
|
||||
trans_jobs: Optional[dict] = None,
|
||||
abc_jobs: Optional[dict] = None,
|
||||
lrc_jobs: Optional[dict] = None,
|
||||
mix_job_id: Optional[str] = None,
|
||||
created_at: Optional[str] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Write (or overwrite) project.json in *project_dir*.
|
||||
|
||||
If the file already exists its contents are merged so that keys not
|
||||
supplied by the caller (e.g. trans_jobs added later) are preserved.
|
||||
"""
|
||||
meta_path = project_dir / _METADATA_FILENAME
|
||||
|
||||
# Load existing data so we can merge incremental updates
|
||||
existing: dict = {}
|
||||
if meta_path.exists():
|
||||
try:
|
||||
existing = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
existing = {}
|
||||
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
updated = {
|
||||
**existing,
|
||||
"project_id": project_id,
|
||||
"name": name,
|
||||
"artist": artist,
|
||||
"title": title,
|
||||
"model": model,
|
||||
"model_label": model_label,
|
||||
"format": format,
|
||||
"format_label": format_label,
|
||||
"created_at": created_at or existing.get("created_at", now),
|
||||
"updated_at": now,
|
||||
"source_file": source_file or existing.get("source_file", ""),
|
||||
"stems": stems if stems is not None else existing.get("stems", {}),
|
||||
"sep_job_id": sep_job_id or existing.get("sep_job_id", ""),
|
||||
"trans_jobs": trans_jobs if trans_jobs is not None else existing.get("trans_jobs", {}),
|
||||
"abc_jobs": abc_jobs if abc_jobs is not None else existing.get("abc_jobs", {}),
|
||||
"lrc_jobs": lrc_jobs if lrc_jobs is not None else existing.get("lrc_jobs", {}),
|
||||
"mix_job_id": mix_job_id if mix_job_id is not None else existing.get("mix_job_id"),
|
||||
}
|
||||
|
||||
meta_path.write_text(
|
||||
json.dumps(updated, indent=2, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def read_project_metadata(project_dir: Path) -> Optional[dict]:
|
||||
"""
|
||||
Read project.json from *project_dir*.
|
||||
Returns None if the file does not exist or cannot be parsed.
|
||||
"""
|
||||
meta_path = project_dir / _METADATA_FILENAME
|
||||
if not meta_path.exists():
|
||||
return None
|
||||
try:
|
||||
return json.loads(meta_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return None
|
||||
Executable
+72
@@ -0,0 +1,72 @@
|
||||
"""
|
||||
Project folder naming and structure creation.
|
||||
|
||||
Derives a human-readable project name from audio metadata and creates the
|
||||
subdirectory layout under MEDIA_DIR.
|
||||
"""
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
_ILLEGAL = re.compile(r'[<>:"/\\|?*\x00-\x1f]')
|
||||
|
||||
|
||||
def _sanitize(s: str) -> str:
|
||||
"""Remove filesystem-illegal characters and trim trailing dots/spaces."""
|
||||
s = _ILLEGAL.sub('', s).strip().rstrip('. ')
|
||||
return s or ''
|
||||
|
||||
|
||||
def project_name(artist: str, title: str, filename_stem: str = '') -> str:
|
||||
"""
|
||||
Return a human-readable project folder name.
|
||||
Format: "Artist - Title". Falls back to filename_stem if both are missing.
|
||||
"""
|
||||
a = _sanitize(artist)
|
||||
t = _sanitize(title)
|
||||
if a and t:
|
||||
return f"{a} - {t}"
|
||||
if t:
|
||||
return t
|
||||
if a:
|
||||
return a
|
||||
fs = _sanitize(filename_stem)
|
||||
return fs if fs else 'Unknown Project'
|
||||
|
||||
|
||||
def make_project_dirs(
|
||||
name: str,
|
||||
media_dir: Path,
|
||||
presets: Dict[str, bool],
|
||||
) -> Dict[str, Path]:
|
||||
"""
|
||||
Create project subdirectories under media_dir/name/.
|
||||
|
||||
Always creates Stems/. Creates Instrumental/, Static/, Dynamic/ only when
|
||||
the corresponding preset flag is True.
|
||||
|
||||
Returns a dict mapping role names ('project', 'stems', 'instrumental',
|
||||
'static', 'dynamic') to their Path objects.
|
||||
"""
|
||||
project_dir = media_dir / name
|
||||
stems_dir = project_dir / 'Stems'
|
||||
stems_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
dirs: Dict[str, Path] = {'project': project_dir, 'stems': stems_dir}
|
||||
|
||||
if presets.get('instrumental'):
|
||||
d = project_dir / 'Instrumental'
|
||||
d.mkdir(exist_ok=True)
|
||||
dirs['instrumental'] = d
|
||||
|
||||
if presets.get('static'):
|
||||
d = project_dir / 'Static'
|
||||
d.mkdir(exist_ok=True)
|
||||
dirs['static'] = d
|
||||
|
||||
if presets.get('dynamic'):
|
||||
d = project_dir / 'Dynamic'
|
||||
d.mkdir(exist_ok=True)
|
||||
dirs['dynamic'] = d
|
||||
|
||||
return dirs
|
||||
Executable
+321
@@ -0,0 +1,321 @@
|
||||
"""
|
||||
SCNet model architecture — vendored from:
|
||||
https://github.com/ZFTurbo/Music-Source-Separation-Training (MIT license)
|
||||
Paper: https://arxiv.org/abs/2401.13276
|
||||
|
||||
Contains SeparationNet and SCNet classes only; no training or I/O code.
|
||||
"""
|
||||
import math
|
||||
from collections import deque
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch.nn.modules.rnn import LSTM
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SeparationNet (dual-path RNN + FFT feature conversion)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class FeatureConversion(nn.Module):
|
||||
def __init__(self, channels, inverse):
|
||||
super().__init__()
|
||||
self.inverse = inverse
|
||||
self.channels = channels
|
||||
|
||||
def forward(self, x):
|
||||
if self.inverse:
|
||||
x = x.float()
|
||||
x_r = x[:, :self.channels // 2, :, :]
|
||||
x_i = x[:, self.channels // 2:, :, :]
|
||||
x = torch.complex(x_r, x_i)
|
||||
x = torch.fft.irfft(x, dim=3, norm="ortho")
|
||||
else:
|
||||
x = x.float()
|
||||
x = torch.fft.rfft(x, dim=3, norm="ortho")
|
||||
x = torch.cat([x.real, x.imag], dim=1)
|
||||
return x
|
||||
|
||||
|
||||
class DualPathRNN(nn.Module):
|
||||
def __init__(self, d_model, expand, bidirectional=True):
|
||||
super().__init__()
|
||||
self.d_model = d_model
|
||||
self.hidden_size = d_model * expand
|
||||
self.lstm_layers = nn.ModuleList([
|
||||
LSTM(d_model, self.hidden_size, num_layers=1, bidirectional=bidirectional, batch_first=True)
|
||||
for _ in range(2)
|
||||
])
|
||||
self.linear_layers = nn.ModuleList([nn.Linear(self.hidden_size * 2, d_model) for _ in range(2)])
|
||||
self.norm_layers = nn.ModuleList([nn.GroupNorm(1, d_model) for _ in range(2)])
|
||||
|
||||
def forward(self, x):
|
||||
B, C, F, T = x.shape
|
||||
original_x = x
|
||||
x = self.norm_layers[0](x)
|
||||
x = x.transpose(1, 3).contiguous().view(B * T, F, C)
|
||||
x, _ = self.lstm_layers[0](x)
|
||||
x = self.linear_layers[0](x)
|
||||
x = x.view(B, T, F, C).transpose(1, 3) + original_x
|
||||
|
||||
original_x = x
|
||||
x = self.norm_layers[1](x)
|
||||
x = x.transpose(1, 2).contiguous().view(B * F, C, T).transpose(1, 2)
|
||||
x, _ = self.lstm_layers[1](x)
|
||||
x = self.linear_layers[1](x)
|
||||
x = x.transpose(1, 2).contiguous().view(B, F, C, T).transpose(1, 2) + original_x
|
||||
return x
|
||||
|
||||
|
||||
class SeparationNet(nn.Module):
|
||||
def __init__(self, channels, expand=1, num_layers=6):
|
||||
super().__init__()
|
||||
self.num_layers = num_layers
|
||||
self.dp_modules = nn.ModuleList([
|
||||
DualPathRNN(channels * (2 if i % 2 == 1 else 1), expand)
|
||||
for i in range(num_layers)
|
||||
])
|
||||
self.feature_conversion = nn.ModuleList([
|
||||
FeatureConversion(channels * 2, inverse=(i % 2 == 1))
|
||||
for i in range(num_layers)
|
||||
])
|
||||
|
||||
def forward(self, x):
|
||||
for dp, fc in zip(self.dp_modules, self.feature_conversion):
|
||||
x = dp(x)
|
||||
x = fc(x)
|
||||
return x
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SCNet building blocks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class Swish(nn.Module):
|
||||
def forward(self, x):
|
||||
return x * x.sigmoid()
|
||||
|
||||
|
||||
class ConvolutionModule(nn.Module):
|
||||
def __init__(self, channels, depth=2, compress=4, kernel=3):
|
||||
super().__init__()
|
||||
assert kernel % 2 == 1
|
||||
hidden = int(channels / compress)
|
||||
norm = lambda d: nn.GroupNorm(1, d)
|
||||
self.layers = nn.ModuleList()
|
||||
for _ in range(abs(depth)):
|
||||
p = kernel // 2
|
||||
self.layers.append(nn.Sequential(
|
||||
norm(channels),
|
||||
nn.Conv1d(channels, hidden * 2, kernel, padding=p),
|
||||
nn.GLU(1),
|
||||
nn.Conv1d(hidden, hidden, kernel, padding=p, groups=hidden),
|
||||
norm(hidden),
|
||||
Swish(),
|
||||
nn.Conv1d(hidden, channels, 1),
|
||||
))
|
||||
|
||||
def forward(self, x):
|
||||
for layer in self.layers:
|
||||
x = x + layer(x)
|
||||
return x
|
||||
|
||||
|
||||
class FusionLayer(nn.Module):
|
||||
def __init__(self, channels, kernel_size=3, stride=1, padding=1):
|
||||
super().__init__()
|
||||
self.conv = nn.Conv2d(channels * 2, channels * 2, kernel_size, stride=stride, padding=padding)
|
||||
|
||||
def forward(self, x, skip=None):
|
||||
if skip is not None:
|
||||
x = x + skip
|
||||
x = self.conv(x.repeat(1, 2, 1, 1))
|
||||
return F.glu(x, dim=1)
|
||||
|
||||
|
||||
class SDlayer(nn.Module):
|
||||
def __init__(self, channels_in, channels_out, band_configs):
|
||||
super().__init__()
|
||||
self.convs = nn.ModuleList()
|
||||
self.strides, self.kernels = [], []
|
||||
for cfg in band_configs.values():
|
||||
self.convs.append(
|
||||
nn.Conv2d(channels_in, channels_out, (cfg['kernel'], 1), (cfg['stride'], 1))
|
||||
)
|
||||
self.strides.append(cfg['stride'])
|
||||
self.kernels.append(cfg['kernel'])
|
||||
self.SR_low = band_configs['low']['SR']
|
||||
self.SR_mid = band_configs['mid']['SR']
|
||||
|
||||
def forward(self, x):
|
||||
B, C, Fr, T = x.shape
|
||||
splits = [
|
||||
(0, math.ceil(Fr * self.SR_low)),
|
||||
(math.ceil(Fr * self.SR_low), math.ceil(Fr * (self.SR_low + self.SR_mid))),
|
||||
(math.ceil(Fr * (self.SR_low + self.SR_mid)), Fr),
|
||||
]
|
||||
outputs, original_lengths = [], []
|
||||
for conv, stride, kernel, (start, end) in zip(self.convs, self.strides, self.kernels, splits):
|
||||
extracted = x[:, :, start:end, :]
|
||||
original_lengths.append(end - start)
|
||||
L = extracted.shape[2]
|
||||
total_pad = (kernel - stride) if stride == 1 else (stride - L % stride) % stride
|
||||
pad_l = total_pad // 2
|
||||
outputs.append(conv(F.pad(extracted, (0, 0, pad_l, total_pad - pad_l))))
|
||||
return outputs, original_lengths
|
||||
|
||||
|
||||
class SUlayer(nn.Module):
|
||||
def __init__(self, channels_in, channels_out, band_configs):
|
||||
super().__init__()
|
||||
self.convtrs = nn.ModuleList([
|
||||
nn.ConvTranspose2d(channels_in, channels_out, [cfg['kernel'], 1], [cfg['stride'], 1])
|
||||
for cfg in band_configs.values()
|
||||
])
|
||||
|
||||
def forward(self, x, lengths, origin_lengths):
|
||||
splits = [
|
||||
(0, lengths[0]),
|
||||
(lengths[0], lengths[0] + lengths[1]),
|
||||
(lengths[0] + lengths[1], None),
|
||||
]
|
||||
outputs = []
|
||||
for convtr, (start, end), orig_len in zip(self.convtrs, splits, origin_lengths):
|
||||
out = convtr(x[:, :, start:end, :])
|
||||
dist = abs(orig_len - out.shape[2]) // 2
|
||||
outputs.append(out[:, :, dist:dist + orig_len, :])
|
||||
return torch.cat(outputs, dim=2)
|
||||
|
||||
|
||||
class SDblock(nn.Module):
|
||||
def __init__(self, channels_in, channels_out, band_configs={}, conv_config={}, depths=[3, 2, 1], kernel_size=3):
|
||||
super().__init__()
|
||||
self.SDlayer = SDlayer(channels_in, channels_out, band_configs)
|
||||
self.conv_modules = nn.ModuleList([
|
||||
ConvolutionModule(channels_out, depth, **conv_config) for depth in depths
|
||||
])
|
||||
self.globalconv = nn.Conv2d(channels_out, channels_out, kernel_size, 1, (kernel_size - 1) // 2)
|
||||
|
||||
def forward(self, x):
|
||||
bands, original_lengths = self.SDlayer(x)
|
||||
bands = [
|
||||
F.gelu(
|
||||
conv(band.permute(0, 2, 1, 3).reshape(-1, band.shape[1], band.shape[3]))
|
||||
.view(band.shape[0], band.shape[2], band.shape[1], band.shape[3])
|
||||
.permute(0, 2, 1, 3)
|
||||
)
|
||||
for conv, band in zip(self.conv_modules, bands)
|
||||
]
|
||||
lengths = [b.size(-2) for b in bands]
|
||||
full_band = torch.cat(bands, dim=2)
|
||||
return self.globalconv(full_band), full_band, lengths, original_lengths
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SCNet
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class SCNet(nn.Module):
|
||||
"""
|
||||
Sparse Compression Network for music source separation.
|
||||
Paper: https://arxiv.org/abs/2401.13276
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sources=['drums', 'bass', 'other', 'vocals'],
|
||||
audio_channels=2,
|
||||
dims=[4, 32, 64, 128],
|
||||
nfft=4096,
|
||||
hop_size=1024,
|
||||
win_size=4096,
|
||||
normalized=True,
|
||||
band_SR=[0.175, 0.392, 0.433],
|
||||
band_stride=[1, 4, 16],
|
||||
band_kernel=[3, 4, 16],
|
||||
conv_depths=[3, 2, 1],
|
||||
compress=4,
|
||||
conv_kernel=3,
|
||||
num_dplayer=6,
|
||||
expand=1,
|
||||
):
|
||||
super().__init__()
|
||||
self.sources = sources
|
||||
self.audio_channels = audio_channels
|
||||
self.dims = dims
|
||||
band_keys = ['low', 'mid', 'high']
|
||||
self.band_configs = {
|
||||
band_keys[i]: {'SR': band_SR[i], 'stride': band_stride[i], 'kernel': band_kernel[i]}
|
||||
for i in range(3)
|
||||
}
|
||||
self.hop_length = hop_size
|
||||
self.conv_config = {'compress': compress, 'kernel': conv_kernel}
|
||||
self.stft_config = {
|
||||
'n_fft': nfft, 'hop_length': hop_size, 'win_length': win_size,
|
||||
'center': True, 'normalized': normalized,
|
||||
}
|
||||
|
||||
self.encoder = nn.ModuleList()
|
||||
self.decoder = nn.ModuleList()
|
||||
|
||||
for i in range(len(dims) - 1):
|
||||
self.encoder.append(SDblock(
|
||||
channels_in=dims[i], channels_out=dims[i + 1],
|
||||
band_configs=self.band_configs, conv_config=self.conv_config,
|
||||
depths=conv_depths,
|
||||
))
|
||||
self.decoder.insert(0, nn.Sequential(
|
||||
FusionLayer(channels=dims[i + 1]),
|
||||
SUlayer(
|
||||
channels_in=dims[i + 1],
|
||||
channels_out=dims[i] if i != 0 else dims[i] * len(sources),
|
||||
band_configs=self.band_configs,
|
||||
),
|
||||
))
|
||||
|
||||
self.separation_net = SeparationNet(
|
||||
channels=dims[-1], expand=expand, num_layers=num_dplayer,
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
B = x.shape[0]
|
||||
padding = self.hop_length - x.shape[-1] % self.hop_length
|
||||
if (x.shape[-1] + padding) // self.hop_length % 2 == 0:
|
||||
padding += self.hop_length
|
||||
x = F.pad(x, (0, padding))
|
||||
|
||||
L = x.shape[-1]
|
||||
x = x.reshape(-1, L)
|
||||
x = torch.stft(x, **self.stft_config, return_complex=True)
|
||||
x = torch.view_as_real(x)
|
||||
x = x.permute(0, 3, 1, 2).reshape(
|
||||
x.shape[0] // self.audio_channels,
|
||||
x.shape[3] * self.audio_channels,
|
||||
x.shape[1], x.shape[2],
|
||||
)
|
||||
B, C, Fr, T = x.shape
|
||||
|
||||
save_skip = deque()
|
||||
save_lengths = deque()
|
||||
save_orig = deque()
|
||||
|
||||
for sd_layer in self.encoder:
|
||||
x, skip, lengths, orig_len = sd_layer(x)
|
||||
save_skip.append(skip)
|
||||
save_lengths.append(lengths)
|
||||
save_orig.append(orig_len)
|
||||
|
||||
x = self.separation_net(x)
|
||||
|
||||
for fusion_layer, su_layer in self.decoder:
|
||||
x = fusion_layer(x, save_skip.pop())
|
||||
x = su_layer(x, save_lengths.pop(), save_orig.pop())
|
||||
|
||||
n = self.dims[0]
|
||||
x = x.view(B, n, -1, Fr, T)
|
||||
x = x.reshape(-1, 2, Fr, T).permute(0, 2, 3, 1)
|
||||
x = torch.view_as_complex(x.contiguous())
|
||||
x = torch.istft(x, **self.stft_config)
|
||||
x = x.reshape(B, len(self.sources), self.audio_channels, -1)
|
||||
return x[:, :, :, :-padding]
|
||||
Executable
+299
@@ -0,0 +1,299 @@
|
||||
"""
|
||||
SCNet separator — native PyTorch inference.
|
||||
Models from MVSEP-Central-Backup (HuggingFace).
|
||||
Architecture vendored from MSST (MIT license):
|
||||
https://github.com/ZFTurbo/Music-Source-Separation-Training
|
||||
"""
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torchaudio
|
||||
import yaml
|
||||
|
||||
from .scnet_arch import SCNet
|
||||
from ..config import SCNET_MODEL_DIR, CPU_SEPARATION
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model registry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SCNET_MODELS = {
|
||||
"scnet_xl": {
|
||||
"model_file": "scnet_checkpoint_musdb18.ckpt",
|
||||
"config_file": "config_musdb18_scnet_xl.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_model_path": "Politrees/UVR_resources/models/SCnet/scnet_checkpoint_musdb18.ckpt",
|
||||
"hf_config_path": "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_xl.yaml",
|
||||
},
|
||||
"scnet_xl_ihf": {
|
||||
"model_file": "scnet_checkpoint_musdb18.ckpt",
|
||||
"config_file": "config_musdb18_scnet_xl.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_model_path": "Politrees/UVR_resources/models/SCnet/scnet_checkpoint_musdb18.ckpt",
|
||||
"hf_config_path": "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_xl.yaml",
|
||||
},
|
||||
"scnet_large": {
|
||||
"model_file": "SCNet-large_starrytong_fixed.ckpt",
|
||||
"config_file": "config_musdb18_scnet_large_starrytong.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_model_path": "Politrees/UVR_resources/models/SCnet/SCNet-large_starrytong_fixed.ckpt",
|
||||
"hf_config_path": "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_large_starrytong.yaml",
|
||||
},
|
||||
"scnet_large_starrytong": {
|
||||
"model_file": "SCNet-large_starrytong_fixed.ckpt",
|
||||
"config_file": "config_musdb18_scnet_large_starrytong.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_model_path": "Politrees/UVR_resources/models/SCnet/SCNet-large_starrytong_fixed.ckpt",
|
||||
"hf_config_path": "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_large_starrytong.yaml",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Audio I/O helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_audio(path: str, sample_rate: int) -> np.ndarray:
|
||||
"""Load audio and return (2, samples) float32 array at target sample rate."""
|
||||
waveform, sr = torchaudio.load(path)
|
||||
if sr != sample_rate:
|
||||
waveform = torchaudio.functional.resample(waveform, sr, sample_rate)
|
||||
if waveform.shape[0] == 1:
|
||||
waveform = waveform.repeat(2, 1)
|
||||
elif waveform.shape[0] > 2:
|
||||
waveform = waveform[:2]
|
||||
return waveform.numpy().astype(np.float32)
|
||||
|
||||
|
||||
def _save_audio(path: str, audio: np.ndarray, sample_rate: int, output_format: str):
|
||||
"""Save (channels, samples) numpy array to file via torchaudio (ffmpeg backend)."""
|
||||
tensor = torch.tensor(audio, dtype=torch.float32)
|
||||
fmt = output_format.lower()
|
||||
if fmt == "mp3":
|
||||
torchaudio.save(path, tensor, sample_rate, format="mp3")
|
||||
elif fmt == "flac":
|
||||
torchaudio.save(path, tensor, sample_rate, format="flac")
|
||||
else:
|
||||
torchaudio.save(path, tensor, sample_rate)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Overlapping-window inference (MSST style)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _build_window(chunk_size: int, step: int) -> np.ndarray:
|
||||
"""
|
||||
Linear fade-in / flat / fade-out window of length chunk_size.
|
||||
Consecutive windows (offset by step) partition of unity: they sum to a
|
||||
constant, so the overlap-add correctly averages.
|
||||
"""
|
||||
fade = step
|
||||
fade_in = np.linspace(0.0, 1.0, fade, endpoint=False, dtype=np.float32)
|
||||
fade_out = np.linspace(1.0, 0.0, fade, endpoint=False, dtype=np.float32)
|
||||
flat_len = chunk_size - 2 * fade
|
||||
if flat_len > 0:
|
||||
return np.concatenate([fade_in, np.ones(flat_len, dtype=np.float32), fade_out])
|
||||
# num_overlap == 2: no flat region
|
||||
return np.concatenate([fade_in, fade_out])
|
||||
|
||||
|
||||
def _demix(
|
||||
model: SCNet,
|
||||
mix: np.ndarray,
|
||||
chunk_size: int,
|
||||
num_overlap: int,
|
||||
device: torch.device,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
MSST-style overlapping-window inference.
|
||||
|
||||
Args:
|
||||
mix: (channels, samples) float32
|
||||
Returns:
|
||||
(num_sources, channels, samples) float32
|
||||
"""
|
||||
C, N = mix.shape
|
||||
step = chunk_size // num_overlap
|
||||
border = chunk_size - step # == (num_overlap-1)*step
|
||||
|
||||
# Reflect-pad both sides so edge frames get proper context.
|
||||
mix_padded = np.pad(mix, ((0, 0), (border, border)), mode="reflect")
|
||||
|
||||
window = _build_window(chunk_size, step)
|
||||
sources_sum: Optional[np.ndarray] = None
|
||||
weight_sum = np.zeros(mix_padded.shape[-1], dtype=np.float32)
|
||||
|
||||
with torch.no_grad():
|
||||
for start in range(0, N, step):
|
||||
p_end = start + chunk_size
|
||||
if p_end > mix_padded.shape[-1]:
|
||||
break
|
||||
chunk = mix_padded[:, start:p_end] # (C, chunk_size)
|
||||
|
||||
# Per-chunk loudness normalisation
|
||||
mean = float(chunk.mean())
|
||||
std = max(float(chunk.std()), 1e-8)
|
||||
chunk_norm = (chunk - mean) / std
|
||||
|
||||
inp = torch.tensor(chunk_norm[np.newaxis], dtype=torch.float32, device=device)
|
||||
out = model(inp) # (1, num_sources, C, chunk_size)
|
||||
out_np = out.squeeze(0).cpu().numpy() # (num_sources, C, chunk_size)
|
||||
out_np = out_np * std + mean # denormalise
|
||||
|
||||
if sources_sum is None:
|
||||
num_sources = out_np.shape[0]
|
||||
sources_sum = np.zeros(
|
||||
(num_sources, C, mix_padded.shape[-1]), dtype=np.float32
|
||||
)
|
||||
|
||||
sources_sum[:, :, start:p_end] += out_np * window[np.newaxis, np.newaxis, :]
|
||||
weight_sum[start:p_end] += window
|
||||
|
||||
if sources_sum is None:
|
||||
raise RuntimeError("Audio is too short for chunked SCNet inference.")
|
||||
|
||||
weight_sum = np.maximum(weight_sum, 1e-8)
|
||||
sources = sources_sum / weight_sum[np.newaxis, np.newaxis, :]
|
||||
return sources[:, :, border : border + N]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Separator class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class SCNetSeparator:
|
||||
"""Native PyTorch inference wrapper for MSST SCNet models."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_id: str = "scnet_xl",
|
||||
cpu_separation: bool = CPU_SEPARATION,
|
||||
output_format: str = "mp3",
|
||||
mp3_bitrate: str = "256k",
|
||||
):
|
||||
if model_id not in SCNET_MODELS:
|
||||
raise ValueError(
|
||||
f"Unknown SCNet model: {model_id}. "
|
||||
f"Available: {list(SCNET_MODELS)}"
|
||||
)
|
||||
|
||||
self.model_id = model_id
|
||||
self.model_cfg = SCNET_MODELS[model_id]
|
||||
self.cpu_separation = cpu_separation
|
||||
self.output_format = output_format
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self.device = torch.device(
|
||||
"cpu" if cpu_separation else ("cuda" if torch.cuda.is_available() else "cpu")
|
||||
)
|
||||
|
||||
self._model: Optional[SCNet] = None
|
||||
self._config: Optional[dict] = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Model management
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _ensure_model(self) -> tuple[Path, Path]:
|
||||
"""Download model and config from HuggingFace if not already cached."""
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
model_dir = SCNET_MODEL_DIR / self.model_id
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
model_path = model_dir / self.model_cfg["model_file"]
|
||||
config_path = model_dir / self.model_cfg["config_file"]
|
||||
|
||||
if not model_path.exists():
|
||||
print(f"[SCNet] Downloading {self.model_cfg['model_file']} …")
|
||||
cached = hf_hub_download(
|
||||
repo_id=self.model_cfg["hf_repo"],
|
||||
filename=self.model_cfg["hf_model_path"],
|
||||
)
|
||||
shutil.copy2(cached, model_path)
|
||||
|
||||
if not config_path.exists():
|
||||
print(f"[SCNet] Downloading {self.model_cfg['config_file']} …")
|
||||
cached = hf_hub_download(
|
||||
repo_id=self.model_cfg["hf_repo"],
|
||||
filename=self.model_cfg["hf_config_path"],
|
||||
)
|
||||
shutil.copy2(cached, config_path)
|
||||
|
||||
return model_path, config_path
|
||||
|
||||
def _load_model(self) -> None:
|
||||
"""Load YAML config and PyTorch checkpoint; instantiate SCNet."""
|
||||
model_path, config_path = self._ensure_model()
|
||||
|
||||
# Some config files use !!python/tuple (e.g. mixup_probs); add a
|
||||
# constructor so safe_load can handle them without crashing.
|
||||
loader = yaml.SafeLoader
|
||||
loader.add_constructor(
|
||||
"tag:yaml.org,2002:python/tuple",
|
||||
lambda ldr, node: tuple(ldr.construct_sequence(node)),
|
||||
)
|
||||
with open(config_path) as f:
|
||||
cfg = yaml.load(f, Loader=loader)
|
||||
self._config = cfg
|
||||
|
||||
model_params = cfg.get("model", {})
|
||||
model = SCNet(**model_params)
|
||||
|
||||
state = torch.load(str(model_path), map_location="cpu", weights_only=False)
|
||||
if "state_dict" in state:
|
||||
state = state["state_dict"]
|
||||
model.load_state_dict(state)
|
||||
model.eval()
|
||||
self._model = model.to(self.device)
|
||||
|
||||
def _get_model(self) -> SCNet:
|
||||
if self._model is None:
|
||||
self._load_model()
|
||||
return self._model
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
||||
"""
|
||||
Separate audio into stems.
|
||||
|
||||
Returns:
|
||||
Dict mapping stem name → output file path.
|
||||
"""
|
||||
model = self._get_model()
|
||||
cfg = self._config
|
||||
|
||||
sample_rate: int = cfg.get("audio", {}).get("sample_rate", 44100)
|
||||
inf_cfg: dict = cfg.get("inference", {})
|
||||
chunk_size: int = inf_cfg.get("chunk_size", 485100)
|
||||
num_overlap: int = inf_cfg.get("num_overlap", 2)
|
||||
|
||||
# Stem order from training config; fall back to model attribute
|
||||
instruments = cfg.get("training", {}).get("instruments") or list(model.sources)
|
||||
|
||||
mix = _load_audio(input_path, sample_rate) # (2, N)
|
||||
|
||||
print(f"[SCNet] Running {self.model_id} on {self.device} …")
|
||||
sources_np = _demix(model, mix, chunk_size, num_overlap, self.device)
|
||||
# sources_np: (num_sources, 2, N)
|
||||
|
||||
out_dir = Path(output_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_paths: Dict[str, str] = {}
|
||||
|
||||
for i, stem in enumerate(instruments):
|
||||
if i >= sources_np.shape[0]:
|
||||
break
|
||||
ext = self.output_format.lower()
|
||||
out_path = str(out_dir / f"{stem}.{ext}")
|
||||
_save_audio(out_path, sources_np[i], sample_rate, self.output_format)
|
||||
output_paths[stem] = out_path
|
||||
print(f"[SCNet] Saved {stem} → {out_path}")
|
||||
|
||||
return output_paths
|
||||
Executable
+1831
File diff suppressed because it is too large.
Load diff
Executable
+281
@@ -0,0 +1,281 @@
|
||||
"""
|
||||
Subsonic REST API client and local import service.
|
||||
|
||||
Auth: token-based (md5(password+salt)), salt generated per request.
|
||||
Config priority:
|
||||
1. MEDIA_DIR/subsonic.json (written by the Settings UI or seed_from_env())
|
||||
2. Environment variables SUBSONIC_URL / SUBSONIC_USERNAME / SUBSONIC_PASSWORD
|
||||
— on first use these are written to subsonic.json so the UI reflects them.
|
||||
|
||||
Credential storage:
|
||||
The password is never written to disk in plaintext. It is encrypted with
|
||||
Fernet (AES-128-CBC + HMAC-SHA256) using a key derived from SECRET_KEY.
|
||||
If SECRET_KEY is not set, credentials can still be supplied via environment
|
||||
variables but cannot be persisted to disk (the Settings UI will save only
|
||||
URL + username and prompt for the password on every restart).
|
||||
"""
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import requests as _requests
|
||||
from cryptography.fernet import Fernet, InvalidToken
|
||||
|
||||
from ..config import SUBSONIC_CONFIG_PATH, IMPORTS_DIR
|
||||
|
||||
_AUDIO_EXTS = {'.mp3', '.flac', '.wav', '.m4a', '.ogg', '.opus', '.aac'}
|
||||
_CLIENT_ID = 'StemEngine'
|
||||
_API_VERSION = '1.16.1'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Encryption helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_fernet: Optional[Fernet] = None
|
||||
_fernet_init = False # sentinel: False = derivation not yet attempted
|
||||
|
||||
|
||||
def _get_fernet() -> Optional[Fernet]:
|
||||
"""
|
||||
Return a Fernet instance keyed by SECRET_KEY, or None if SECRET_KEY is not set.
|
||||
Key is derived once via PBKDF2-SHA256 (200k rounds) and cached for the
|
||||
process lifetime.
|
||||
"""
|
||||
global _fernet, _fernet_init
|
||||
if _fernet_init:
|
||||
return _fernet
|
||||
_fernet_init = True
|
||||
|
||||
raw_key = os.environ.get('SECRET_KEY', '').strip()
|
||||
if not raw_key:
|
||||
_fernet = None
|
||||
return None
|
||||
|
||||
# Derive a 32-byte key from whatever the user provides
|
||||
derived = hashlib.pbkdf2_hmac(
|
||||
'sha256', raw_key.encode(), b'stemengine-subsonic', 200_000, dklen=32
|
||||
)
|
||||
_fernet = Fernet(base64.urlsafe_b64encode(derived))
|
||||
return _fernet
|
||||
|
||||
|
||||
def _encrypt(plaintext: str) -> str:
|
||||
f = _get_fernet()
|
||||
if f is None:
|
||||
raise RuntimeError(
|
||||
"SECRET_KEY is not set — cannot persist credentials to disk. "
|
||||
"Add SECRET_KEY to docker-compose.yml to enable Settings UI credential storage."
|
||||
)
|
||||
return f.encrypt(plaintext.encode()).decode()
|
||||
|
||||
|
||||
def _decrypt(ciphertext: str) -> str:
|
||||
f = _get_fernet()
|
||||
if f is None:
|
||||
raise RuntimeError(
|
||||
"SECRET_KEY is not set — cannot decrypt stored credentials. "
|
||||
"Set SECRET_KEY or re-enter credentials via the Settings UI."
|
||||
)
|
||||
try:
|
||||
return f.decrypt(ciphertext.encode()).decode()
|
||||
except InvalidToken as exc:
|
||||
raise RuntimeError(
|
||||
"Credential decryption failed — SECRET_KEY may have changed. "
|
||||
"Re-enter credentials via the Settings UI."
|
||||
) from exc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _config_from_env() -> Optional[dict]:
|
||||
"""Return config from environment variables, or None if not fully set."""
|
||||
url = os.environ.get('SUBSONIC_URL', '').strip()
|
||||
username = os.environ.get('SUBSONIC_USERNAME', '').strip()
|
||||
password = os.environ.get('SUBSONIC_PASSWORD', '').strip()
|
||||
if url and username and password:
|
||||
return {'url': url, 'username': username, 'password': password}
|
||||
return None
|
||||
|
||||
|
||||
def load_config() -> Optional[dict]:
|
||||
"""
|
||||
Return the active Subsonic config dict (plaintext password ready for use),
|
||||
or None if no credentials are available.
|
||||
|
||||
Handles encrypted-at-rest configs (_enc=True) and the legacy plaintext format.
|
||||
"""
|
||||
if SUBSONIC_CONFIG_PATH.exists():
|
||||
try:
|
||||
stored = json.loads(SUBSONIC_CONFIG_PATH.read_text())
|
||||
if stored.get('_enc'):
|
||||
pwd = _decrypt(stored['password'])
|
||||
return {'url': stored['url'], 'username': stored['username'], 'password': pwd}
|
||||
else:
|
||||
# Legacy plaintext — serve as-is; the UI can re-save to upgrade
|
||||
return {k: v for k, v in stored.items() if not k.startswith('_')}
|
||||
except RuntimeError:
|
||||
raise # Surface decryption / key errors to the caller
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
env_cfg = _config_from_env()
|
||||
if env_cfg:
|
||||
try:
|
||||
save_config(env_cfg) # Persist encrypted if SECRET_KEY is available
|
||||
except RuntimeError:
|
||||
pass # No key — serve from env, don't write to disk
|
||||
return env_cfg
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def save_config(config: dict) -> None:
|
||||
"""
|
||||
Persist Subsonic config with the password encrypted at rest.
|
||||
Raises RuntimeError if SECRET_KEY is not set.
|
||||
"""
|
||||
stored = {
|
||||
'url': config['url'],
|
||||
'username': config['username'],
|
||||
'password': _encrypt(config['password']),
|
||||
'_enc': True,
|
||||
}
|
||||
SUBSONIC_CONFIG_PATH.write_text(json.dumps(stored, indent=2))
|
||||
|
||||
|
||||
def delete_config() -> None:
|
||||
if SUBSONIC_CONFIG_PATH.exists():
|
||||
SUBSONIC_CONFIG_PATH.unlink()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth / URL helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_token(password: str) -> tuple[str, str]:
|
||||
"""Return (token, salt) for Subsonic token auth."""
|
||||
salt = secrets.token_hex(8)
|
||||
token = hashlib.md5((password + salt).encode()).hexdigest()
|
||||
return token, salt
|
||||
|
||||
|
||||
def _params(config: dict) -> dict:
|
||||
token, salt = _make_token(config['password'])
|
||||
return {
|
||||
'u': config['username'],
|
||||
't': token,
|
||||
's': salt,
|
||||
'v': _API_VERSION,
|
||||
'c': _CLIENT_ID,
|
||||
'f': 'json',
|
||||
}
|
||||
|
||||
|
||||
def _url(config: dict, endpoint: str) -> str:
|
||||
return f"{config['url'].rstrip('/')}/rest/{endpoint}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API methods
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_connection(config: dict) -> bool:
|
||||
"""Ping the Subsonic server; return True if auth succeeds."""
|
||||
try:
|
||||
r = _requests.get(_url(config, 'ping.view'), params=_params(config), timeout=10)
|
||||
data = r.json()
|
||||
return data.get('subsonic-response', {}).get('status') == 'ok'
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def search_songs(query: str, count: int = 20) -> list[dict]:
|
||||
"""Search for songs by keyword. Returns a list of song dicts."""
|
||||
config = load_config()
|
||||
if not config:
|
||||
raise RuntimeError("Subsonic server not configured")
|
||||
|
||||
p = _params(config)
|
||||
p.update({'query': query, 'songCount': count, 'artistCount': 0, 'albumCount': 0})
|
||||
|
||||
r = _requests.get(_url(config, 'search3.view'), params=p, timeout=15)
|
||||
r.raise_for_status()
|
||||
resp = r.json().get('subsonic-response', {})
|
||||
|
||||
if resp.get('status') != 'ok':
|
||||
msg = resp.get('error', {}).get('message', 'Subsonic API error')
|
||||
raise RuntimeError(msg)
|
||||
|
||||
return [
|
||||
{
|
||||
'id': s.get('id', ''),
|
||||
'title': s.get('title', 'Unknown'),
|
||||
'artist': s.get('artist', ''),
|
||||
'album': s.get('album', ''),
|
||||
'duration': s.get('duration', 0),
|
||||
'suffix': s.get('suffix', 'mp3'),
|
||||
'size': s.get('size', 0),
|
||||
'bitRate': s.get('bitRate', 0),
|
||||
}
|
||||
for s in resp.get('searchResult3', {}).get('song', [])
|
||||
]
|
||||
|
||||
|
||||
def import_song(song_id: str, title: str, artist: str = '', suffix: str = 'mp3') -> dict:
|
||||
"""Download a Subsonic song to IMPORTS_DIR."""
|
||||
config = load_config()
|
||||
if not config:
|
||||
raise RuntimeError("Subsonic server not configured")
|
||||
|
||||
def _safe(s: str) -> str:
|
||||
return "".join(c for c in s if c.isalnum() or c in ' -_().').strip()
|
||||
|
||||
safe_title = _safe(title) or 'Unknown'
|
||||
safe_artist = _safe(artist)
|
||||
base_name = f"{safe_artist} - {safe_title}" if safe_artist else safe_title
|
||||
dest = IMPORTS_DIR / f"{base_name}.{suffix}"
|
||||
|
||||
counter = 1
|
||||
while dest.exists():
|
||||
dest = IMPORTS_DIR / f"{base_name} ({counter}).{suffix}"
|
||||
counter += 1
|
||||
|
||||
p = _params(config)
|
||||
p['id'] = song_id
|
||||
|
||||
r = _requests.get(_url(config, 'stream.view'), params=p, stream=True, timeout=120)
|
||||
r.raise_for_status()
|
||||
|
||||
with open(dest, 'wb') as f:
|
||||
for chunk in r.iter_content(chunk_size=65536):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
|
||||
return {
|
||||
'filename': dest.name,
|
||||
'path': str(dest),
|
||||
'size': dest.stat().st_size,
|
||||
'title': title,
|
||||
'artist': artist,
|
||||
}
|
||||
|
||||
|
||||
def list_imports() -> list[dict]:
|
||||
"""Return metadata for all audio files in IMPORTS_DIR, newest first."""
|
||||
if not IMPORTS_DIR.exists():
|
||||
return []
|
||||
results = []
|
||||
for f in sorted(IMPORTS_DIR.iterdir(), key=lambda x: x.stat().st_mtime, reverse=True):
|
||||
if f.is_file() and f.suffix.lower() in _AUDIO_EXTS:
|
||||
results.append({
|
||||
'filename': f.name,
|
||||
'path': str(f),
|
||||
'size': f.stat().st_size,
|
||||
})
|
||||
return results
|
||||
Executable
+56
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
MIDI transcription via basic-pitch (in-process Python API).
|
||||
Supports two modes mapped to inference parameter presets:
|
||||
polyphonic — general multi-instrument transcription
|
||||
melody — single-voice / vocal melody (melodia trick, min frequency)
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import JOBS_DIR, TRANSCRIBE_PRESETS
|
||||
|
||||
|
||||
def run_transcription(job_id: str, input_path: str, mode: str):
|
||||
"""
|
||||
Background task: transcribe audio to MIDI using basic-pitch.
|
||||
mode: 'polyphonic' | 'melody'
|
||||
"""
|
||||
from basic_pitch.inference import predict
|
||||
from basic_pitch import ICASSP_2022_MODEL_PATH
|
||||
|
||||
output_dir = JOBS_DIR / job_id
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Loading basic-pitch model...")
|
||||
|
||||
try:
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
params = TRANSCRIBE_PRESETS.get(mode, TRANSCRIBE_PRESETS["polyphonic"])
|
||||
jm.update_job(job_id, progress=f"Transcribing with basic-pitch ({mode})...")
|
||||
|
||||
_, midi_data, _ = predict(
|
||||
input_path,
|
||||
ICASSP_2022_MODEL_PATH,
|
||||
**params,
|
||||
)
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
output_name = f"{Path(input_path).stem}-{mode}.mid"
|
||||
dest = output_dir / output_name
|
||||
midi_data.write(str(dest))
|
||||
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.DONE,
|
||||
progress="Transcription complete.",
|
||||
files=[{"name": dest.name, "path": str(dest)}],
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
|
||||
Whitespace-only changes.
+129
@@ -0,0 +1,129 @@
|
||||
from music21 import environment, converter, metadata, stream
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import os
|
||||
import traceback
|
||||
|
||||
|
||||
def _post_process_abc_content(abc_text: str, title: str) -> str:
|
||||
"""Cleans midi2abc output, sets title, removes comments."""
|
||||
lines = abc_text.splitlines()
|
||||
cleaned_lines = []
|
||||
seen_midi_program_in_voice = False
|
||||
|
||||
for line in lines:
|
||||
# The midi2abc tool generates a default title from the filename.
|
||||
if line.startswith('T:'):
|
||||
cleaned_lines.append(f'T: {title}')
|
||||
continue
|
||||
|
||||
if line.strip().startswith('% Last note suggests'):
|
||||
continue
|
||||
|
||||
if line.strip().startswith('V:'):
|
||||
seen_midi_program_in_voice = False
|
||||
if line.strip().startswith('%%MIDI program'):
|
||||
if not seen_midi_program_in_voice:
|
||||
seen_midi_program_in_voice = True
|
||||
cleaned_lines.append(line)
|
||||
continue
|
||||
|
||||
cleaned_lines.append(line)
|
||||
|
||||
return '\n'.join(cleaned_lines)
|
||||
|
||||
|
||||
def convert_midi_to_abc(
|
||||
midi_paths: list, song_title: str
|
||||
) -> dict[str, str] | None:
|
||||
"""Converts MIDI files to a dictionary of ABC strings."""
|
||||
us = environment.UserSettings()
|
||||
us['directoryScratch'] = '/tmp'
|
||||
|
||||
temp_dir = Path(us['directoryScratch'])
|
||||
|
||||
abc_results = {}
|
||||
|
||||
for midi_path in midi_paths:
|
||||
part_name = Path(midi_path).stem
|
||||
part_title = f"{song_title} ({part_name})"
|
||||
print(f" -> Processing part: {part_name}.mid")
|
||||
|
||||
temp_midi_path = temp_dir / f"{part_title}.mid"
|
||||
temp_abc_path = temp_dir / f"{part_title}.abc"
|
||||
|
||||
try:
|
||||
score_part = converter.parse(midi_path, forceSource=True)
|
||||
if not score_part or not score_part.flatten().notesAndRests:
|
||||
print(f" -> Skipping empty MIDI: {Path(midi_path).name}") # noqa
|
||||
continue
|
||||
|
||||
single_part_score = stream.Score()
|
||||
single_part_score.insert(0, metadata.Metadata())
|
||||
single_part_score.metadata.title = part_title
|
||||
score_part.parts[0].id = part_name
|
||||
single_part_score.insert(0, score_part.parts[0])
|
||||
|
||||
single_part_score.write('midi', fp=str(temp_midi_path))
|
||||
|
||||
cmd = ['midi2abc', str(temp_midi_path),
|
||||
'-o', str(temp_abc_path)]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, check=False)
|
||||
|
||||
if result.returncode != 0:
|
||||
print(f" -> ERROR: midi2abc failed for {part_name}: "
|
||||
f"{result.stderr}")
|
||||
continue
|
||||
|
||||
if not temp_abc_path.exists() or temp_abc_path.stat().st_size == 0:
|
||||
print(f" -> ERROR: midi2abc created an empty file for {part_name}.")
|
||||
continue
|
||||
|
||||
with open(temp_abc_path, 'r', encoding='utf-8') as f:
|
||||
abc_text_content = f.read()
|
||||
|
||||
abc_results[part_name] = _post_process_abc_content(
|
||||
abc_text_content, part_title)
|
||||
print(f" -> Successfully converted {part_name} to ABC.")
|
||||
|
||||
except Exception as e:
|
||||
print(
|
||||
f" -> ERROR during ABC conversion for {part_name}: {e}")
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
if temp_midi_path.exists():
|
||||
os.remove(temp_midi_path)
|
||||
if temp_abc_path.exists():
|
||||
os.remove(temp_abc_path)
|
||||
|
||||
return abc_results if abc_results else None
|
||||
|
||||
|
||||
def generate_mix_abc(mix_midi_path: str, song_title: str) -> str | None:
|
||||
"""Generates ABC notation for a single 'mix' MIDI file."""
|
||||
us = environment.UserSettings()
|
||||
us['directoryScratch'] = '/tmp'
|
||||
temp_dir = Path(us['directoryScratch'])
|
||||
mix_title = f"{song_title} (Mix)"
|
||||
temp_abc_path = temp_dir / f"{mix_title}.abc"
|
||||
|
||||
try:
|
||||
print("\n -> Creating combined 'Mix' ABC score...")
|
||||
cmd = ['midi2abc', mix_midi_path, '-o', str(temp_abc_path)]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, check=False)
|
||||
|
||||
if result.returncode == 0 and temp_abc_path.exists():
|
||||
raw_mix_abc = temp_abc_path.read_text(encoding='utf-8')
|
||||
cleaned_mix_abc = _post_process_abc_content(raw_mix_abc, mix_title)
|
||||
print(" -> Successfully created 'Mix' ABC.")
|
||||
return cleaned_mix_abc
|
||||
else:
|
||||
print(
|
||||
f" -> ERROR: midi2abc failed for Mix: {result.stderr}")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f" -> ERROR during Mix ABC conversion: {e}")
|
||||
return None
|
||||
finally:
|
||||
if temp_abc_path.exists():
|
||||
os.remove(temp_abc_path)
|
||||
Executable
+70
@@ -0,0 +1,70 @@
|
||||
"""
|
||||
ABC notation conversion via abc_generator (backend/services/abc_generator.py).
|
||||
"""
|
||||
from .. import job_manager as jm
|
||||
from ..config import JOBS_DIR
|
||||
from . import abc_generator
|
||||
|
||||
|
||||
def run_abc_conversion(job_id: str, midi_paths: list[str], song_title: str):
|
||||
"""
|
||||
Background task: convert MIDI file(s) to ABC notation.
|
||||
midi_paths: list of MIDI file paths
|
||||
song_title: title to use in ABC headers
|
||||
"""
|
||||
output_dir = JOBS_DIR / job_id
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Converting MIDI to ABC...")
|
||||
|
||||
try:
|
||||
abc_results = abc_generator.convert_midi_to_abc(midi_paths, song_title)
|
||||
|
||||
if not abc_results:
|
||||
raise RuntimeError("No ABC output produced — MIDI may be empty or unreadable.")
|
||||
|
||||
files = []
|
||||
for part_name, abc_text in abc_results.items():
|
||||
safe_name = part_name.replace("/", "_").replace("\\", "_")
|
||||
filename = f"{safe_name}.abc"
|
||||
dest = output_dir / filename
|
||||
dest.write_text(abc_text, encoding="utf-8")
|
||||
files.append({"name": filename, "path": str(dest)})
|
||||
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.DONE,
|
||||
progress="ABC conversion complete.",
|
||||
files=files,
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
|
||||
|
||||
|
||||
def run_mix_abc_conversion(job_id: str, mix_midi_path: str, song_title: str):
|
||||
"""Convert a single mix MIDI to ABC notation."""
|
||||
output_dir = JOBS_DIR / job_id
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Converting mix MIDI to ABC...")
|
||||
|
||||
try:
|
||||
abc_text = abc_generator.generate_mix_abc(mix_midi_path, song_title)
|
||||
|
||||
if not abc_text:
|
||||
raise RuntimeError("No ABC output produced.")
|
||||
|
||||
dest = output_dir / "mix.abc"
|
||||
dest.write_text(abc_text, encoding="utf-8")
|
||||
|
||||
files = [{"name": "mix.abc", "path": str(dest)}]
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.DONE,
|
||||
progress="ABC conversion complete.",
|
||||
files=files,
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
"""
|
||||
Apollo separator - Vocal enhancement and reverb removal
|
||||
Models from MVSEP-Central-Backup: https://huggingface.co/miercolesv/MVSEP-Central-Backup
|
||||
"""
|
||||
import os
|
||||
import torch
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
from ..config import APOLLO_MODEL_DIR, CPU_SEPARATION
|
||||
|
||||
|
||||
# Model configurations
|
||||
APOLLO_MODELS = {
|
||||
"apollo_vocals": {
|
||||
"model_path": "epoch=54-val_loss=-17.6221.ckpt",
|
||||
"config_path": "config_apollo_vocals_ep_54.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "Apollo-Vocal-MSST",
|
||||
"stems": ["dry_vocals", "reverb"],
|
||||
"description": "Apollo vocal enhancement/reverb removal",
|
||||
},
|
||||
"apollo_edm": {
|
||||
"model_path": "apollo_edm_by_essid.ckpt",
|
||||
"config_path": "apollo_edm_by_essid.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "Politrees/UVR_resources/models/Apollo",
|
||||
"stems": ["enhanced", "residual"],
|
||||
"description": "Apollo EDM enhancement",
|
||||
},
|
||||
"apollo_edm_big": {
|
||||
"model_path": "apollo_edm_big_by_essid.ckpt",
|
||||
"config_path": "apollo_edm_big_by_essid.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "Politrees/UVR_resources/models/Apollo",
|
||||
"stems": ["enhanced", "residual"],
|
||||
"description": "Apollo EDM Big enhancement",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class ApolloSeparator:
|
||||
"""
|
||||
Apollo separator for vocal enhancement and reverb removal.
|
||||
|
||||
Apollo models separate vocals into dry vocals and reverb,
|
||||
useful for cleaning up vocal recordings.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_id: str = "apollo_vocals",
|
||||
cpu_separation: bool = CPU_SEPARATION,
|
||||
output_format: str = "mp3",
|
||||
mp3_bitrate: str = "256k",
|
||||
):
|
||||
if model_id not in APOLLO_MODELS:
|
||||
raise ValueError(f"Unknown Apollo model: {model_id}")
|
||||
|
||||
self.model_id = model_id
|
||||
self.model_config = APOLLO_MODELS[model_id]
|
||||
self.cpu_separation = cpu_separation
|
||||
self.output_format = output_format
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self._model = None
|
||||
|
||||
def _ensure_model(self):
|
||||
"""Download model from HuggingFace if not present."""
|
||||
model_dir = APOLLO_MODEL_DIR / self.model_id
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
model_path = model_dir / self.model_config["model_path"]
|
||||
config_path = model_dir / self.model_config["config_path"]
|
||||
|
||||
if not model_path.exists() or not config_path.exists():
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
print(f"Downloading Apollo model: {self.model_id}")
|
||||
|
||||
if not model_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.model_config["hf_repo"],
|
||||
filename=f"{self.model_config['hf_subfolder']}/{self.model_config['model_path']}",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
if not config_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.model_config["hf_repo"],
|
||||
filename=f"{self.model_config['hf_subfolder']}/{self.model_config['config_path']}",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to download Apollo model: {e}")
|
||||
|
||||
return str(model_path), str(config_path)
|
||||
|
||||
def _load_model(self):
|
||||
"""Lazy-load the model."""
|
||||
if self._model is None:
|
||||
model_path, config_path = self._ensure_model()
|
||||
self._model = None # Placeholder for actual model loading
|
||||
return self._model
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
||||
"""
|
||||
Separate vocals into dry vocals and reverb.
|
||||
|
||||
Returns:
|
||||
Dict mapping stem names to output file paths
|
||||
"""
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# For now, use audio-separator as fallback
|
||||
# Full implementation would load the actual Apollo model
|
||||
try:
|
||||
from audio_separator.separator import Separator
|
||||
|
||||
sep = Separator(
|
||||
model_file_dir=str(APOLLO_MODEL_DIR),
|
||||
output_dir=str(output_path),
|
||||
output_format=self.output_format.upper(),
|
||||
output_bitrate=self.mp3_bitrate if self.output_format == "mp3" else None,
|
||||
)
|
||||
|
||||
# Use dereverb model as approximation
|
||||
sep.load_model()
|
||||
result = sep.separate(input_path)
|
||||
|
||||
stems = self.model_config.get("stems", ["dry_vocals", "reverb"])
|
||||
output_paths = {}
|
||||
|
||||
if isinstance(result, list):
|
||||
for i, path in enumerate(result):
|
||||
if i < len(stems):
|
||||
stem_name = stems[i]
|
||||
final_path = output_path / f"{stem_name}.{self.output_format}"
|
||||
Path(path).rename(final_path)
|
||||
output_paths[stem_name] = str(final_path)
|
||||
else:
|
||||
final_path = output_path / f"dry_vocals.{self.output_format}"
|
||||
Path(result).rename(final_path)
|
||||
output_paths["dry_vocals"] = str(final_path)
|
||||
|
||||
return output_paths
|
||||
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"audio-separator is required for Apollo separation. "
|
||||
"Install with: pip install audio-separator"
|
||||
)
|
||||
+144
@@ -0,0 +1,144 @@
|
||||
"""
|
||||
Aspiration separator - Breath/aspiration removal from vocals
|
||||
Models from MVSEP-Central-Backup: https://huggingface.co/miercolesv/MVSEP-Central-Backup
|
||||
"""
|
||||
import os
|
||||
import torch
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
from ..config import ASPIRATION_MODEL_DIR, CPU_SEPARATION
|
||||
|
||||
|
||||
# Model configurations
|
||||
ASPIRATION_MODELS = {
|
||||
"aspiration_full": {
|
||||
"model_path": "aspiration_mel_band_roformer_sdr_18.9845.ckpt",
|
||||
"config_path": "config_aspiration_mel_band_roformer.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "Aspiration-MelBand-Sucial",
|
||||
"stems": ["clean_vocals", "aspiration"],
|
||||
"aggression": "full",
|
||||
},
|
||||
"aspiration_light": {
|
||||
"model_path": "aspiration_mel_band_roformer_less_aggr_sdr_18.1201.ckpt",
|
||||
"config_path": "config_aspiration_mel_band_roformer.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "Aspiration-MelBand-Sucial",
|
||||
"stems": ["clean_vocals", "aspiration"],
|
||||
"aggression": "light",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class AspirationSeparator:
|
||||
"""
|
||||
Aspiration separator for breath/aspiration removal from vocals.
|
||||
|
||||
Useful for cleaning vocal recordings by separating breath sounds
|
||||
from the main vocal performance.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_id: str = "aspiration_full",
|
||||
cpu_separation: bool = CPU_SEPARATION,
|
||||
output_format: str = "mp3",
|
||||
mp3_bitrate: str = "256k",
|
||||
):
|
||||
if model_id not in ASPIRATION_MODELS:
|
||||
raise ValueError(f"Unknown Aspiration model: {model_id}")
|
||||
|
||||
self.model_id = model_id
|
||||
self.model_config = ASPIRATION_MODELS[model_id]
|
||||
self.cpu_separation = cpu_separation
|
||||
self.output_format = output_format
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self._model = None
|
||||
|
||||
def _ensure_model(self):
|
||||
"""Download model from HuggingFace if not present."""
|
||||
model_dir = ASPIRATION_MODEL_DIR / self.model_id
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
model_path = model_dir / self.model_config["model_path"]
|
||||
config_path = model_dir / self.model_config["config_path"]
|
||||
|
||||
if not model_path.exists() or not config_path.exists():
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
print(f"Downloading Aspiration model: {self.model_id}")
|
||||
|
||||
if not model_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.model_config["hf_repo"],
|
||||
filename=f"{self.model_config['hf_subfolder']}/{self.model_config['model_path']}",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
if not config_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.model_config["hf_repo"],
|
||||
filename=f"{self.model_config['hf_subfolder']}/{self.model_config['config_path']}",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to download Aspiration model: {e}")
|
||||
|
||||
return str(model_path), str(config_path)
|
||||
|
||||
def _load_model(self):
|
||||
"""Lazy-load the model."""
|
||||
if self._model is None:
|
||||
model_path, config_path = self._ensure_model()
|
||||
self._model = None # Placeholder for actual model loading
|
||||
return self._model
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
||||
"""
|
||||
Separate vocals into clean vocals and aspiration (breath).
|
||||
|
||||
Returns:
|
||||
Dict mapping stem names to output file paths
|
||||
"""
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
from audio_separator.separator import Separator
|
||||
|
||||
sep = Separator(
|
||||
model_file_dir=str(ASPIRATION_MODEL_DIR),
|
||||
output_dir=str(output_path),
|
||||
output_format=self.output_format.upper(),
|
||||
output_bitrate=self.mp3_bitrate if self.output_format == "mp3" else None,
|
||||
)
|
||||
|
||||
sep.load_model()
|
||||
result = sep.separate(input_path)
|
||||
|
||||
stems = self.model_config.get("stems", ["clean_vocals", "aspiration"])
|
||||
output_paths = {}
|
||||
|
||||
if isinstance(result, list):
|
||||
for i, path in enumerate(result):
|
||||
if i < len(stems):
|
||||
stem_name = stems[i]
|
||||
final_path = output_path / f"{stem_name}.{self.output_format}"
|
||||
Path(path).rename(final_path)
|
||||
output_paths[stem_name] = str(final_path)
|
||||
else:
|
||||
final_path = output_path / f"clean_vocals.{self.output_format}"
|
||||
Path(result).rename(final_path)
|
||||
output_paths["clean_vocals"] = str(final_path)
|
||||
|
||||
return output_paths
|
||||
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"audio-separator is required for Aspiration separation. "
|
||||
"Install with: pip install audio-separator"
|
||||
)
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
"""
|
||||
Bandit-v2 separator - Multi-language/multi-stem separation
|
||||
Models from MVSEP-Central-Backup: https://huggingface.co/miercolesv/MVSEP-Central-Backup
|
||||
"""
|
||||
import os
|
||||
import torch
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
from ..config import BANDIT_MODEL_DIR, CPU_SEPARATION
|
||||
|
||||
|
||||
# Model configurations
|
||||
BANDIT_MODELS = {
|
||||
"bandit_v2_multi": {
|
||||
"model_path": "checkpoint-multi_state_dict.ckpt",
|
||||
"config_path": "config_dnr_bandit_v2_mus64.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "Bandit-v2",
|
||||
"stems": ["vocals", "drums", "bass", "other"],
|
||||
"language": "multi",
|
||||
},
|
||||
"bandit_v2_eng": {
|
||||
"model_path": "checkpoint-eng_state_dict.ckpt",
|
||||
"config_path": "config_dnr_bandit_v2_mus64.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "Bandit-v2",
|
||||
"stems": ["vocals", "drums", "bass", "other"],
|
||||
"language": "english",
|
||||
},
|
||||
# Additional language models available:
|
||||
# - checkpoint-cmn_state_dict.ckpt (Chinese)
|
||||
# - checkpoint-deu_state_dict.ckpt (German)
|
||||
# - checkpoint-fao_state_dict.ckpt (Faroese)
|
||||
# - checkpoint-fra_state_dict.ckpt (French)
|
||||
# - checkpoint-spa_state_dict.ckpt (Spanish)
|
||||
}
|
||||
|
||||
|
||||
class BanditSeparator:
|
||||
"""
|
||||
Bandit-v2 separator for multi-language/multi-stem separation.
|
||||
|
||||
Bandit models are trained on different languages and provide
|
||||
high-quality separation optimized for specific language characteristics.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_id: str = "bandit_v2_multi",
|
||||
cpu_separation: bool = CPU_SEPARATION,
|
||||
output_format: str = "mp3",
|
||||
mp3_bitrate: str = "256k",
|
||||
):
|
||||
if model_id not in BANDIT_MODELS:
|
||||
raise ValueError(f"Unknown Bandit model: {model_id}")
|
||||
|
||||
self.model_id = model_id
|
||||
self.model_config = BANDIT_MODELS[model_id]
|
||||
self.cpu_separation = cpu_separation
|
||||
self.output_format = output_format
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self._model = None
|
||||
|
||||
def _ensure_model(self):
|
||||
"""Download model from HuggingFace if not present."""
|
||||
model_dir = BANDIT_MODEL_DIR / self.model_id
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
model_path = model_dir / self.model_config["model_path"]
|
||||
config_path = model_dir / self.model_config["config_path"]
|
||||
|
||||
if not model_path.exists() or not config_path.exists():
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
print(f"Downloading Bandit model: {self.model_id}")
|
||||
|
||||
if not model_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.model_config["hf_repo"],
|
||||
filename=f"{self.model_config['hf_subfolder']}/{self.model_config['model_path']}",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
if not config_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.model_config["hf_repo"],
|
||||
filename=f"{self.model_config['hf_subfolder']}/{self.model_config['config_path']}",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to download Bandit model: {e}")
|
||||
|
||||
return str(model_path), str(config_path)
|
||||
|
||||
def _load_model(self):
|
||||
"""Lazy-load the model."""
|
||||
if self._model is None:
|
||||
model_path, config_path = self._ensure_model()
|
||||
self._model = None # Placeholder for actual model loading
|
||||
return self._model
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
||||
"""
|
||||
Separate audio into stems using Bandit-v2.
|
||||
|
||||
Returns:
|
||||
Dict mapping stem names to output file paths
|
||||
"""
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
from audio_separator.separator import Separator
|
||||
|
||||
sep = Separator(
|
||||
model_file_dir=str(BANDIT_MODEL_DIR),
|
||||
output_dir=str(output_path),
|
||||
output_format=self.output_format.upper(),
|
||||
output_bitrate=self.mp3_bitrate if self.output_format == "mp3" else None,
|
||||
)
|
||||
|
||||
sep.load_model()
|
||||
result = sep.separate(input_path)
|
||||
|
||||
stems = self.model_config.get("stems", ["vocals", "drums", "bass", "other"])
|
||||
output_paths = {}
|
||||
|
||||
if isinstance(result, list):
|
||||
for i, path in enumerate(result):
|
||||
if i < len(stems):
|
||||
stem_name = stems[i]
|
||||
final_path = output_path / f"{stem_name}.{self.output_format}"
|
||||
Path(path).rename(final_path)
|
||||
output_paths[stem_name] = str(final_path)
|
||||
else:
|
||||
final_path = output_path / f"vocals.{self.output_format}"
|
||||
Path(result).rename(final_path)
|
||||
output_paths["vocals"] = str(final_path)
|
||||
|
||||
return output_paths
|
||||
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"audio-separator is required for Bandit separation. "
|
||||
"Install with: pip install audio-separator"
|
||||
)
|
||||
@@ -0,0 +1,3 @@
|
||||
from .bs_roformer import BSRoformer
|
||||
|
||||
__all__ = ['BSRoformer']
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
from functools import wraps
|
||||
from packaging import version
|
||||
from collections import namedtuple
|
||||
|
||||
import torch
|
||||
from torch import nn, einsum
|
||||
import torch.nn.functional as F
|
||||
|
||||
from einops import rearrange, reduce
|
||||
|
||||
"""
|
||||
Code adapted from https://github.com/Anjok07/ultimatevocalremovergui/blob/v5.6.0_roformer_add%2Bdirectml/lib_v5/bs_roformer.py and https://github.com/lucidrains/BS-RoFormer which are copyrighted under the terms of the MIT license.
|
||||
"""
|
||||
|
||||
# constants
|
||||
|
||||
FlashAttentionConfig = namedtuple('FlashAttentionConfig', ['enable_flash', 'enable_math', 'enable_mem_efficient'])
|
||||
|
||||
# helpers
|
||||
|
||||
def exists(val):
|
||||
return val is not None
|
||||
|
||||
def once(fn):
|
||||
called = False
|
||||
@wraps(fn)
|
||||
def inner(x):
|
||||
nonlocal called
|
||||
if called:
|
||||
return
|
||||
called = True
|
||||
return fn(x)
|
||||
return inner
|
||||
|
||||
print_once = once(print)
|
||||
|
||||
# main class
|
||||
|
||||
class Attend(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
dropout = 0.,
|
||||
flash = False,
|
||||
scale = None
|
||||
):
|
||||
super().__init__()
|
||||
self.dropout = dropout
|
||||
self.attn_dropout = nn.Dropout(dropout)
|
||||
self.scale = scale
|
||||
|
||||
self.flash = flash
|
||||
assert not (flash and version.parse(torch.__version__) < version.parse('2.0.0')), 'in order to use flash attention, you must be using pytorch 2.0 or above'
|
||||
|
||||
# determine efficient attention configs for cuda and cpu
|
||||
|
||||
self.cpu_config = FlashAttentionConfig(True, True, True)
|
||||
self.cuda_config = None
|
||||
|
||||
if not torch.cuda.is_available() or not flash:
|
||||
return
|
||||
|
||||
device_properties = torch.cuda.get_device_properties(torch.device('cuda'))
|
||||
|
||||
if device_properties.major == 8 and device_properties.minor == 0:
|
||||
print_once('A100 GPU detected, using flash attention if input tensor is on cuda')
|
||||
self.cuda_config = FlashAttentionConfig(True, False, False)
|
||||
else:
|
||||
self.cuda_config = FlashAttentionConfig(False, True, True)
|
||||
|
||||
def flash_attn(self, q, k, v):
|
||||
_, heads, q_len, _, k_len, is_cuda, device = *q.shape, k.shape[-2], q.is_cuda, q.device
|
||||
|
||||
# Check if there is a compatible device for flash attention
|
||||
|
||||
config = self.cuda_config if is_cuda else self.cpu_config
|
||||
|
||||
# pytorch 2.0 flash attn: q, k, v, mask, dropout, softmax_scale
|
||||
|
||||
with torch.backends.cuda.sdp_kernel(**config._asdict()):
|
||||
out = F.scaled_dot_product_attention(
|
||||
q, k, v,
|
||||
dropout_p = self.dropout if self.training else 0.
|
||||
)
|
||||
|
||||
return out
|
||||
|
||||
def forward(self, q, k, v):
|
||||
"""
|
||||
einstein notation
|
||||
b - batch
|
||||
h - heads
|
||||
n, i, j - sequence length (base sequence length, source, target)
|
||||
d - feature dimension
|
||||
"""
|
||||
|
||||
q_len, k_len, device = q.shape[-2], k.shape[-2], q.device
|
||||
|
||||
scale = self.scale if self.scale is not None else q.shape[-1] ** -0.5
|
||||
|
||||
if self.flash:
|
||||
return self.flash_attn(q, k, v)
|
||||
|
||||
# similarity
|
||||
|
||||
sim = einsum(f"b h i d, b h j d -> b h i j", q, k) * scale
|
||||
|
||||
# attention
|
||||
|
||||
attn = sim.softmax(dim=-1)
|
||||
attn = self.attn_dropout(attn)
|
||||
|
||||
# aggregate values
|
||||
|
||||
out = einsum(f"b h i j, b h j d -> b h i d", attn, v)
|
||||
|
||||
return out
|
||||
+604
@@ -0,0 +1,604 @@
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
from torch import nn, einsum, Tensor
|
||||
from torch.nn import Module, ModuleList
|
||||
import torch.nn.functional as F
|
||||
|
||||
from .attend import Attend
|
||||
|
||||
from beartype.typing import Tuple, Optional, List, Callable
|
||||
from beartype import beartype
|
||||
|
||||
from rotary_embedding_torch import RotaryEmbedding
|
||||
|
||||
from einops import rearrange, pack, unpack
|
||||
from einops.layers.torch import Rearrange
|
||||
|
||||
"""
|
||||
Code adapted from https://github.com/Anjok07/ultimatevocalremovergui/blob/v5.6.0_roformer_add%2Bdirectml/lib_v5/bs_roformer.py and https://github.com/lucidrains/BS-RoFormer which are copyrighted under the terms of the MIT license.
|
||||
"""
|
||||
|
||||
# helper functions
|
||||
|
||||
def exists(val):
|
||||
return val is not None
|
||||
|
||||
|
||||
def default(v, d):
|
||||
return v if exists(v) else d
|
||||
|
||||
|
||||
def pack_one(t, pattern):
|
||||
return pack([t], pattern)
|
||||
|
||||
|
||||
def unpack_one(t, ps, pattern):
|
||||
return unpack(t, ps, pattern)[0]
|
||||
|
||||
|
||||
# norm
|
||||
|
||||
def l2norm(t):
|
||||
return F.normalize(t, dim = -1, p = 2)
|
||||
|
||||
|
||||
class RMSNorm(Module):
|
||||
def __init__(self, dim):
|
||||
super().__init__()
|
||||
self.scale = dim ** 0.5
|
||||
self.gamma = nn.Parameter(torch.ones(dim))
|
||||
|
||||
def forward(self, x):
|
||||
x = x.to(self.gamma.device)
|
||||
return F.normalize(x, dim=-1) * self.scale * self.gamma
|
||||
|
||||
|
||||
# attention
|
||||
|
||||
class FeedForward(Module):
|
||||
def __init__(
|
||||
self,
|
||||
dim,
|
||||
mult=4,
|
||||
dropout=0.
|
||||
):
|
||||
super().__init__()
|
||||
dim_inner = int(dim * mult)
|
||||
self.net = nn.Sequential(
|
||||
RMSNorm(dim),
|
||||
nn.Linear(dim, dim_inner),
|
||||
nn.GELU(),
|
||||
nn.Dropout(dropout),
|
||||
nn.Linear(dim_inner, dim),
|
||||
nn.Dropout(dropout)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
return self.net(x)
|
||||
|
||||
|
||||
class Attention(Module):
|
||||
def __init__(
|
||||
self,
|
||||
dim,
|
||||
heads=8,
|
||||
dim_head=64,
|
||||
dropout=0.,
|
||||
rotary_embed=None,
|
||||
flash=True
|
||||
):
|
||||
super().__init__()
|
||||
self.heads = heads
|
||||
self.scale = dim_head ** -0.5
|
||||
dim_inner = heads * dim_head
|
||||
|
||||
self.rotary_embed = rotary_embed
|
||||
|
||||
self.attend = Attend(flash=flash, dropout=dropout)
|
||||
|
||||
self.norm = RMSNorm(dim)
|
||||
self.to_qkv = nn.Linear(dim, dim_inner * 3, bias=False)
|
||||
|
||||
self.to_gates = nn.Linear(dim, heads)
|
||||
|
||||
self.to_out = nn.Sequential(
|
||||
nn.Linear(dim_inner, dim, bias=False),
|
||||
nn.Dropout(dropout)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.norm(x)
|
||||
|
||||
q, k, v = rearrange(self.to_qkv(x), 'b n (qkv h d) -> qkv b h n d', qkv=3, h=self.heads)
|
||||
|
||||
if exists(self.rotary_embed):
|
||||
q = self.rotary_embed.rotate_queries_or_keys(q)
|
||||
k = self.rotary_embed.rotate_queries_or_keys(k)
|
||||
|
||||
out = self.attend(q, k, v)
|
||||
|
||||
gates = self.to_gates(x)
|
||||
out = out * rearrange(gates, 'b n h -> b h n 1').sigmoid()
|
||||
|
||||
out = rearrange(out, 'b h n d -> b n (h d)')
|
||||
return self.to_out(out)
|
||||
|
||||
|
||||
class LinearAttention(Module):
|
||||
"""
|
||||
this flavor of linear attention proposed in https://arxiv.org/abs/2106.09681 by El-Nouby et al.
|
||||
"""
|
||||
|
||||
@beartype
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
dim,
|
||||
dim_head=32,
|
||||
heads=8,
|
||||
scale=8,
|
||||
flash=False,
|
||||
dropout=0.
|
||||
):
|
||||
super().__init__()
|
||||
dim_inner = dim_head * heads
|
||||
self.norm = RMSNorm(dim)
|
||||
|
||||
self.to_qkv = nn.Sequential(
|
||||
nn.Linear(dim, dim_inner * 3, bias=False),
|
||||
Rearrange('b n (qkv h d) -> qkv b h d n', qkv=3, h=heads)
|
||||
)
|
||||
|
||||
self.temperature = nn.Parameter(torch.ones(heads, 1, 1))
|
||||
|
||||
self.attend = Attend(
|
||||
scale=scale,
|
||||
dropout=dropout,
|
||||
flash=flash
|
||||
)
|
||||
|
||||
self.to_out = nn.Sequential(
|
||||
Rearrange('b h d n -> b n (h d)'),
|
||||
nn.Linear(dim_inner, dim, bias=False)
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x
|
||||
):
|
||||
x = self.norm(x)
|
||||
|
||||
q, k, v = self.to_qkv(x)
|
||||
|
||||
q, k = map(l2norm, (q, k))
|
||||
q = q * self.temperature.exp()
|
||||
|
||||
out = self.attend(q, k, v)
|
||||
|
||||
return self.to_out(out)
|
||||
|
||||
|
||||
class Transformer(Module):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
dim,
|
||||
depth,
|
||||
dim_head=64,
|
||||
heads=8,
|
||||
attn_dropout=0.,
|
||||
ff_dropout=0.,
|
||||
ff_mult=4,
|
||||
norm_output=True,
|
||||
rotary_embed=None,
|
||||
flash_attn=True,
|
||||
linear_attn=False
|
||||
):
|
||||
super().__init__()
|
||||
self.layers = ModuleList([])
|
||||
|
||||
for _ in range(depth):
|
||||
if linear_attn:
|
||||
attn = LinearAttention(dim=dim, dim_head=dim_head, heads=heads, dropout=attn_dropout, flash=flash_attn)
|
||||
else:
|
||||
attn = Attention(dim=dim, dim_head=dim_head, heads=heads, dropout=attn_dropout,
|
||||
rotary_embed=rotary_embed, flash=flash_attn)
|
||||
|
||||
self.layers.append(ModuleList([
|
||||
attn,
|
||||
FeedForward(dim=dim, mult=ff_mult, dropout=ff_dropout)
|
||||
]))
|
||||
|
||||
self.norm = RMSNorm(dim) if norm_output else nn.Identity()
|
||||
|
||||
def forward(self, x):
|
||||
|
||||
for attn, ff in self.layers:
|
||||
x = attn(x) + x
|
||||
x = ff(x) + x
|
||||
|
||||
return self.norm(x)
|
||||
|
||||
|
||||
# bandsplit module
|
||||
|
||||
class BandSplit(Module):
|
||||
@beartype
|
||||
def __init__(
|
||||
self,
|
||||
dim,
|
||||
dim_inputs: Tuple[int, ...]
|
||||
):
|
||||
super().__init__()
|
||||
self.dim_inputs = dim_inputs
|
||||
self.to_features = ModuleList([])
|
||||
|
||||
for dim_in in dim_inputs:
|
||||
net = nn.Sequential(
|
||||
RMSNorm(dim_in),
|
||||
nn.Linear(dim_in, dim)
|
||||
)
|
||||
|
||||
self.to_features.append(net)
|
||||
|
||||
def forward(self, x):
|
||||
x = x.split(self.dim_inputs, dim=-1)
|
||||
|
||||
outs = []
|
||||
for split_input, to_feature in zip(x, self.to_features):
|
||||
split_output = to_feature(split_input)
|
||||
outs.append(split_output)
|
||||
|
||||
return torch.stack(outs, dim=-2)
|
||||
|
||||
|
||||
def MLP(
|
||||
dim_in,
|
||||
dim_out,
|
||||
dim_hidden=None,
|
||||
depth=1,
|
||||
activation=nn.Tanh
|
||||
):
|
||||
dim_hidden = default(dim_hidden, dim_in)
|
||||
|
||||
net = []
|
||||
dims = (dim_in, *((dim_hidden,) * (depth - 1)), dim_out)
|
||||
|
||||
for ind, (layer_dim_in, layer_dim_out) in enumerate(zip(dims[:-1], dims[1:])):
|
||||
is_last = ind == (len(dims) - 2)
|
||||
|
||||
net.append(nn.Linear(layer_dim_in, layer_dim_out))
|
||||
|
||||
if is_last:
|
||||
continue
|
||||
|
||||
net.append(activation())
|
||||
|
||||
return nn.Sequential(*net)
|
||||
|
||||
|
||||
class MaskEstimator(Module):
|
||||
@beartype
|
||||
def __init__(
|
||||
self,
|
||||
dim,
|
||||
dim_inputs: Tuple[int, ...],
|
||||
depth,
|
||||
mlp_expansion_factor=4
|
||||
):
|
||||
super().__init__()
|
||||
self.dim_inputs = dim_inputs
|
||||
self.to_freqs = ModuleList([])
|
||||
dim_hidden = dim * mlp_expansion_factor
|
||||
|
||||
for dim_in in dim_inputs:
|
||||
net = []
|
||||
|
||||
mlp = nn.Sequential(
|
||||
MLP(dim, dim_in * 2, dim_hidden=dim_hidden, depth=depth),
|
||||
nn.GLU(dim=-1)
|
||||
)
|
||||
|
||||
self.to_freqs.append(mlp)
|
||||
|
||||
def forward(self, x):
|
||||
x = x.unbind(dim=-2)
|
||||
|
||||
outs = []
|
||||
|
||||
for band_features, mlp in zip(x, self.to_freqs):
|
||||
freq_out = mlp(band_features)
|
||||
outs.append(freq_out)
|
||||
|
||||
return torch.cat(outs, dim=-1)
|
||||
|
||||
|
||||
# main class
|
||||
|
||||
DEFAULT_FREQS_PER_BANDS = (
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2,
|
||||
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
|
||||
12, 12, 12, 12, 12, 12, 12, 12,
|
||||
24, 24, 24, 24, 24, 24, 24, 24,
|
||||
48, 48, 48, 48, 48, 48, 48, 48,
|
||||
128, 129,
|
||||
)
|
||||
|
||||
|
||||
class BSRoformer(Module):
|
||||
|
||||
@beartype
|
||||
def __init__(
|
||||
self,
|
||||
dim,
|
||||
*,
|
||||
depth,
|
||||
stereo=False,
|
||||
num_stems=1,
|
||||
time_transformer_depth=2,
|
||||
freq_transformer_depth=2,
|
||||
linear_transformer_depth=0,
|
||||
freqs_per_bands: Tuple[int, ...] = DEFAULT_FREQS_PER_BANDS,
|
||||
# in the paper, they divide into ~60 bands, test with 1 for starters
|
||||
dim_head=64,
|
||||
heads=8,
|
||||
attn_dropout=0.,
|
||||
ff_dropout=0.,
|
||||
flash_attn=True,
|
||||
dim_freqs_in=1025,
|
||||
stft_n_fft=2048,
|
||||
stft_hop_length=512,
|
||||
# 10ms at 44100Hz, from sections 4.1, 4.4 in the paper - @faroit recommends // 2 or // 4 for better reconstruction
|
||||
stft_win_length=2048,
|
||||
stft_normalized=False,
|
||||
stft_window_fn: Optional[Callable] = None,
|
||||
mask_estimator_depth=2,
|
||||
multi_stft_resolution_loss_weight=1.,
|
||||
multi_stft_resolutions_window_sizes: Tuple[int, ...] = (4096, 2048, 1024, 512, 256),
|
||||
multi_stft_hop_size=147,
|
||||
multi_stft_normalized=False,
|
||||
multi_stft_window_fn: Callable = torch.hann_window,
|
||||
# Extra kwargs to ignore from config
|
||||
**kwargs
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.stereo = stereo
|
||||
self.audio_channels = 2 if stereo else 1
|
||||
self.num_stems = num_stems
|
||||
|
||||
self.layers = ModuleList([])
|
||||
|
||||
transformer_kwargs = dict(
|
||||
dim=dim,
|
||||
heads=heads,
|
||||
dim_head=dim_head,
|
||||
attn_dropout=attn_dropout,
|
||||
ff_dropout=ff_dropout,
|
||||
flash_attn=flash_attn,
|
||||
norm_output=False
|
||||
)
|
||||
|
||||
time_rotary_embed = RotaryEmbedding(dim=dim_head)
|
||||
freq_rotary_embed = RotaryEmbedding(dim=dim_head)
|
||||
|
||||
for _ in range(depth):
|
||||
tran_modules = []
|
||||
if linear_transformer_depth > 0:
|
||||
tran_modules.append(Transformer(depth=linear_transformer_depth, linear_attn=True, **transformer_kwargs))
|
||||
tran_modules.append(
|
||||
Transformer(depth=time_transformer_depth, rotary_embed=time_rotary_embed, **transformer_kwargs)
|
||||
)
|
||||
tran_modules.append(
|
||||
Transformer(depth=freq_transformer_depth, rotary_embed=freq_rotary_embed, **transformer_kwargs)
|
||||
)
|
||||
self.layers.append(nn.ModuleList(tran_modules))
|
||||
|
||||
self.final_norm = RMSNorm(dim)
|
||||
|
||||
self.stft_kwargs = dict(
|
||||
n_fft=stft_n_fft,
|
||||
hop_length=stft_hop_length,
|
||||
win_length=stft_win_length,
|
||||
normalized=stft_normalized
|
||||
)
|
||||
|
||||
self.stft_window_fn = partial(default(stft_window_fn, torch.hann_window), stft_win_length)
|
||||
|
||||
freqs = torch.stft(torch.randn(1, 4096), **self.stft_kwargs, return_complex=True).shape[1]
|
||||
|
||||
assert len(freqs_per_bands) > 1
|
||||
assert sum(
|
||||
freqs_per_bands) == freqs, f'the number of freqs in the bands must equal {freqs} based on the STFT settings, but got {sum(freqs_per_bands)}'
|
||||
|
||||
freqs_per_bands_with_complex = tuple(2 * f * self.audio_channels for f in freqs_per_bands)
|
||||
|
||||
self.band_split = BandSplit(
|
||||
dim=dim,
|
||||
dim_inputs=freqs_per_bands_with_complex
|
||||
)
|
||||
|
||||
self.mask_estimators = nn.ModuleList([])
|
||||
|
||||
for _ in range(num_stems):
|
||||
mask_estimator = MaskEstimator(
|
||||
dim=dim,
|
||||
dim_inputs=freqs_per_bands_with_complex,
|
||||
depth=mask_estimator_depth
|
||||
)
|
||||
|
||||
self.mask_estimators.append(mask_estimator)
|
||||
|
||||
# for the multi-resolution stft loss
|
||||
|
||||
self.multi_stft_resolution_loss_weight = multi_stft_resolution_loss_weight
|
||||
self.multi_stft_resolutions_window_sizes = multi_stft_resolutions_window_sizes
|
||||
self.multi_stft_n_fft = stft_n_fft
|
||||
self.multi_stft_window_fn = multi_stft_window_fn
|
||||
|
||||
self.multi_stft_kwargs = dict(
|
||||
hop_length=multi_stft_hop_size,
|
||||
normalized=multi_stft_normalized
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
raw_audio,
|
||||
target=None,
|
||||
return_loss_breakdown=False
|
||||
):
|
||||
"""
|
||||
einops
|
||||
|
||||
b - batch
|
||||
f - freq
|
||||
t - time
|
||||
s - audio channel (1 for mono, 2 for stereo)
|
||||
n - number of 'stems'
|
||||
c - complex (2)
|
||||
d - feature dimension
|
||||
"""
|
||||
|
||||
original_device = raw_audio.device
|
||||
|
||||
x_is_mps = True if original_device.type == 'mps' else False
|
||||
|
||||
if x_is_mps:
|
||||
raw_audio = raw_audio.cpu()
|
||||
|
||||
device = raw_audio.device
|
||||
|
||||
if raw_audio.ndim == 2:
|
||||
raw_audio = rearrange(raw_audio, 'b t -> b 1 t')
|
||||
|
||||
channels = raw_audio.shape[1]
|
||||
assert (not self.stereo and channels == 1) or (
|
||||
self.stereo and channels == 2), 'stereo needs to be set to True if passing in audio signal that is stereo (channel dimension of 2). also need to be False if mono (channel dimension of 1)'
|
||||
|
||||
# to stft
|
||||
|
||||
raw_audio, batch_audio_channel_packed_shape = pack_one(raw_audio, '* t')
|
||||
|
||||
stft_window = self.stft_window_fn(device=device)
|
||||
|
||||
stft_repr = torch.stft(raw_audio, **self.stft_kwargs, window=stft_window, return_complex=True)
|
||||
stft_repr = torch.view_as_real(stft_repr)
|
||||
|
||||
stft_repr = unpack_one(stft_repr, batch_audio_channel_packed_shape, '* f t c')
|
||||
stft_repr = rearrange(stft_repr,
|
||||
'b s f t c -> b (f s) t c') # merge stereo / mono into the frequency, with frequency leading dimension, for band splitting
|
||||
|
||||
x = rearrange(stft_repr, 'b f t c -> b t (f c)')
|
||||
|
||||
x = self.band_split(x)
|
||||
|
||||
# axial / hierarchical attention
|
||||
|
||||
for transformer_block in self.layers:
|
||||
|
||||
if len(transformer_block) == 3:
|
||||
linear_transformer, time_transformer, freq_transformer = transformer_block
|
||||
|
||||
x, ft_ps = pack([x], 'b * d')
|
||||
x = linear_transformer(x)
|
||||
x, = unpack(x, ft_ps, 'b * d')
|
||||
else:
|
||||
time_transformer, freq_transformer = transformer_block
|
||||
|
||||
x = rearrange(x, 'b t f d -> b f t d')
|
||||
x, ps = pack([x], '* t d')
|
||||
|
||||
x = time_transformer(x)
|
||||
|
||||
x, = unpack(x, ps, '* t d')
|
||||
x = rearrange(x, 'b f t d -> b t f d')
|
||||
x, ps = pack([x], '* f d')
|
||||
|
||||
x = freq_transformer(x)
|
||||
|
||||
x, = unpack(x, ps, '* f d')
|
||||
|
||||
x = self.final_norm(x)
|
||||
|
||||
num_stems = len(self.mask_estimators)
|
||||
|
||||
mask = torch.stack([fn(x) for fn in self.mask_estimators], dim=1)
|
||||
mask = rearrange(mask, 'b n t (f c) -> b n f t c', c=2)
|
||||
|
||||
if x_is_mps:
|
||||
mask = mask.to('cpu')
|
||||
|
||||
# modulate frequency representation
|
||||
|
||||
stft_repr = rearrange(stft_repr, 'b f t c -> b 1 f t c')
|
||||
|
||||
# complex number multiplication
|
||||
|
||||
stft_repr = torch.view_as_complex(stft_repr)
|
||||
mask = torch.view_as_complex(mask)
|
||||
|
||||
stft_repr = stft_repr * mask
|
||||
|
||||
# istft
|
||||
|
||||
stft_repr = rearrange(stft_repr, 'b n (f s) t -> (b n s) f t', s=self.audio_channels)
|
||||
|
||||
recon_audio = torch.istft(stft_repr, **self.stft_kwargs, window=stft_window, return_complex=False)
|
||||
|
||||
recon_audio = rearrange(recon_audio, '(b n s) t -> b n s t', s=self.audio_channels, n=num_stems)
|
||||
|
||||
if num_stems == 1:
|
||||
recon_audio = rearrange(recon_audio, 'b 1 s t -> b s t')
|
||||
|
||||
# if a target is passed in, calculate loss for learning
|
||||
|
||||
if not exists(target):
|
||||
return recon_audio
|
||||
|
||||
if self.num_stems > 1:
|
||||
assert target.ndim == 4 and target.shape[1] == self.num_stems
|
||||
|
||||
if target.ndim == 2:
|
||||
target = rearrange(target, '... t -> ... 1 t')
|
||||
|
||||
target = target[..., :recon_audio.shape[-1]]
|
||||
|
||||
loss = F.l1_loss(recon_audio, target)
|
||||
|
||||
multi_stft_resolution_loss = 0.
|
||||
|
||||
for window_size in self.multi_stft_resolutions_window_sizes:
|
||||
res_stft_kwargs = dict(
|
||||
n_fft=max(window_size, self.multi_stft_n_fft),
|
||||
win_length=window_size,
|
||||
return_complex=True,
|
||||
window=self.multi_stft_window_fn(window_size, device=device),
|
||||
**self.multi_stft_kwargs,
|
||||
)
|
||||
|
||||
recon_Y = torch.stft(rearrange(recon_audio, '... s t -> (... s) t'), **res_stft_kwargs)
|
||||
target_Y = torch.stft(rearrange(target, '... s t -> (... s) t'), **res_stft_kwargs)
|
||||
|
||||
multi_stft_resolution_loss = multi_stft_resolution_loss + F.l1_loss(recon_Y, target_Y)
|
||||
|
||||
weighted_multi_resolution_loss = multi_stft_resolution_loss * self.multi_stft_resolution_loss_weight
|
||||
|
||||
total_loss = loss + weighted_multi_resolution_loss
|
||||
|
||||
|
||||
if not return_loss_breakdown:
|
||||
# Move the result back to the original device if it was moved to CPU for MPS compatibility
|
||||
if x_is_mps:
|
||||
total_loss = total_loss.to(original_device)
|
||||
return total_loss
|
||||
|
||||
# For detailed loss breakdown, ensure all components are moved back to the original device for MPS
|
||||
if x_is_mps:
|
||||
loss = loss.to(original_device)
|
||||
multi_stft_resolution_loss = multi_stft_resolution_loss.to(original_device)
|
||||
weighted_multi_resolution_loss = weighted_multi_resolution_loss.to(original_device)
|
||||
|
||||
return total_loss, (loss, multi_stft_resolution_loss)
|
||||
@@ -0,0 +1,258 @@
|
||||
"""
|
||||
BS-RoFormer Chorus separator for male/female vocals separation.
|
||||
Uses the Chorus_Male_Female_BS_Roformer model.
|
||||
"""
|
||||
import gc
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import yaml
|
||||
from omegaconf import OmegaConf
|
||||
from tqdm import tqdm
|
||||
|
||||
from .bs_roformer import BSRoformer
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
# HuggingFace model coordinates
|
||||
HF_REPO_ID = 'Sucial/Chorus_Male_Female_BS_Roformer'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
|
||||
def _try_download_model(model_dir: Path, model_path: Path, config_path: Path) -> None:
|
||||
"""Download model from HuggingFace if not present."""
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
from huggingface_hub import snapshot_download
|
||||
snapshot_download(
|
||||
repo_id=HF_REPO_ID,
|
||||
revision=HF_REVISION,
|
||||
local_dir=str(model_dir),
|
||||
)
|
||||
|
||||
|
||||
def _load_audio(path: str) -> np.ndarray:
|
||||
"""Load audio as (channels, samples) float32 array."""
|
||||
import soundfile as sf
|
||||
|
||||
try:
|
||||
data, sr = sf.read(path, dtype='float32', always_2d=True)
|
||||
data = data.T
|
||||
except Exception:
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', path, '-ar', str(SAMPLE_RATE),
|
||||
'-ac', '2', '-f', 'wav', tmp_path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
data, sr = sf.read(tmp_path, dtype='float32', always_2d=True)
|
||||
data = data.T
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
if sr != SAMPLE_RATE:
|
||||
import librosa
|
||||
data = np.array([
|
||||
librosa.resample(ch, orig_sr=sr, target_sr=SAMPLE_RATE)
|
||||
for ch in data
|
||||
], dtype=np.float32)
|
||||
|
||||
if data.shape[0] == 1:
|
||||
data = np.vstack([data, data])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def _save_audio(path: str, data: np.ndarray, output_format: str, bitrate: str = '256k') -> None:
|
||||
"""Save (samples, channels) array to audio file."""
|
||||
import soundfile as sf
|
||||
|
||||
ext = output_format.lower()
|
||||
|
||||
if ext in ('wav', 'flac'):
|
||||
sf.write(path, data, SAMPLE_RATE, format=ext.upper(),
|
||||
subtype='PCM_24' if ext == 'wav' else 'PCM_16')
|
||||
else:
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
sf.write(tmp_path, data, SAMPLE_RATE, format='WAV', subtype='PCM_16')
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', tmp_path, '-b:a', bitrate, path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
|
||||
class BSRoformerChorusSeparator:
|
||||
"""
|
||||
BS-RoFormer separator for male/female chorus separation.
|
||||
Separates male and female voices in choir singing.
|
||||
|
||||
Note: This is an experimental model trained primarily on Chinese songs.
|
||||
Works best when male and female sing simultaneously (not alternately).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
):
|
||||
self.model_dir = Path(model_dir) if model_dir else Path('/opt/bs_roformer_models/chorus')
|
||||
self.model_path = self.model_dir / 'model_chorus_bs_roformer_ep_267_sdr_24.1275.ckpt'
|
||||
self.config_path = self.model_dir / 'config.yaml'
|
||||
self.device = 'cpu' if cpu_separation else 'cuda'
|
||||
self.output_format = output_format.lower()
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self._config = None
|
||||
|
||||
def _ensure_model(self):
|
||||
_try_download_model(self.model_dir, self.model_path, self.config_path)
|
||||
|
||||
def _load_config(self):
|
||||
if self._config is None:
|
||||
with open(self.config_path, 'r') as f:
|
||||
config = yaml.load(f, Loader=yaml.UnsafeLoader)
|
||||
self._config = OmegaConf.create(config)
|
||||
return self._config
|
||||
|
||||
def get_model(self) -> BSRoformer:
|
||||
self._ensure_model()
|
||||
config = self._load_config()
|
||||
model_cfg = OmegaConf.to_container(config.model, resolve=True)
|
||||
|
||||
# Convert lists to tuples where needed
|
||||
for key in ('freqs_per_bands', 'multi_stft_resolutions_window_sizes'):
|
||||
if key in model_cfg and isinstance(model_cfg[key], list):
|
||||
model_cfg[key] = tuple(model_cfg[key])
|
||||
|
||||
model = BSRoformer(**model_cfg)
|
||||
checkpoint = torch.load(str(self.model_path), map_location='cpu', weights_only=False)
|
||||
|
||||
if 'state_dict' in checkpoint:
|
||||
state_dict = checkpoint['state_dict']
|
||||
elif 'model' in checkpoint:
|
||||
state_dict = checkpoint['model']
|
||||
else:
|
||||
state_dict = checkpoint
|
||||
|
||||
model.load_state_dict(state_dict)
|
||||
model.to(self.device).eval()
|
||||
return model
|
||||
|
||||
def _demix(self, mix: np.ndarray, model: BSRoformer,
|
||||
batch_size_override: Optional[int] = None,
|
||||
num_overlap_override: Optional[int] = None) -> Dict[str, np.ndarray]:
|
||||
"""Separate into male and female vocals."""
|
||||
config = self._load_config()
|
||||
hop_length = config.model.stft_hop_length
|
||||
segment_size = getattr(config.inference, 'dim_t', 2000)
|
||||
C = hop_length * (segment_size - 1)
|
||||
N = num_overlap_override if num_overlap_override is not None else getattr(config.inference, 'num_overlap', 4)
|
||||
step = int(C // N)
|
||||
fade_size = C // 10
|
||||
batch_size = batch_size_override if batch_size_override is not None else getattr(config.inference, 'batch_size', 1)
|
||||
device = self.device
|
||||
|
||||
mix_t = torch.tensor(mix, dtype=torch.float32)
|
||||
length_init = mix_t.shape[-1]
|
||||
|
||||
if length_init > 2 * (C - step) and (C - step > 0):
|
||||
mix_t = nn.functional.pad(mix_t, (C - step, C - step), mode='reflect')
|
||||
|
||||
fadein = torch.linspace(0, 1, fade_size).to(device)
|
||||
fadeout = torch.linspace(1, 0, fade_size).to(device)
|
||||
window_start = torch.ones(C).to(device)
|
||||
window_start[-fade_size:] *= fadeout
|
||||
window_finish = torch.ones(C).to(device)
|
||||
window_finish[:fade_size] *= fadein
|
||||
window_middle = torch.ones(C).to(device)
|
||||
window_middle[:fade_size] *= fadein
|
||||
window_middle[-fade_size:] *= fadeout
|
||||
|
||||
# This model outputs: instrumental, male, female (3 stems)
|
||||
stems = ['instrumental', 'male', 'female']
|
||||
num_stems = len(stems)
|
||||
req_shape = (num_stems,) + tuple(mix_t.shape)
|
||||
|
||||
with torch.inference_mode():
|
||||
result = torch.zeros(req_shape, dtype=torch.float32, device=device)
|
||||
counter = torch.zeros(req_shape, dtype=torch.float32, device=device)
|
||||
batch_data, batch_locs = [], []
|
||||
i = 0
|
||||
|
||||
total_seg = sum(1 for _ in range(0, mix_t.shape[1], step))
|
||||
|
||||
with tqdm(total=total_seg, desc='Separating', unit='segment', ncols=100) as pbar:
|
||||
while i < mix_t.shape[1]:
|
||||
part = mix_t[:, i:i + C].to(device)
|
||||
l = part.shape[-1]
|
||||
if l < C:
|
||||
part = nn.functional.pad(
|
||||
part,
|
||||
(0, C - l) if l > C // 2 + 1 else (0, C - l, 0, 0),
|
||||
mode='reflect' if l > C // 2 + 1 else 'constant',
|
||||
)
|
||||
batch_data.append(part)
|
||||
batch_locs.append((i, l))
|
||||
i += step
|
||||
|
||||
if len(batch_data) >= batch_size or i >= mix_t.shape[1]:
|
||||
arr = torch.stack(batch_data, dim=0)
|
||||
x = model(arr)
|
||||
for j, (start, ln) in enumerate(batch_locs):
|
||||
window = (window_start if start == 0
|
||||
else window_finish if i >= mix_t.shape[1]
|
||||
else window_middle)
|
||||
result[..., start:start + ln] += x[j][..., :ln] * window[..., :ln]
|
||||
counter[..., start:start + ln] += window[..., :ln]
|
||||
pbar.update(len(batch_data))
|
||||
batch_data, batch_locs = [], []
|
||||
|
||||
estimated = result / counter.clamp(min=1e-10)
|
||||
if length_init > 2 * (C - step) and (C - step > 0):
|
||||
estimated = estimated[..., (C - step):-(C - step)]
|
||||
|
||||
sources = {}
|
||||
for idx, name in enumerate(stems):
|
||||
sources[name] = estimated[idx].cpu().numpy()
|
||||
return sources
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str,
|
||||
batch_size_override: Optional[int] = None,
|
||||
num_overlap_override: Optional[int] = None) -> Dict[str, str]:
|
||||
"""Separate audio and write stems."""
|
||||
output_dir = Path(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
audio = _load_audio(input_path)
|
||||
model = self.get_model()
|
||||
sources = self._demix(audio, model,
|
||||
batch_size_override=batch_size_override,
|
||||
num_overlap_override=num_overlap_override)
|
||||
|
||||
del model
|
||||
torch.cuda.empty_cache()
|
||||
gc.collect()
|
||||
|
||||
ext = self.output_format
|
||||
result = {}
|
||||
for name, data in sources.items():
|
||||
dest = str(output_dir / f'{name}.{ext}')
|
||||
_save_audio(dest, data.T, ext, self.mp3_bitrate)
|
||||
result[name] = dest
|
||||
return result
|
||||
+322
@@ -0,0 +1,322 @@
|
||||
"""
|
||||
Standalone BS-RoFormer separator — no Django, no Spleeter/TensorFlow dependency.
|
||||
|
||||
Audio I/O uses soundfile (WAV/FLAC) + subprocess ffmpeg (MP3 encoding).
|
||||
Model architecture is unchanged from the original spleeter-web implementation.
|
||||
"""
|
||||
import gc
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import yaml
|
||||
from omegaconf import OmegaConf
|
||||
from tqdm import tqdm
|
||||
|
||||
from .bs_roformer import BSRoformer
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HuggingFace model coordinates
|
||||
# ---------------------------------------------------------------------------
|
||||
HF_REPO_ID = 'jarredou/BS-ROFO-SW-Fixed'
|
||||
HF_REVISION = 'f99fc5c2e4bc201413cfc1ca5767cf62ddb1a9a4'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Default model paths (overridable via env var BS_ROFORMER_MODEL_DIR)
|
||||
# ---------------------------------------------------------------------------
|
||||
_MODEL_DIR_ENV = os.environ.get('BS_ROFORMER_MODEL_DIR', '/opt/bs_roformer_models')
|
||||
DEFAULT_MODEL_DIR = Path(_MODEL_DIR_ENV)
|
||||
DEFAULT_MODEL_PATH = DEFAULT_MODEL_DIR / 'BS-Rofo-SW-Fixed.ckpt'
|
||||
DEFAULT_CONFIG_PATH = DEFAULT_MODEL_DIR / 'BS-Rofo-SW-Fixed.yaml'
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _try_download_model(
|
||||
model_dir: Path = DEFAULT_MODEL_DIR,
|
||||
model_path: Path = DEFAULT_MODEL_PATH,
|
||||
config_path: Path = DEFAULT_CONFIG_PATH,
|
||||
) -> None:
|
||||
"""Download BS-RoFormer model from HuggingFace if not already present."""
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
from huggingface_hub import snapshot_download
|
||||
snapshot_download(
|
||||
repo_id=HF_REPO_ID,
|
||||
revision=HF_REVISION,
|
||||
local_dir=str(model_dir),
|
||||
)
|
||||
|
||||
|
||||
def _load_config(config_path: Path):
|
||||
"""Load YAML config, supporting !!python/tuple tags."""
|
||||
with open(config_path, 'r') as f:
|
||||
config = yaml.load(f, Loader=yaml.UnsafeLoader)
|
||||
return OmegaConf.create(config)
|
||||
|
||||
|
||||
def _load_audio(path: str) -> np.ndarray:
|
||||
"""
|
||||
Load audio file and return as float32 numpy array of shape (channels, samples).
|
||||
Uses soundfile for WAV/FLAC; falls back to ffmpeg decode for MP3 and other formats.
|
||||
"""
|
||||
import soundfile as sf
|
||||
|
||||
try:
|
||||
data, sr = sf.read(path, dtype='float32', always_2d=True)
|
||||
# soundfile returns (samples, channels); transpose to (channels, samples)
|
||||
data = data.T
|
||||
except Exception:
|
||||
# Fallback: decode via ffmpeg to a temporary WAV
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', path, '-ar', str(SAMPLE_RATE),
|
||||
'-ac', '2', '-f', 'wav', tmp_path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
data, sr = sf.read(tmp_path, dtype='float32', always_2d=True)
|
||||
data = data.T
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
# Resample if needed
|
||||
if sr != SAMPLE_RATE:
|
||||
import librosa
|
||||
data = np.array([
|
||||
librosa.resample(ch, orig_sr=sr, target_sr=SAMPLE_RATE)
|
||||
for ch in data
|
||||
], dtype=np.float32)
|
||||
|
||||
# Ensure stereo
|
||||
if data.shape[0] == 1:
|
||||
data = np.vstack([data, data])
|
||||
|
||||
return data # (channels, samples)
|
||||
|
||||
|
||||
def _save_audio(path: str, data: np.ndarray, output_format: str, bitrate: str = '256k') -> None:
|
||||
"""
|
||||
Save (samples, channels) float32 array as audio file.
|
||||
WAV and FLAC use soundfile directly; MP3 encodes via ffmpeg.
|
||||
"""
|
||||
import soundfile as sf
|
||||
|
||||
ext = output_format.lower()
|
||||
|
||||
if ext in ('wav', 'flac'):
|
||||
sf.write(path, data, SAMPLE_RATE, format=ext.upper(),
|
||||
subtype='PCM_24' if ext == 'wav' else 'PCM_16')
|
||||
else:
|
||||
# Encode to MP3 via ffmpeg
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
sf.write(tmp_path, data, SAMPLE_RATE, format='WAV', subtype='PCM_16')
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', tmp_path,
|
||||
'-b:a', bitrate, path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Separator class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class BSRoformerSeparator:
|
||||
"""
|
||||
Performs source separation using the BS-RoFormer model.
|
||||
Pure PyTorch — no TensorFlow / Spleeter dependency.
|
||||
"""
|
||||
|
||||
STEM_MODE_4 = '4stem'
|
||||
STEM_MODE_5_GUITAR = '5stem_guitar'
|
||||
STEM_MODE_5_PIANO = '5stem_piano'
|
||||
STEM_MODE_6 = '6stem'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_path: Optional[str] = None,
|
||||
config_path: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3', # 'mp3' | 'flac' | 'wav'
|
||||
mp3_bitrate: str = '256k',
|
||||
stem_mode: str = '4stem',
|
||||
):
|
||||
self.model_path = Path(model_path) if model_path else DEFAULT_MODEL_PATH
|
||||
self.config_path = Path(config_path) if config_path else DEFAULT_CONFIG_PATH
|
||||
self.device = 'cpu' if cpu_separation else 'cuda'
|
||||
self.output_format = output_format.lower()
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self.stem_mode = stem_mode
|
||||
self._config = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Model loading
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _ensure_model(self):
|
||||
_try_download_model(DEFAULT_MODEL_DIR, self.model_path, self.config_path)
|
||||
|
||||
def _load_config(self):
|
||||
if self._config is None:
|
||||
self._config = _load_config(self.config_path)
|
||||
return self._config
|
||||
|
||||
def get_model(self) -> BSRoformer:
|
||||
self._ensure_model()
|
||||
config = self._load_config()
|
||||
model_cfg = OmegaConf.to_container(config.model, resolve=True)
|
||||
|
||||
# OmegaConf loads tuples as lists; the model requires tuples
|
||||
for key in ('freqs_per_bands', 'multi_stft_resolutions_window_sizes'):
|
||||
if key in model_cfg and isinstance(model_cfg[key], list):
|
||||
model_cfg[key] = tuple(model_cfg[key])
|
||||
|
||||
model = BSRoformer(**model_cfg)
|
||||
checkpoint = torch.load(str(self.model_path), map_location='cpu', weights_only=False)
|
||||
model.load_state_dict(checkpoint)
|
||||
model.to(self.device).eval()
|
||||
return model
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Inference
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _demix(self, mix: np.ndarray, model: BSRoformer,
|
||||
batch_size_override: Optional[int] = None,
|
||||
num_overlap_override: Optional[int] = None) -> Dict[str, np.ndarray]:
|
||||
"""Separate (channels, samples) array into a dict of stem arrays."""
|
||||
config = self._load_config()
|
||||
hop_length = config.model.stft_hop_length
|
||||
segment_size = config.inference.dim_t
|
||||
C = hop_length * (segment_size - 1)
|
||||
N = num_overlap_override if num_overlap_override is not None else config.inference.num_overlap
|
||||
step = int(C // N)
|
||||
fade_size = C // 10
|
||||
batch_size = batch_size_override if batch_size_override is not None else config.inference.batch_size
|
||||
device = self.device
|
||||
|
||||
mix_t = torch.tensor(mix, dtype=torch.float32)
|
||||
length_init = mix_t.shape[-1]
|
||||
|
||||
if length_init > 2 * (C - step) and (C - step > 0):
|
||||
mix_t = nn.functional.pad(mix_t, (C - step, C - step), mode='reflect')
|
||||
|
||||
fadein = torch.linspace(0, 1, fade_size).to(device)
|
||||
fadeout = torch.linspace(1, 0, fade_size).to(device)
|
||||
window_start = torch.ones(C).to(device); window_start[-fade_size:] *= fadeout
|
||||
window_finish = torch.ones(C).to(device); window_finish[:fade_size] *= fadein
|
||||
window_middle = torch.ones(C).to(device)
|
||||
window_middle[:fade_size] *= fadein; window_middle[-fade_size:] *= fadeout
|
||||
|
||||
S = len(config.training.instruments)
|
||||
req_shape = (S,) + tuple(mix_t.shape)
|
||||
|
||||
with torch.inference_mode():
|
||||
result = torch.zeros(req_shape, dtype=torch.float32, device=device)
|
||||
counter = torch.zeros(req_shape, dtype=torch.float32, device=device)
|
||||
batch_data, batch_locs = [], []
|
||||
i = 0
|
||||
|
||||
# count total segments for tqdm
|
||||
total_seg = sum(1 for _ in range(0, mix_t.shape[1], step))
|
||||
|
||||
with tqdm(total=total_seg, desc='Separating', unit='segment', ncols=100) as pbar:
|
||||
while i < mix_t.shape[1]:
|
||||
part = mix_t[:, i:i + C].to(device)
|
||||
l = part.shape[-1]
|
||||
if l < C:
|
||||
part = nn.functional.pad(
|
||||
part,
|
||||
(0, C - l) if l > C // 2 + 1 else (0, C - l, 0, 0),
|
||||
mode='reflect' if l > C // 2 + 1 else 'constant',
|
||||
)
|
||||
batch_data.append(part)
|
||||
batch_locs.append((i, l))
|
||||
i += step
|
||||
|
||||
if len(batch_data) >= batch_size or i >= mix_t.shape[1]:
|
||||
arr = torch.stack(batch_data, dim=0)
|
||||
x = model(arr)
|
||||
for j, (start, ln) in enumerate(batch_locs):
|
||||
window = (window_start if start == 0
|
||||
else window_finish if i >= mix_t.shape[1]
|
||||
else window_middle)
|
||||
result[..., start:start + ln] += x[j][..., :ln] * window[..., :ln]
|
||||
counter[..., start:start + ln] += window[..., :ln]
|
||||
pbar.update(len(batch_data))
|
||||
batch_data, batch_locs = [], []
|
||||
|
||||
estimated = result / counter.clamp(min=1e-10)
|
||||
if length_init > 2 * (C - step) and (C - step > 0):
|
||||
estimated = estimated[..., (C - step):-(C - step)]
|
||||
|
||||
sources = {}
|
||||
for idx, name in enumerate(config.training.instruments):
|
||||
sources[name] = estimated[idx].cpu().numpy()
|
||||
return sources
|
||||
|
||||
def _merge_stems(self, sources_6: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]:
|
||||
"""Map 6-stem model output to the requested stem_mode output."""
|
||||
v, d, b, o, g, p = (
|
||||
sources_6['vocals'], sources_6['drums'], sources_6['bass'],
|
||||
sources_6['other'], sources_6['guitar'], sources_6['piano'],
|
||||
)
|
||||
if self.stem_mode == self.STEM_MODE_4:
|
||||
return {'vocals': v, 'drums': d, 'bass': b, 'other': o + g + p}
|
||||
if self.stem_mode == self.STEM_MODE_5_GUITAR:
|
||||
return {'vocals': v, 'drums': d, 'bass': b, 'guitar': g, 'other': o + p}
|
||||
if self.stem_mode == self.STEM_MODE_5_PIANO:
|
||||
return {'vocals': v, 'drums': d, 'bass': b, 'piano': p, 'other': o + g}
|
||||
# 6stem — return all
|
||||
return sources_6
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str,
|
||||
batch_size_override: Optional[int] = None,
|
||||
num_overlap_override: Optional[int] = None) -> Dict[str, str]:
|
||||
"""
|
||||
Separate audio at input_path and write one file per stem into output_dir.
|
||||
Returns {stem_name: output_file_path}.
|
||||
"""
|
||||
output_dir = Path(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
audio = _load_audio(input_path) # (channels, samples)
|
||||
model = self.get_model()
|
||||
sources_6 = self._demix(audio, model,
|
||||
batch_size_override=batch_size_override,
|
||||
num_overlap_override=num_overlap_override)
|
||||
|
||||
del model
|
||||
torch.cuda.empty_cache()
|
||||
gc.collect()
|
||||
|
||||
stems = self._merge_stems(sources_6)
|
||||
ext = self.output_format
|
||||
result = {}
|
||||
for name, data in stems.items():
|
||||
dest = str(output_dir / f'{name}.{ext}')
|
||||
_save_audio(dest, data.T, ext, self.mp3_bitrate) # T: (channels,samples)→(samples,channels)
|
||||
result[name] = dest
|
||||
return result
|
||||
+106
@@ -0,0 +1,106 @@
|
||||
"""
|
||||
BS-RoFormer SW separator - Swin Transformer variant for 6-stem separation
|
||||
Models from mvsep.com / audio-separator
|
||||
"""
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
try:
|
||||
from audio_separator.separator import Separator
|
||||
AUDIO_SEPARATOR_AVAILABLE = True
|
||||
except ImportError:
|
||||
AUDIO_SEPARATOR_AVAILABLE = False
|
||||
|
||||
from ..config import BS_ROFORMER_MODEL_DIR, CPU_SEPARATION
|
||||
|
||||
|
||||
class BSRoformerSWSeparator:
|
||||
"""
|
||||
BS-RoFormer SW (Swin Transformer) separator.
|
||||
|
||||
This variant uses Swin Transformer architecture and provides
|
||||
6-stem separation: vocals, drums, bass, guitar, piano, other.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
cpu_separation: bool = CPU_SEPARATION,
|
||||
output_format: str = "mp3",
|
||||
mp3_bitrate: str = "256k",
|
||||
):
|
||||
if not AUDIO_SEPARATOR_AVAILABLE:
|
||||
raise ImportError(
|
||||
"audio-separator is required for BS-RoFormer SW separation. "
|
||||
"Install with: pip install audio-separator"
|
||||
)
|
||||
|
||||
self.cpu_separation = cpu_separation
|
||||
self.output_format = output_format
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self._separator = None
|
||||
|
||||
def _get_separator(self):
|
||||
"""Lazy-load the separator with SW model."""
|
||||
if self._separator is None:
|
||||
# BS-RoFormer SW model is typically available in audio-separator
|
||||
# as a pre-configured model
|
||||
self._separator = Separator(
|
||||
model_file_dir=str(BS_ROFORMER_MODEL_DIR),
|
||||
output_dir=None,
|
||||
output_format=self.output_format,
|
||||
output_bitrate=self.mp3_bitrate if self.output_format == "mp3" else None,
|
||||
normalization_threshold=0.9,
|
||||
enable_denoise=False,
|
||||
cpu_separation=self.cpu_separation,
|
||||
)
|
||||
|
||||
# Load the SW model - this may need specific model name
|
||||
# audio-separator typically provides this as "model_bs_roformer_sw"
|
||||
try:
|
||||
self._separator.load_model()
|
||||
except Exception as e:
|
||||
# Fallback: try to load specific model file
|
||||
sw_model_path = BS_ROFORMER_MODEL_DIR / "bs_roformer_sw.ckpt"
|
||||
if sw_model_path.exists():
|
||||
self._separator.load_model(str(sw_model_path))
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"BS-RoFormer SW model not found. Please download it first. Error: {e}"
|
||||
)
|
||||
|
||||
return self._separator
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
||||
"""
|
||||
Separate audio into 6 stems.
|
||||
|
||||
Returns:
|
||||
Dict mapping stem names to output file paths
|
||||
Stems: vocals, drums, bass, guitar, piano, other
|
||||
"""
|
||||
sep = self._get_separator()
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Run separation
|
||||
result = sep.separate(input_path)
|
||||
|
||||
# Map outputs to 6 stems
|
||||
stems = ["vocals", "drums", "bass", "guitar", "piano", "other"]
|
||||
output_paths = {}
|
||||
|
||||
if isinstance(result, list):
|
||||
for i, path in enumerate(result):
|
||||
if i < len(stems):
|
||||
stem_name = stems[i]
|
||||
final_path = output_path / f"{stem_name}.{self.output_format}"
|
||||
Path(path).rename(final_path)
|
||||
output_paths[stem_name] = str(final_path)
|
||||
else:
|
||||
# Fallback - single output
|
||||
final_path = output_path / f"vocals.{self.output_format}"
|
||||
Path(result).rename(final_path)
|
||||
output_paths["vocals"] = str(final_path)
|
||||
|
||||
return output_paths
|
||||
+263
@@ -0,0 +1,263 @@
|
||||
"""
|
||||
Ensemble separator - Combines multiple models for best quality separation
|
||||
Uses model averaging/ensembling from multiple high-quality models.
|
||||
"""
|
||||
import os
|
||||
import torch
|
||||
import torchaudio
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
import tempfile
|
||||
import shutil
|
||||
|
||||
from ..config import ENSEMBLE_CACHE_DIR, CPU_SEPARATION
|
||||
|
||||
|
||||
# Ensemble configurations
|
||||
ENSEMBLE_CONFIGS = {
|
||||
"ensemble_vocal_best": {
|
||||
"description": "Best vocal separation using BS-RoFormer x2 + MelBand RoFormer ft + SCNet XL IHF",
|
||||
"models": [
|
||||
{"type": "bs_roformer", "weight": 0.30, "stem_mode": "4stem"},
|
||||
{"type": "mel_band", "weight": 0.25, "variant": "vocals"},
|
||||
{"type": "scnet_xl_ihf", "weight": 0.45},
|
||||
],
|
||||
"stems": ["vocals", "instrumental"],
|
||||
},
|
||||
"ensemble_4stem": {
|
||||
"description": "Best 4-stem separation using multiple models averaged",
|
||||
"models": [
|
||||
{"type": "demucs", "weight": 0.20, "model": "htdemucs_ft"},
|
||||
{"type": "bs_roformer", "weight": 0.30, "stem_mode": "4stem"},
|
||||
{"type": "mel_band", "weight": 0.25, "variant": "4stem"},
|
||||
{"type": "scnet_large", "weight": 0.25},
|
||||
],
|
||||
"stems": ["vocals", "drums", "bass", "other"],
|
||||
},
|
||||
"ensemble_6stem": {
|
||||
"description": "Best 6-stem separation using multiple models averaged",
|
||||
"models": [
|
||||
{"type": "bs_roformer", "weight": 0.35, "stem_mode": "6stem"},
|
||||
{"type": "bs_roformer_sw", "weight": 0.35},
|
||||
{"type": "demucs", "weight": 0.30, "model": "htdemucs_6s"},
|
||||
],
|
||||
"stems": ["vocals", "drums", "bass", "guitar", "piano", "other"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class EnsembleSeparator:
|
||||
"""
|
||||
Ensemble separator that combines multiple models for best quality.
|
||||
|
||||
Uses weighted averaging of outputs from multiple separation models
|
||||
to produce higher quality results than any single model alone.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ensemble_id: str = "ensemble_vocal_best",
|
||||
cpu_separation: bool = CPU_SEPARATION,
|
||||
output_format: str = "mp3",
|
||||
mp3_bitrate: str = "256k",
|
||||
):
|
||||
if ensemble_id not in ENSEMBLE_CONFIGS:
|
||||
raise ValueError(f"Unknown ensemble config: {ensemble_id}")
|
||||
|
||||
self.ensemble_id = ensemble_id
|
||||
self.config = ENSEMBLE_CONFIGS[ensemble_id]
|
||||
self.cpu_separation = cpu_separation
|
||||
self.output_format = output_format
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self.cache_dir = Path(ENSEMBLE_CACHE_DIR)
|
||||
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _load_audio(self, path: str) -> tuple:
|
||||
"""Load audio file and return waveform and sample rate."""
|
||||
waveform, sr = torchaudio.load(path)
|
||||
return waveform, sr
|
||||
|
||||
def _save_audio(self, waveform: torch.Tensor, sr: int, path: str):
|
||||
"""Save audio waveform to file."""
|
||||
torchaudio.save(path, waveform, sr)
|
||||
|
||||
def _weighted_average(self, audio_tensors: List[torch.Tensor], weights: List[float]) -> torch.Tensor:
|
||||
"""Compute weighted average of audio tensors."""
|
||||
if len(audio_tensors) != len(weights):
|
||||
raise ValueError("Number of tensors must match number of weights")
|
||||
|
||||
# Normalize weights
|
||||
total_weight = sum(weights)
|
||||
normalized_weights = [w / total_weight for w in weights]
|
||||
|
||||
# Compute weighted sum
|
||||
result = torch.zeros_like(audio_tensors[0])
|
||||
for tensor, weight in zip(audio_tensors, normalized_weights):
|
||||
result += tensor * weight
|
||||
|
||||
return result
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
||||
"""
|
||||
Run ensemble separation combining multiple models.
|
||||
|
||||
This runs multiple models and averages their outputs for best quality.
|
||||
Note: This takes significantly longer than single-model separation.
|
||||
|
||||
Returns:
|
||||
Dict mapping stem names to output file paths
|
||||
"""
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create temporary directory for intermediate results
|
||||
temp_dir = Path(tempfile.mkdtemp(dir=self.cache_dir))
|
||||
|
||||
try:
|
||||
stems = self.config["stems"]
|
||||
stem_audio_data = {stem: [] for stem in stems}
|
||||
stem_weights = {stem: [] for stem in stems}
|
||||
|
||||
# Run each model in the ensemble
|
||||
for model_config in self.config["models"]:
|
||||
model_type = model_config["type"]
|
||||
weight = model_config["weight"]
|
||||
|
||||
print(f"Running ensemble component: {model_type} (weight: {weight})")
|
||||
|
||||
# Run individual model and get stems
|
||||
model_stems = self._run_model_component(
|
||||
model_config, input_path, temp_dir
|
||||
)
|
||||
|
||||
# Add to collection for averaging
|
||||
for stem_name, stem_path in model_stems.items():
|
||||
if stem_name in stem_audio_data:
|
||||
waveform, sr = self._load_audio(stem_path)
|
||||
stem_audio_data[stem_name].append(waveform)
|
||||
stem_weights[stem_name].append(weight)
|
||||
|
||||
# Compute weighted averages and save final outputs
|
||||
output_paths = {}
|
||||
sample_rate = 44100 # Default, will be updated from actual audio
|
||||
|
||||
for stem_name in stems:
|
||||
if stem_name in stem_audio_data and stem_audio_data[stem_name]:
|
||||
# Compute weighted average
|
||||
averaged = self._weighted_average(
|
||||
stem_audio_data[stem_name],
|
||||
stem_weights[stem_name]
|
||||
)
|
||||
|
||||
# Save result
|
||||
output_file = output_path / f"{stem_name}.{self.output_format}"
|
||||
|
||||
# Convert to desired format
|
||||
if self.output_format == "mp3":
|
||||
# Save as WAV first, then convert
|
||||
temp_wav = temp_dir / f"{stem_name}_temp.wav"
|
||||
self._save_audio(averaged, sample_rate, str(temp_wav))
|
||||
|
||||
# Convert to MP3
|
||||
import subprocess
|
||||
bitrate = self.mp3_bitrate.replace("k", "")
|
||||
subprocess.run([
|
||||
"ffmpeg", "-y", "-i", str(temp_wav),
|
||||
"-b:a", f"{bitrate}k",
|
||||
"-f", "mp3", str(output_file)
|
||||
], check=True, capture_output=True)
|
||||
else:
|
||||
self._save_audio(averaged, sample_rate, str(output_file))
|
||||
|
||||
output_paths[stem_name] = str(output_file)
|
||||
|
||||
return output_paths
|
||||
|
||||
finally:
|
||||
# Cleanup temp directory
|
||||
try:
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
except:
|
||||
pass
|
||||
|
||||
def _run_model_component(
|
||||
self,
|
||||
model_config: Dict,
|
||||
input_path: str,
|
||||
output_dir: Path
|
||||
) -> Dict[str, str]:
|
||||
"""
|
||||
Run a single model component and return its output stems.
|
||||
|
||||
Returns:
|
||||
Dict mapping stem names to file paths
|
||||
"""
|
||||
model_type = model_config["type"]
|
||||
|
||||
if model_type == "bs_roformer":
|
||||
from .bs_roformer_separator import BSRoformerSeparator
|
||||
sep = BSRoformerSeparator(
|
||||
cpu_separation=self.cpu_separation,
|
||||
output_format="wav", # Use WAV for lossless intermediate
|
||||
mp3_bitrate="320k",
|
||||
stem_mode=model_config.get("stem_mode", "4stem"),
|
||||
)
|
||||
return sep.separate_into_parts(input_path, str(output_dir / "bsr"))
|
||||
|
||||
elif model_type == "bs_roformer_sw":
|
||||
from .bs_roformer_sw_separator import BSRoformerSWSeparator
|
||||
sep = BSRoformerSWSeparator(
|
||||
cpu_separation=self.cpu_separation,
|
||||
output_format="wav",
|
||||
mp3_bitrate="320k",
|
||||
)
|
||||
return sep.separate_into_parts(input_path, str(output_dir / "bsr_sw"))
|
||||
|
||||
elif model_type == "mel_band":
|
||||
from .mel_band_specialized_separator import MelBandVocalsSeparator
|
||||
variant = model_config.get("variant", "vocals")
|
||||
if variant == "vocals":
|
||||
sep = MelBandVocalsSeparator(
|
||||
cpu_separation=self.cpu_separation,
|
||||
output_format="wav",
|
||||
mp3_bitrate="320k",
|
||||
)
|
||||
else:
|
||||
from .mel_band_roformer_separator import MelBandRoformerSeparator
|
||||
sep = MelBandRoformerSeparator(
|
||||
cpu_separation=self.cpu_separation,
|
||||
output_format="wav",
|
||||
mp3_bitrate="320k",
|
||||
)
|
||||
return sep.separate_into_parts(input_path, str(output_dir / "mbr"))
|
||||
|
||||
elif model_type == "scnet_xl_ihf":
|
||||
from .scnet_separator import SCNetSeparator
|
||||
sep = SCNetSeparator(
|
||||
model_id="scnet_xl_ihf",
|
||||
cpu_separation=self.cpu_separation,
|
||||
output_format="wav",
|
||||
mp3_bitrate="320k",
|
||||
)
|
||||
return sep.separate_into_parts(input_path, str(output_dir / "scnet"))
|
||||
|
||||
elif model_type == "scnet_large":
|
||||
from .scnet_separator import SCNetSeparator
|
||||
sep = SCNetSeparator(
|
||||
model_id="scnet_large",
|
||||
cpu_separation=self.cpu_separation,
|
||||
output_format="wav",
|
||||
mp3_bitrate="320k",
|
||||
)
|
||||
return sep.separate_into_parts(input_path, str(output_dir / "scnet"))
|
||||
|
||||
elif model_type == "demucs":
|
||||
# For Demucs, we'd need to run via CLI subprocess
|
||||
# This is a placeholder
|
||||
model_name = model_config.get("model", "htdemucs_ft")
|
||||
# Return empty dict for now - would need actual implementation
|
||||
return {}
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown model type in ensemble: {model_type}")
|
||||
Executable
+847
@@ -0,0 +1,847 @@
|
||||
"""
|
||||
LRC alignment service for creating enhanced ELRC files with word-level timestamps.
|
||||
Uses the lyrics-sync module for vocal extraction and forced alignment.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Tuple
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import JOBS_DIR, LRC_SYNC_DIR
|
||||
from .metadata_util import (
|
||||
extract_lyrics_from_file,
|
||||
lyrics_to_plaintext,
|
||||
format_elrc,
|
||||
is_lrc_synced
|
||||
)
|
||||
|
||||
# Add lyrics-sync to path
|
||||
_LRC_SYNC_PATH = Path(__file__).parent.parent.parent / "lyrics-sync"
|
||||
if str(_LRC_SYNC_PATH) not in sys.path:
|
||||
sys.path.insert(0, str(_LRC_SYNC_PATH))
|
||||
|
||||
|
||||
class _LyricsSyncNoExtract:
|
||||
"""
|
||||
Drop-in replacement for LyricsSync that skips Demucs vocal extraction.
|
||||
|
||||
Use this when the input audio is already a separated vocals stem (e.g. from
|
||||
a project's Stems folder), so we avoid loading and running htdemucs again.
|
||||
The alignment pipeline is otherwise identical to the standard LyricsSync.
|
||||
"""
|
||||
|
||||
def __init__(self, lang: str = "en-US", blank_id: int = 0, device: str = None) -> None:
|
||||
from lsync.phoneme_recognizer import PhonemeRecognizer
|
||||
from lsync.lyrics_processor import LyricsProcessor
|
||||
self.phone_rec = PhonemeRecognizer(lang=lang, device=device)
|
||||
self.lp = LyricsProcessor(lang=lang)
|
||||
self.blank_id = blank_id
|
||||
|
||||
def sync(self, audio_fn: str, text_fn: str, save: bool = True, timing_precision: str = 'word'):
|
||||
import os
|
||||
import librosa
|
||||
from lsync.config import TARGET_SR
|
||||
from lsync.alignment import Aligner
|
||||
from lsync.util import read_text, save_lrc, save_words
|
||||
from lsync.lrc_formatter import LrcFormatter
|
||||
|
||||
audio_name = os.path.splitext(os.path.basename(audio_fn))[0]
|
||||
|
||||
# Load vocals directly (skip Demucs), resample to TARGET_SR
|
||||
vocals, sr = librosa.load(audio_fn, sr=None, mono=True)
|
||||
if sr != TARGET_SR:
|
||||
vocals = librosa.resample(vocals, orig_sr=sr, target_sr=TARGET_SR)
|
||||
|
||||
# Rest of alignment pipeline is unchanged
|
||||
emission, _, _, frame_duration = self.phone_rec.recognize(vocals)
|
||||
tokens = self.phone_rec.tokenize(text_fn)
|
||||
path = Aligner.align(emission, tokens, blank_id=self.blank_id)
|
||||
processed_lyrics = self.lp.process(text_fn)
|
||||
|
||||
# Always compute word-level for the basic LRC output
|
||||
words = self.lp.get_words_from_path(processed_lyrics, path, frame_duration)
|
||||
original_lyrics = read_text(text_fn)
|
||||
lrc = LrcFormatter.words2lrc(words, original_lyrics)
|
||||
|
||||
if save:
|
||||
save_words(words, audio_name)
|
||||
save_lrc(lrc, audio_name)
|
||||
|
||||
# Return syllable tokens for ELRC when requested
|
||||
if timing_precision == 'syllable':
|
||||
return_words = self.lp.get_syllables_from_path(processed_lyrics, path, frame_duration)
|
||||
else:
|
||||
return_words = words
|
||||
|
||||
return return_words, lrc
|
||||
|
||||
|
||||
# ── Lyrics cleaning helpers ───────────────────────────────────────────────────
|
||||
|
||||
_SECTION_MARKER_RE = re.compile(r'^\[(?!\d{1,2}:\d{2})[^\]]*\]\s*$')
|
||||
|
||||
def clean_lyrics_text(text: str) -> str:
|
||||
"""Remove section markers (e.g. [Chorus], [Verse 1]) and blank lines from plain lyrics."""
|
||||
cleaned = []
|
||||
for line in text.splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
if _SECTION_MARKER_RE.match(stripped):
|
||||
continue
|
||||
cleaned.append(stripped)
|
||||
return '\n'.join(cleaned)
|
||||
|
||||
|
||||
_LRC_SECTION_LINE_RE = re.compile(
|
||||
r'^\[\d{1,2}:\d{2}[.\d]*\]\s*\[(?!\d)[^\]]*\]\s*$'
|
||||
)
|
||||
|
||||
def clean_lrc_content(content: str) -> tuple:
|
||||
"""Remove section-marker lines from LRC/ELRC content.
|
||||
|
||||
A section-marker line looks like ``[01:30.00][Chorus]`` — a timestamp
|
||||
followed by a non-timestamp bracket label and nothing else.
|
||||
Returns ``(cleaned_str, lines_removed_count)``.
|
||||
"""
|
||||
lines = content.splitlines()
|
||||
cleaned, removed = [], 0
|
||||
for line in lines:
|
||||
if _LRC_SECTION_LINE_RE.match(line):
|
||||
removed += 1
|
||||
else:
|
||||
cleaned.append(line)
|
||||
return '\n'.join(cleaned), removed
|
||||
|
||||
|
||||
def run_lrc_alignment(
|
||||
job_id: str,
|
||||
audio_path: str,
|
||||
lyrics_source: str = "auto", # 'auto', 'tag', 'file', 'text'
|
||||
lyrics_content: Optional[str] = None,
|
||||
lyrics_file_path: Optional[str] = None,
|
||||
output_filename: Optional[str] = None,
|
||||
style_options: Optional[dict] = None,
|
||||
is_vocals: bool = False,
|
||||
output_dir: Optional[str] = None,
|
||||
timing_precision: str = 'word', # 'word' | 'syllable'
|
||||
lrc_power_preset: str = 'standard', # 'eco' | 'standard' | 'performance'
|
||||
):
|
||||
"""
|
||||
Background task: align lyrics with vocals to create ELRC file.
|
||||
|
||||
Args:
|
||||
job_id: Job identifier
|
||||
audio_path: Path to the audio file (with vocals)
|
||||
lyrics_source: Where to get lyrics from
|
||||
- 'auto': Try metadata tags, then external LRC, then fail
|
||||
- 'tag': Only use metadata tags
|
||||
- 'file': Use provided lyrics_file_path
|
||||
- 'text': Use provided lyrics_content
|
||||
lyrics_content: Raw lyrics text (when lyrics_source='text')
|
||||
lyrics_file_path: Path to LRC/txt file (when lyrics_source='file')
|
||||
output_filename: Optional custom filename for output ELRC
|
||||
is_vocals: When True, the audio_path is already a separated vocals stem —
|
||||
skip the internal Demucs extraction step for faster processing.
|
||||
output_dir: Optional custom output directory (defaults to LRC_SYNC_DIR / job_id)
|
||||
"""
|
||||
if output_dir:
|
||||
out_path = Path(output_dir)
|
||||
else:
|
||||
out_path = LRC_SYNC_DIR / job_id
|
||||
out_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
# Import lyrics-sync here to handle any import issues gracefully
|
||||
try:
|
||||
from lsync import LyricsSync
|
||||
except ImportError as e:
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.ERROR,
|
||||
error=f"Lyrics sync module not available: {e}",
|
||||
progress="Error."
|
||||
)
|
||||
return
|
||||
|
||||
# Step 1: Get lyrics
|
||||
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Reading lyrics...")
|
||||
|
||||
lyrics_text = ""
|
||||
lyrics_synced = False
|
||||
|
||||
if lyrics_source == "text" and lyrics_content:
|
||||
lyrics_text = lyrics_content
|
||||
lyrics_synced = is_lrc_synced(lyrics_content)
|
||||
elif lyrics_source == "file" and lyrics_file_path:
|
||||
lyrics_data = extract_lyrics_from_file(lyrics_file_path)
|
||||
lyrics_text = lyrics_data["plain"]
|
||||
lyrics_synced = lyrics_data["synced"]
|
||||
elif lyrics_source in ("auto", "tag"):
|
||||
lyrics_data = extract_lyrics_from_file(audio_path)
|
||||
lyrics_text = lyrics_data["plain"]
|
||||
lyrics_synced = lyrics_data["synced"]
|
||||
|
||||
# If no lyrics in tags, try external LRC file
|
||||
if not lyrics_text and lyrics_source == "auto":
|
||||
audio_dir = Path(audio_path).parent
|
||||
audio_stem = Path(audio_path).stem
|
||||
for ext in ['.lrc', '.LRC', '.txt', '.TXT']:
|
||||
lrc_file = audio_dir / f"{audio_stem}{ext}"
|
||||
if lrc_file.exists():
|
||||
lyrics_data = extract_lyrics_from_file(str(lrc_file))
|
||||
lyrics_text = lyrics_data["plain"]
|
||||
lyrics_synced = lyrics_data["synced"]
|
||||
break
|
||||
|
||||
if not lyrics_text:
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.ERROR,
|
||||
error="No lyrics found. Please provide lyrics text or ensure audio file has lyrics tags.",
|
||||
progress="Error."
|
||||
)
|
||||
return
|
||||
|
||||
# Strip section markers ([Chorus], [Verse 1], etc.) and blank lines
|
||||
if not lyrics_synced:
|
||||
lyrics_text = clean_lyrics_text(lyrics_text)
|
||||
|
||||
# Choose sync class: skip Demucs when audio is already vocals.
|
||||
# In eco mode, force CPU for the phoneme recogniser to avoid GPU contention
|
||||
# that causes display glitches on shared-GPU (WSL2) setups.
|
||||
_align_device = "cpu" if lrc_power_preset == "eco" else None
|
||||
SyncClass = _LyricsSyncNoExtract if is_vocals else LyricsSync
|
||||
|
||||
# If lyrics are already synced (LRC format), convert to ELRC with word-level timestamps
|
||||
if lyrics_synced:
|
||||
jm.update_job(job_id, progress="Lyrics already synced. Enhancing to word-level...")
|
||||
_enhance_existing_lrc(
|
||||
job_id, audio_path, lyrics_text, out_path,
|
||||
output_filename, SyncClass, style_options, timing_precision,
|
||||
align_device=_align_device,
|
||||
)
|
||||
else:
|
||||
# Run full alignment pipeline
|
||||
jm.update_job(job_id, progress="Starting lyrics alignment...")
|
||||
_run_full_alignment(
|
||||
job_id, audio_path, lyrics_text, out_path,
|
||||
output_filename, SyncClass, style_options, timing_precision,
|
||||
align_device=_align_device,
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
|
||||
|
||||
|
||||
def _run_full_alignment(
|
||||
job_id: str,
|
||||
audio_path: str,
|
||||
lyrics_text: str,
|
||||
output_dir: Path,
|
||||
output_filename: Optional[str],
|
||||
LyricsSyncClass,
|
||||
style_options: Optional[dict] = None,
|
||||
timing_precision: str = 'word',
|
||||
align_device: Optional[str] = None,
|
||||
):
|
||||
"""Run full alignment using lyrics-sync."""
|
||||
# Write lyrics to temp file
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False, encoding='utf-8') as f:
|
||||
f.write(lyrics_text)
|
||||
lyrics_temp_path = f.name
|
||||
|
||||
try:
|
||||
# Initialize lyrics sync
|
||||
is_no_extract = LyricsSyncClass is _LyricsSyncNoExtract
|
||||
device_note = f" [{align_device.upper()}]" if align_device else ""
|
||||
jm.update_job(job_id, progress=f"Initializing alignment model{device_note}...")
|
||||
if is_no_extract:
|
||||
lsync = LyricsSyncClass(lang="en-US", device=align_device)
|
||||
else:
|
||||
lsync = LyricsSyncClass(lang="en-US")
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
# Run alignment
|
||||
prec_label = "syllable-level" if timing_precision == 'syllable' else "word-level"
|
||||
if LyricsSyncClass is _LyricsSyncNoExtract:
|
||||
prog_msg = f"Aligning lyrics ({prec_label})..."
|
||||
else:
|
||||
prog_msg = f"Extracting vocals and aligning lyrics ({prec_label})... (this may take a while)"
|
||||
jm.update_job(job_id, progress=prog_msg)
|
||||
words, lrc = lsync.sync(audio_path, lyrics_temp_path, save=False, timing_precision=timing_precision)
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
# Convert to ELRC format
|
||||
elrc_content = _convert_to_elrc(words, lyrics_text, timing_precision)
|
||||
|
||||
# Save ELRC file
|
||||
if not output_filename:
|
||||
output_filename = f"{Path(audio_path).stem}.elrc"
|
||||
elif not output_filename.endswith('.elrc'):
|
||||
output_filename += '.elrc'
|
||||
|
||||
output_path = output_dir / output_filename
|
||||
output_path.write_text(elrc_content, encoding='utf-8')
|
||||
|
||||
# Also save plain LRC version
|
||||
lrc_filename = output_filename.replace('.elrc', '.lrc')
|
||||
lrc_path = output_dir / lrc_filename
|
||||
lrc_path.write_text(lrc, encoding='utf-8')
|
||||
|
||||
files = [
|
||||
{"name": output_path.name, "path": str(output_path), "type": "elrc"},
|
||||
{"name": lrc_path.name, "path": str(lrc_path), "type": "lrc"},
|
||||
]
|
||||
|
||||
# Generate styled HTML preview if style options provided
|
||||
if style_options:
|
||||
jm.update_job(job_id, progress="Generating styled preview...")
|
||||
html_content = _generate_styled_html(elrc_content, style_options, Path(audio_path).stem)
|
||||
html_filename = output_filename.replace('.elrc', '.html')
|
||||
html_path = output_dir / html_filename
|
||||
html_path.write_text(html_content, encoding='utf-8')
|
||||
files.append({"name": html_path.name, "path": str(html_path), "type": "html"})
|
||||
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.DONE,
|
||||
progress="ELRC alignment complete.",
|
||||
files=files
|
||||
)
|
||||
|
||||
finally:
|
||||
# Cleanup temp file
|
||||
try:
|
||||
os.unlink(lyrics_temp_path)
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
def _enhance_existing_lrc(
|
||||
job_id: str,
|
||||
audio_path: str,
|
||||
lrc_content: str,
|
||||
output_dir: Path,
|
||||
output_filename: Optional[str],
|
||||
LyricsSyncClass,
|
||||
style_options: Optional[dict] = None,
|
||||
timing_precision: str = 'word',
|
||||
align_device: Optional[str] = None,
|
||||
):
|
||||
"""Enhance existing LRC with word-level timestamps."""
|
||||
from .metadata_util import parse_lrc
|
||||
|
||||
# Parse existing LRC
|
||||
lines = parse_lrc(lrc_content)
|
||||
|
||||
# Get plain lyrics text
|
||||
plain_lyrics = lyrics_to_plaintext(lrc_content)
|
||||
|
||||
# Run alignment
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False, encoding='utf-8') as f:
|
||||
f.write(plain_lyrics)
|
||||
lyrics_temp_path = f.name
|
||||
|
||||
try:
|
||||
if LyricsSyncClass is _LyricsSyncNoExtract:
|
||||
lsync = LyricsSyncClass(lang="en-US", device=align_device)
|
||||
else:
|
||||
lsync = LyricsSyncClass(lang="en-US")
|
||||
|
||||
prec_label = "syllable-level" if timing_precision == 'syllable' else "word-level"
|
||||
if LyricsSyncClass is _LyricsSyncNoExtract:
|
||||
prog_msg = f"Aligning words ({prec_label})..."
|
||||
else:
|
||||
prog_msg = f"Extracting vocals and aligning words ({prec_label})..."
|
||||
jm.update_job(job_id, progress=prog_msg)
|
||||
words, lrc = lsync.sync(audio_path, lyrics_temp_path, save=False, timing_precision=timing_precision)
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
# Convert to ELRC
|
||||
elrc_content = _convert_to_elrc(words, plain_lyrics, timing_precision)
|
||||
|
||||
# Save files
|
||||
if not output_filename:
|
||||
output_filename = f"{Path(audio_path).stem}.elrc"
|
||||
elif not output_filename.endswith('.elrc'):
|
||||
output_filename += '.elrc'
|
||||
|
||||
output_path = output_dir / output_filename
|
||||
output_path.write_text(elrc_content, encoding='utf-8')
|
||||
|
||||
lrc_filename = output_filename.replace('.elrc', '.lrc')
|
||||
lrc_path = output_dir / lrc_filename
|
||||
lrc_path.write_text(lrc, encoding='utf-8')
|
||||
|
||||
files = [
|
||||
{"name": output_path.name, "path": str(output_path), "type": "elrc"},
|
||||
{"name": lrc_path.name, "path": str(lrc_path), "type": "lrc"},
|
||||
]
|
||||
|
||||
# Generate styled HTML preview if style options provided
|
||||
if style_options:
|
||||
jm.update_job(job_id, progress="Generating styled preview...")
|
||||
html_content = _generate_styled_html(elrc_content, style_options, Path(audio_path).stem)
|
||||
html_filename = output_filename.replace('.elrc', '.html')
|
||||
html_path = output_dir / html_filename
|
||||
html_path.write_text(html_content, encoding='utf-8')
|
||||
files.append({"name": html_path.name, "path": str(html_path), "type": "html"})
|
||||
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.DONE,
|
||||
progress="ELRC enhancement complete.",
|
||||
files=files
|
||||
)
|
||||
|
||||
finally:
|
||||
try:
|
||||
os.unlink(lyrics_temp_path)
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
def _fmt_ts(seconds: float) -> str:
|
||||
"""Format a time value as an ELRC inline timestamp <mm:ss.xx>."""
|
||||
m = int(seconds // 60)
|
||||
s = int(seconds % 60)
|
||||
cs = int((seconds % 1) * 100)
|
||||
return f"<{m:02d}:{s:02d}.{cs:02d}>"
|
||||
|
||||
|
||||
def _line_ts(seconds: float) -> str:
|
||||
"""Format a time value as an ELRC line timestamp [mm:ss.xx]."""
|
||||
m = int(seconds // 60)
|
||||
s = int(seconds % 60)
|
||||
cs = int((seconds % 1) * 100)
|
||||
return f"[{m:02d}:{s:02d}.{cs:02d}]"
|
||||
|
||||
|
||||
def _convert_to_elrc(words, original_lyrics: str, timing_precision: str = 'word') -> str:
|
||||
"""
|
||||
Convert aligned words/syllables to ELRC format.
|
||||
|
||||
Word mode → ``[line_ts]<ts>Hello <ts>world``
|
||||
Syllable mode → ``[line_ts]<ts>Hel<ts>lo <ts>world``
|
||||
(no space between intra-word syllables; space between words)
|
||||
|
||||
Args:
|
||||
words: List of Word objects (label, start, end, is_word_end)
|
||||
original_lyrics: Original lyrics text for line structure
|
||||
timing_precision: 'word' or 'syllable'
|
||||
|
||||
Returns:
|
||||
ELRC formatted string
|
||||
"""
|
||||
original_lines = [line.strip() for line in original_lyrics.split('\n') if line.strip()]
|
||||
|
||||
elrc_lines = []
|
||||
word_idx = 0
|
||||
|
||||
for line_text in original_lines:
|
||||
if word_idx >= len(words):
|
||||
break
|
||||
|
||||
orig_words = line_text.split()
|
||||
if not orig_words:
|
||||
continue
|
||||
|
||||
if timing_precision == 'syllable':
|
||||
# Collect all syllables for this line by consuming until is_word_end
|
||||
# for each original word
|
||||
line_syls = []
|
||||
for _ in orig_words:
|
||||
while word_idx < len(words):
|
||||
syl = words[word_idx]
|
||||
line_syls.append(syl)
|
||||
word_idx += 1
|
||||
if getattr(syl, 'is_word_end', False):
|
||||
break
|
||||
|
||||
if not line_syls:
|
||||
continue
|
||||
|
||||
line_start = line_syls[0].start
|
||||
line_content = ""
|
||||
for i, syl in enumerate(line_syls):
|
||||
# Space before a syllable only when the previous one ended a word
|
||||
prefix = " " if i > 0 and getattr(line_syls[i - 1], 'is_word_end', True) else ""
|
||||
line_content += f"{prefix}{_fmt_ts(syl.start)}{syl.label}"
|
||||
|
||||
else:
|
||||
# Word mode: consume one Word per original word
|
||||
line_aligned = []
|
||||
for i in range(len(orig_words)):
|
||||
if word_idx + i < len(words):
|
||||
w = words[word_idx + i]
|
||||
line_aligned.append(w)
|
||||
|
||||
if not line_aligned:
|
||||
continue
|
||||
|
||||
line_start = line_aligned[0].start
|
||||
line_content = " ".join(f"{_fmt_ts(w.start)}{w.label}" for w in line_aligned)
|
||||
word_idx += len(orig_words)
|
||||
|
||||
elrc_lines.append(f"{_line_ts(line_start)}{line_content}")
|
||||
|
||||
return '\n'.join(elrc_lines)
|
||||
|
||||
|
||||
def _generate_styled_html(elrc_content: str, style_options: dict, title: str) -> str:
|
||||
"""
|
||||
Generate an HTML file with styled lyrics preview.
|
||||
|
||||
Args:
|
||||
elrc_content: ELRC formatted lyrics content
|
||||
style_options: Dictionary with style configuration
|
||||
title: Song title for the HTML page
|
||||
|
||||
Returns:
|
||||
HTML content as string
|
||||
"""
|
||||
from .metadata_util import parse_elrc
|
||||
|
||||
# Extract style options with defaults
|
||||
font_family = style_options.get('fontFamily', 'system-ui')
|
||||
font_size = style_options.get('fontSize', 24)
|
||||
text_color = style_options.get('textColor', '#e2e8f0')
|
||||
highlight_color = style_options.get('highlightColor', '#3b82f6')
|
||||
background_color = style_options.get('backgroundColor', '#0f172a')
|
||||
word_highlight_color = style_options.get('wordHighlightColor', '#60a5fa')
|
||||
line_spacing = style_options.get('lineSpacing', 1.5)
|
||||
text_align = style_options.get('textAlign', 'center')
|
||||
|
||||
# Parse ELRC content
|
||||
lines = parse_elrc(elrc_content)
|
||||
|
||||
# Build HTML lines
|
||||
html_lines = []
|
||||
for line_ts, line_text, words in lines:
|
||||
if words:
|
||||
# Build line with word spans for animation
|
||||
word_spans = []
|
||||
for word_ts, word in words:
|
||||
minutes = int(word_ts // 60)
|
||||
seconds = int(word_ts % 60)
|
||||
millis = int((word_ts % 1) * 1000)
|
||||
time_attr = f"{minutes:02d}:{seconds:02d}.{millis:03d}"
|
||||
word_spans.append(
|
||||
f'<span class="word" data-time="{time_attr}">{word}</span>'
|
||||
)
|
||||
line_html = ' '.join(word_spans)
|
||||
else:
|
||||
line_html = line_text
|
||||
|
||||
minutes = int(line_ts // 60)
|
||||
seconds = int(line_ts % 60)
|
||||
millis = int((line_ts % 1) * 1000)
|
||||
line_time = f"{minutes:02d}:{seconds:02d}.{millis:03d}"
|
||||
|
||||
html_lines.append(
|
||||
f'<div class="lyric-line" data-time="{line_time}">{line_html}</div>'
|
||||
)
|
||||
|
||||
lyrics_html = '\n'.join(html_lines)
|
||||
|
||||
html_template = f'''<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>{title} - Lyrics</title>
|
||||
<style>
|
||||
* {{
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}}
|
||||
|
||||
body {{
|
||||
font-family: {font_family};
|
||||
background-color: {background_color};
|
||||
color: {text_color};
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
padding: 2rem 1rem;
|
||||
}}
|
||||
|
||||
.container {{
|
||||
max-width: 800px;
|
||||
width: 100%;
|
||||
}}
|
||||
|
||||
h1 {{
|
||||
font-size: {font_size * 1.5}px;
|
||||
margin-bottom: 2rem;
|
||||
text-align: {text_align};
|
||||
color: {text_color};
|
||||
}}
|
||||
|
||||
.lyrics-container {{
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: {font_size * line_spacing * 0.5}px;
|
||||
}}
|
||||
|
||||
.lyric-line {{
|
||||
font-size: {font_size}px;
|
||||
line-height: {line_spacing};
|
||||
text-align: {text_align};
|
||||
color: {text_color};
|
||||
opacity: 0.6;
|
||||
transition: all 0.3s ease;
|
||||
padding: 0.5rem;
|
||||
border-radius: 0.5rem;
|
||||
}}
|
||||
|
||||
.lyric-line.active {{
|
||||
color: {highlight_color};
|
||||
opacity: 1;
|
||||
transform: scale(1.02);
|
||||
}}
|
||||
|
||||
.word {{
|
||||
display: inline-block;
|
||||
transition: color 0.2s ease;
|
||||
padding: 0 0.1em;
|
||||
}}
|
||||
|
||||
.word.active {{
|
||||
color: {word_highlight_color};
|
||||
text-shadow: 0 0 10px {word_highlight_color}40;
|
||||
}}
|
||||
|
||||
.controls {{
|
||||
position: fixed;
|
||||
bottom: 2rem;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
background: rgba(0, 0, 0, 0.8);
|
||||
padding: 1rem 2rem;
|
||||
border-radius: 2rem;
|
||||
display: flex;
|
||||
gap: 1rem;
|
||||
align-items: center;
|
||||
backdrop-filter: blur(10px);
|
||||
}}
|
||||
|
||||
button {{
|
||||
background: {highlight_color};
|
||||
color: white;
|
||||
border: none;
|
||||
padding: 0.5rem 1rem;
|
||||
border-radius: 0.5rem;
|
||||
cursor: pointer;
|
||||
font-size: 14px;
|
||||
transition: opacity 0.2s;
|
||||
}}
|
||||
|
||||
button:hover {{
|
||||
opacity: 0.8;
|
||||
}}
|
||||
|
||||
input[type="range"] {{
|
||||
width: 200px;
|
||||
}}
|
||||
|
||||
.time-display {{
|
||||
font-family: monospace;
|
||||
font-size: 14px;
|
||||
color: {text_color};
|
||||
}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>{title}</h1>
|
||||
<div class="lyrics-container">
|
||||
{lyrics_html}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="controls">
|
||||
<button id="playBtn">▶ Play</button>
|
||||
<input type="range" id="progress" min="0" max="100" value="0">
|
||||
<span class="time-display" id="timeDisplay">00:00</span>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const lines = document.querySelectorAll('.lyric-line');
|
||||
const words = document.querySelectorAll('.word');
|
||||
const playBtn = document.getElementById('playBtn');
|
||||
const progress = document.getElementById('progress');
|
||||
const timeDisplay = document.getElementById('timeDisplay');
|
||||
|
||||
let isPlaying = false;
|
||||
let currentTime = 0;
|
||||
// Derive duration from the last timestamp in the ELRC content
|
||||
const allTimestamps = Array.from(document.querySelectorAll('[data-start]'))
|
||||
.map(el => parseFloat(el.getAttribute('data-start')))
|
||||
.filter(t => !isNaN(t));
|
||||
let duration = allTimestamps.length > 0 ? Math.ceil(Math.max(...allTimestamps) + 10) : 300;
|
||||
let animationId = null;
|
||||
|
||||
// Parse time string to seconds
|
||||
function parseTime(timeStr) {{
|
||||
const [mins, secs] = timeStr.split(':');
|
||||
return parseInt(mins) * 60 + parseFloat(secs);
|
||||
}}
|
||||
|
||||
// Format seconds to time string
|
||||
function formatTime(seconds) {{
|
||||
const mins = Math.floor(seconds / 60);
|
||||
const secs = Math.floor(seconds % 60);
|
||||
return `${{mins.toString().padStart(2, '0')}}:${{secs.toString().padStart(2, '0')}}`;
|
||||
}}
|
||||
|
||||
// Track last-active elements to avoid redundant DOM updates
|
||||
let _activeLine = null;
|
||||
let _activeWord = null;
|
||||
|
||||
// Update active line/word based on current time; scroll active line into view
|
||||
function updateActiveElements() {{
|
||||
// Find the last line whose timestamp <= currentTime
|
||||
let newLine = null;
|
||||
lines.forEach(line => {{
|
||||
if (currentTime >= parseTime(line.dataset.time)) newLine = line;
|
||||
}});
|
||||
|
||||
if (newLine !== _activeLine) {{
|
||||
if (_activeLine) _activeLine.classList.remove('active');
|
||||
if (newLine) {{
|
||||
newLine.classList.add('active');
|
||||
const ctrl = document.querySelector('.controls');
|
||||
const ctrlH = ctrl ? ctrl.offsetHeight + 32 : 80;
|
||||
const rect = newLine.getBoundingClientRect();
|
||||
const lineMid = rect.top + rect.height / 2;
|
||||
const availH = window.innerHeight - ctrlH;
|
||||
const target = window.scrollY + lineMid - availH / 2;
|
||||
window.scrollTo({{ top: Math.max(0, target), behavior: 'smooth' }});
|
||||
}}
|
||||
_activeLine = newLine;
|
||||
}}
|
||||
|
||||
// Find the last word whose timestamp <= currentTime
|
||||
let newWord = null;
|
||||
words.forEach(word => {{
|
||||
if (currentTime >= parseTime(word.dataset.time)) newWord = word;
|
||||
}});
|
||||
|
||||
if (newWord !== _activeWord) {{
|
||||
if (_activeWord) _activeWord.classList.remove('active');
|
||||
if (newWord) newWord.classList.add('active');
|
||||
_activeWord = newWord;
|
||||
}}
|
||||
}}
|
||||
|
||||
// Animation loop — use performance.now() delta for accurate wall-clock timing
|
||||
let _lastTs = null;
|
||||
function animate(ts) {{
|
||||
if (isPlaying) {{
|
||||
if (_lastTs !== null) currentTime += (ts - _lastTs) / 1000;
|
||||
_lastTs = ts;
|
||||
if (currentTime > duration) {{
|
||||
currentTime = 0;
|
||||
isPlaying = false;
|
||||
playBtn.textContent = '▶ Play';
|
||||
_lastTs = null;
|
||||
}}
|
||||
progress.value = (currentTime / duration) * 100;
|
||||
timeDisplay.textContent = formatTime(currentTime);
|
||||
updateActiveElements();
|
||||
}} else {{
|
||||
_lastTs = null;
|
||||
}}
|
||||
animationId = requestAnimationFrame(animate);
|
||||
}}
|
||||
|
||||
// Play/Pause toggle
|
||||
playBtn.addEventListener('click', () => {{
|
||||
isPlaying = !isPlaying;
|
||||
playBtn.textContent = isPlaying ? '⏸ Pause' : '▶ Play';
|
||||
}});
|
||||
|
||||
// Progress bar interaction
|
||||
progress.addEventListener('input', (e) => {{
|
||||
currentTime = (e.target.value / 100) * duration;
|
||||
timeDisplay.textContent = formatTime(currentTime);
|
||||
updateActiveElements();
|
||||
}});
|
||||
|
||||
// Start animation loop
|
||||
animate();
|
||||
</script>
|
||||
</body>
|
||||
</html>'''
|
||||
|
||||
return html_template
|
||||
|
||||
|
||||
def extract_vocals_for_alignment(
|
||||
job_id: str,
|
||||
audio_path: str,
|
||||
output_dir: Path
|
||||
) -> str:
|
||||
"""
|
||||
Extract vocals from audio file for alignment purposes.
|
||||
Returns path to extracted vocal file.
|
||||
"""
|
||||
jm.update_job(job_id, progress="Extracting vocals...")
|
||||
|
||||
try:
|
||||
import torch
|
||||
from demucs.pretrained import get_model
|
||||
from demucs.apply import apply_model
|
||||
from demucs.separate import load_track
|
||||
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
model = get_model(name="htdemucs", repo=None)
|
||||
model.to(device)
|
||||
model.eval()
|
||||
|
||||
vocals_idx = model.sources.index("vocals")
|
||||
|
||||
# Load audio
|
||||
wav = load_track(audio_path, 2, model.samplerate)
|
||||
|
||||
# Separate
|
||||
ref = wav.mean(0)
|
||||
wav = (wav - ref.mean()) / ref.std()
|
||||
|
||||
with torch.no_grad():
|
||||
sources = apply_model(
|
||||
model, wav[None], device=device, shifts=1,
|
||||
split=True, overlap=0.25, progress=False
|
||||
)
|
||||
|
||||
vocals = sources[0][vocals_idx]
|
||||
vocals_np = vocals.cpu().numpy()[0, ...]
|
||||
|
||||
# Save vocals
|
||||
import soundfile as sf
|
||||
vocals_path = output_dir / f"{Path(audio_path).stem}_vocals.wav"
|
||||
sf.write(vocals_path, vocals_np, model.samplerate)
|
||||
|
||||
return str(vocals_path)
|
||||
|
||||
except Exception as e:
|
||||
# If vocal extraction fails, return original audio
|
||||
jm.update_job(job_id, progress=f"Vocal extraction failed, using original audio: {e}")
|
||||
return audio_path
|
||||
+920
@@ -0,0 +1,920 @@
|
||||
"""
|
||||
Lyric transcription service using OpenAI Whisper.
|
||||
Based on Scriberr's transcription approach - provides word-level timestamps
|
||||
and speaker diarization support for lyrics.
|
||||
|
||||
This service transcribes vocals to text with precise timing information,
|
||||
suitable for generating LRC and ELRC files.
|
||||
"""
|
||||
import contextlib
|
||||
import threading
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union
|
||||
from dataclasses import dataclass
|
||||
|
||||
# Whisper's timing.py tries to JIT-compile Triton CUDA kernels for DTW alignment.
|
||||
# This requires the full CUDA toolkit (nvcc), which is not present in the runtime
|
||||
# image. The fallback (NumPy median kernel) is correct and only marginally slower,
|
||||
# so suppress the noisy UserWarning.
|
||||
warnings.filterwarnings(
|
||||
"ignore",
|
||||
message="Failed to launch Triton kernels",
|
||||
category=UserWarning,
|
||||
)
|
||||
|
||||
# PyTorch 2.6 changed torch.load's default from weights_only=False to True.
|
||||
# WhisperX / wav2vec2 / pyannote checkpoints contain arbitrary omegaconf and
|
||||
# other custom types that are not in the default allowlist, and chasing each
|
||||
# class individually is fragile. Restore the pre-2.6 default (weights_only
|
||||
# not set by caller → False) by monkey-patching torch.load at startup.
|
||||
# All checkpoint sources used here (HuggingFace, faster-whisper, pyannote)
|
||||
# are trusted, so the security trade-off is acceptable.
|
||||
try:
|
||||
import torch as _torch
|
||||
_orig_torch_load = _torch.load
|
||||
def _patched_torch_load(*args, **kwargs):
|
||||
# Force weights_only=False regardless of what the caller passed.
|
||||
# transformers explicitly passes weights_only=True (not just as a
|
||||
# default), so setdefault is insufficient — we must overwrite it.
|
||||
# All checkpoint sources used here (HuggingFace, faster-whisper,
|
||||
# pyannote) are trusted, so the security trade-off is acceptable.
|
||||
kwargs['weights_only'] = False
|
||||
return _orig_torch_load(*args, **kwargs)
|
||||
_torch.load = _patched_torch_load
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import JOBS_DIR, LRC_SYNC_DIR
|
||||
|
||||
|
||||
@dataclass
|
||||
class WordTimestamp:
|
||||
"""Word-level timestamp information."""
|
||||
word: str
|
||||
start: float
|
||||
end: float
|
||||
confidence: Optional[float] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Segment:
|
||||
"""Segment (line/phrase) level timestamp information."""
|
||||
text: str
|
||||
start: float
|
||||
end: float
|
||||
words: List[WordTimestamp]
|
||||
speaker: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class LyricTranscriptionResult:
|
||||
"""Complete transcription result."""
|
||||
text: str
|
||||
segments: List[Segment]
|
||||
language: str
|
||||
duration: float
|
||||
model: str
|
||||
|
||||
|
||||
# Available Whisper models with their characteristics
|
||||
WHISPER_MODELS = {
|
||||
"tiny": {
|
||||
"id": "tiny",
|
||||
"label": "Tiny (fastest, least accurate)",
|
||||
"size": "39 MB",
|
||||
"relative_speed": "~32x",
|
||||
"english_only": False,
|
||||
},
|
||||
"base": {
|
||||
"id": "base",
|
||||
"label": "Base (fast, good accuracy)",
|
||||
"size": "74 MB",
|
||||
"relative_speed": "~16x",
|
||||
"english_only": False,
|
||||
},
|
||||
"small": {
|
||||
"id": "small",
|
||||
"label": "Small (balanced)",
|
||||
"size": "244 MB",
|
||||
"relative_speed": "~6x",
|
||||
"english_only": False,
|
||||
},
|
||||
"medium": {
|
||||
"id": "medium",
|
||||
"label": "Medium (slower, better accuracy)",
|
||||
"size": "769 MB",
|
||||
"relative_speed": "~2x",
|
||||
"english_only": False,
|
||||
},
|
||||
"large": {
|
||||
"id": "large",
|
||||
"label": "Large (slowest, best accuracy)",
|
||||
"size": "1550 MB",
|
||||
"relative_speed": "1x",
|
||||
"english_only": False,
|
||||
},
|
||||
"large-v2": {
|
||||
"id": "large-v2",
|
||||
"label": "Large v2 (recommended)",
|
||||
"size": "1550 MB",
|
||||
"relative_speed": "1x",
|
||||
"english_only": False,
|
||||
},
|
||||
"large-v3": {
|
||||
"id": "large-v3",
|
||||
"label": "Large v3 (latest)",
|
||||
"size": "1550 MB",
|
||||
"relative_speed": "1x",
|
||||
"english_only": False,
|
||||
},
|
||||
}
|
||||
|
||||
DEFAULT_WHISPER_MODEL = "large-v2"
|
||||
|
||||
|
||||
_whisper_cache: Dict[str, Any] = {}
|
||||
|
||||
# Set to False when a CUDA OOM or "unknown error" corrupts the GPU context.
|
||||
# Once False, all subsequent Whisper loads are forced to CPU so the server
|
||||
# keeps working without requiring a restart.
|
||||
_cuda_ok: bool = True
|
||||
|
||||
|
||||
def _mark_cuda_failed() -> None:
|
||||
"""Record that the CUDA context is bad and purge any GPU-resident models."""
|
||||
global _cuda_ok
|
||||
_cuda_ok = False
|
||||
_whisper_cache.clear()
|
||||
try:
|
||||
import torch
|
||||
torch.cuda.empty_cache()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def reset_cuda() -> dict:
|
||||
"""
|
||||
Re-enable GPU acceleration after a CUDA error recovery.
|
||||
|
||||
The transcriber falls back to CPU when a CUDA OOM or 'unknown error'
|
||||
occurs. Call this once the GPU is confirmed free to restore GPU usage
|
||||
without restarting the container.
|
||||
"""
|
||||
global _cuda_ok
|
||||
_cuda_ok = True
|
||||
_whisper_cache.clear()
|
||||
cuda_available = False
|
||||
try:
|
||||
import torch
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
cuda_available = True
|
||||
except Exception:
|
||||
pass
|
||||
return {"ok": True, "cuda_available": cuda_available, "cuda_ok": _cuda_ok}
|
||||
|
||||
|
||||
def get_whisper_model(model_name: str = DEFAULT_WHISPER_MODEL, device: Optional[str] = None):
|
||||
"""
|
||||
Load and in-process-cache a Whisper model.
|
||||
|
||||
Reuses an already-loaded model of the same name to avoid reloading ~1.5 GB
|
||||
weights on every transcription call. If a *different* model is requested,
|
||||
the cached one is evicted and the CUDA allocator cache is flushed first so
|
||||
that the new model can fit alongside any other resident GPU allocations.
|
||||
|
||||
When `device` is explicitly passed (e.g. "cpu") it overrides the auto-detect
|
||||
logic. This is used by the CUDA-error recovery path in `transcribe_lyrics`.
|
||||
"""
|
||||
import whisper
|
||||
|
||||
cache_key = f"{model_name}:{device or 'auto'}"
|
||||
|
||||
if cache_key in _whisper_cache:
|
||||
return _whisper_cache[cache_key]
|
||||
|
||||
# Evict any previously cached model and release its GPU memory before
|
||||
# loading the new one.
|
||||
if _whisper_cache:
|
||||
_whisper_cache.clear()
|
||||
try:
|
||||
import torch
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
load_kwargs: dict = {}
|
||||
if device is not None:
|
||||
load_kwargs["device"] = device
|
||||
|
||||
model = whisper.load_model(model_name, **load_kwargs)
|
||||
_whisper_cache[cache_key] = model
|
||||
return model
|
||||
|
||||
|
||||
class _LineCapture(io.RawIOBase):
|
||||
"""
|
||||
File-like object that buffers writes and calls `callback` once per complete line.
|
||||
Used to intercept Whisper's verbose stdout output for live progress reporting.
|
||||
"""
|
||||
def __init__(self, callback: Callable[[str], None]) -> None:
|
||||
self._cb = callback
|
||||
self._buf = ''
|
||||
|
||||
def write(self, text) -> int: # type: ignore[override]
|
||||
if isinstance(text, (bytes, bytearray)):
|
||||
text = text.decode('utf-8', errors='replace')
|
||||
self._buf += text
|
||||
while '\n' in self._buf:
|
||||
line, self._buf = self._buf.split('\n', 1)
|
||||
cleaned = line.strip()
|
||||
if cleaned:
|
||||
self._cb(cleaned)
|
||||
return len(text)
|
||||
|
||||
def flush(self) -> None:
|
||||
pass
|
||||
|
||||
# Some libraries call fileno(); raise so they fall back gracefully.
|
||||
def fileno(self): # type: ignore[override]
|
||||
raise io.UnsupportedOperation('fileno')
|
||||
|
||||
|
||||
|
||||
def transcribe_lyrics(
|
||||
audio_path: str,
|
||||
model_name: str = DEFAULT_WHISPER_MODEL,
|
||||
language: Optional[str] = None,
|
||||
task: Literal["transcribe", "translate"] = "transcribe",
|
||||
include_word_timestamps: bool = True,
|
||||
initial_prompt: Optional[str] = None,
|
||||
temperature: float = 0.0,
|
||||
condition_on_previous_text: bool = False,
|
||||
compression_ratio_threshold: float = 1.35,
|
||||
beam_size: Optional[int] = None,
|
||||
line_callback: Optional[Callable[[str], None]] = None,
|
||||
) -> LyricTranscriptionResult:
|
||||
"""
|
||||
Transcribe vocals to lyrics with word-level timestamps.
|
||||
|
||||
Anti-hallucination defaults (differ from stock Whisper):
|
||||
- condition_on_previous_text=False — prevents the model from feeding
|
||||
a repeated phrase back as context, breaking the repetition loop.
|
||||
- temperature fallback tuple — when temperature=0.0, Whisper is
|
||||
automatically given (0.0, 0.2, 0.4, 0.6, 0.8, 1.0) so it can retry
|
||||
a stuck segment with increasing randomness.
|
||||
- compression_ratio_threshold=1.35 — flags repetitive segments sooner
|
||||
(stock default is 2.4) and triggers the temperature fallback.
|
||||
- _deduplicate_segments post-pass — removes any repeated segments or
|
||||
internally-looped phrases that still slip through.
|
||||
|
||||
Args:
|
||||
audio_path: Path to audio file (should be vocals stem for best results)
|
||||
model_name: Whisper model to use
|
||||
language: ISO-639-1 language code (auto-detected if None)
|
||||
task: 'transcribe' for original language, 'translate' for English
|
||||
include_word_timestamps: Whether to include word-level timestamps
|
||||
initial_prompt: Optional text to guide the model's style
|
||||
temperature: Base sampling temperature; 0.0 enables the fallback tuple
|
||||
condition_on_previous_text: Feed previous segment as context (False prevents loops)
|
||||
compression_ratio_threshold: Retry segment above this gzip ratio (lower = stricter)
|
||||
|
||||
Returns:
|
||||
LyricTranscriptionResult with text, segments, and timestamps
|
||||
"""
|
||||
import whisper
|
||||
|
||||
# Free any reserved-but-unallocated GPU memory from prior jobs (e.g. Demucs,
|
||||
# BS-RoFormer) before the Whisper model is loaded / used.
|
||||
try:
|
||||
import torch
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# If a previous job corrupted the CUDA context (OOM → "unknown error"),
|
||||
# force CPU so this job still completes rather than failing immediately.
|
||||
forced_device: Optional[str] = None if _cuda_ok else "cpu"
|
||||
model = get_whisper_model(model_name, device=forced_device)
|
||||
|
||||
# When temperature is 0 (greedy), pass a fallback sequence so Whisper can
|
||||
# recover from stuck segments by retrying with increasing randomness.
|
||||
effective_temperature: Any = (0.0, 0.2, 0.4, 0.6, 0.8, 1.0) if temperature == 0.0 else temperature
|
||||
|
||||
decode_options: dict = {
|
||||
"task": task,
|
||||
"temperature": effective_temperature,
|
||||
"condition_on_previous_text": condition_on_previous_text,
|
||||
"compression_ratio_threshold": compression_ratio_threshold,
|
||||
}
|
||||
|
||||
if language:
|
||||
decode_options["language"] = language
|
||||
if initial_prompt:
|
||||
decode_options["initial_prompt"] = initial_prompt
|
||||
if beam_size is not None:
|
||||
decode_options["beam_size"] = beam_size
|
||||
|
||||
def _run_transcribe(mdl: Any) -> Any:
|
||||
"""Run model.transcribe(), capturing verbose output when a callback is set."""
|
||||
if line_callback is not None:
|
||||
_cap = _LineCapture(line_callback)
|
||||
with contextlib.redirect_stdout(_cap), contextlib.redirect_stderr(_cap):
|
||||
return mdl.transcribe(
|
||||
audio_path,
|
||||
word_timestamps=include_word_timestamps,
|
||||
verbose=True,
|
||||
**decode_options,
|
||||
)
|
||||
return mdl.transcribe(
|
||||
audio_path,
|
||||
word_timestamps=include_word_timestamps,
|
||||
**decode_options,
|
||||
)
|
||||
|
||||
# Run transcription with word timestamps.
|
||||
# On any CUDA error (OOM already happened, or the context was already
|
||||
# corrupted), mark the GPU as failed, reload the model on CPU, and retry.
|
||||
# This keeps the server usable without a restart.
|
||||
try:
|
||||
result = _run_transcribe(model)
|
||||
except RuntimeError as exc:
|
||||
err_str = str(exc)
|
||||
if "CUDA" in err_str or "cuda" in err_str:
|
||||
_mark_cuda_failed()
|
||||
if line_callback:
|
||||
line_callback("⚠️ CUDA error — retrying on CPU (server restart not required)")
|
||||
cpu_model = get_whisper_model(model_name, device="cpu")
|
||||
result = _run_transcribe(cpu_model)
|
||||
else:
|
||||
raise
|
||||
|
||||
# Parse results into structured format
|
||||
raw_segments: List[Segment] = []
|
||||
for seg in result["segments"]:
|
||||
words = []
|
||||
if include_word_timestamps and "words" in seg:
|
||||
for word_info in seg["words"]:
|
||||
words.append(WordTimestamp(
|
||||
word=word_info["word"].strip(),
|
||||
start=word_info["start"],
|
||||
end=word_info["end"],
|
||||
confidence=word_info.get("probability")
|
||||
))
|
||||
|
||||
raw_segments.append(Segment(
|
||||
text=seg["text"].strip(),
|
||||
start=seg["start"],
|
||||
end=seg["end"],
|
||||
words=words
|
||||
))
|
||||
|
||||
# Calculate total duration
|
||||
duration = max(seg.end for seg in raw_segments) if raw_segments else 0.0
|
||||
|
||||
return LyricTranscriptionResult(
|
||||
text=result["text"].strip(),
|
||||
segments=raw_segments,
|
||||
language=result.get("language", "unknown"),
|
||||
duration=duration,
|
||||
model=f"whisper-{model_name}"
|
||||
)
|
||||
|
||||
|
||||
def result_to_lrc(result: LyricTranscriptionResult) -> str:
|
||||
"""Convert transcription result to LRC format (line-level timestamps)."""
|
||||
lines = []
|
||||
for seg in result.segments:
|
||||
minutes = int(seg.start // 60)
|
||||
seconds = int(seg.start % 60)
|
||||
centis = int((seg.start % 1) * 100)
|
||||
lines.append(f"[{minutes:02d}:{seconds:02d}.{centis:02d}]{seg.text}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def result_to_elrc(result: LyricTranscriptionResult) -> str:
|
||||
"""Convert transcription result to ELRC format (word-level timestamps)."""
|
||||
lines = []
|
||||
for seg in result.segments:
|
||||
if not seg.words:
|
||||
# Fallback to segment-level if no word timestamps
|
||||
minutes = int(seg.start // 60)
|
||||
seconds = int(seg.start % 60)
|
||||
centis = int((seg.start % 1) * 100)
|
||||
lines.append(f"[{minutes:02d}:{seconds:02d}.{centis:02d}]{seg.text}")
|
||||
continue
|
||||
|
||||
# Line timestamp is start of first word
|
||||
line_start = seg.words[0].start
|
||||
line_min = int(line_start // 60)
|
||||
line_sec = int(line_start % 60)
|
||||
line_centi = int((line_start % 1) * 100)
|
||||
|
||||
# Build word-level timestamps
|
||||
word_parts = []
|
||||
for word in seg.words:
|
||||
w_min = int(word.start // 60)
|
||||
w_sec = int(word.start % 60)
|
||||
w_centi = int((word.start % 1) * 100)
|
||||
word_parts.append(f"<{w_min:02d}:{w_sec:02d}.{w_centi:02d}>{word.word}")
|
||||
|
||||
lines.append(f"[{line_min:02d}:{line_sec:02d}.{line_centi:02d}]{' '.join(word_parts)}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def result_to_json(result: LyricTranscriptionResult) -> Dict[str, Any]:
|
||||
"""Convert transcription result to JSON-serializable dict."""
|
||||
return {
|
||||
"text": result.text,
|
||||
"language": result.language,
|
||||
"duration": result.duration,
|
||||
"model": result.model,
|
||||
"segments": [
|
||||
{
|
||||
"text": seg.text,
|
||||
"start": seg.start,
|
||||
"end": seg.end,
|
||||
"speaker": seg.speaker,
|
||||
"words": [
|
||||
{
|
||||
"word": w.word,
|
||||
"start": w.start,
|
||||
"end": w.end,
|
||||
"confidence": w.confidence
|
||||
}
|
||||
for w in seg.words
|
||||
]
|
||||
}
|
||||
for seg in result.segments
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def result_to_txt(result: LyricTranscriptionResult) -> str:
|
||||
"""Convert transcription result to plain text.
|
||||
|
||||
One Whisper segment per line so the output can be fed directly into the
|
||||
ctc-forced-aligner (lrc_service) which splits on newlines to identify
|
||||
individual lyric lines. Falls back to the joined text if no segments.
|
||||
"""
|
||||
if result.segments:
|
||||
return '\n'.join(seg.text.strip() for seg in result.segments if seg.text.strip())
|
||||
return result.text
|
||||
|
||||
|
||||
def result_to_srt(result: LyricTranscriptionResult) -> str:
|
||||
"""Convert transcription result to SRT subtitle format."""
|
||||
lines = []
|
||||
for i, seg in enumerate(result.segments, 1):
|
||||
# Format: HH:MM:SS,mmm
|
||||
def fmt_srt_time(t: float) -> str:
|
||||
hours = int(t // 3600)
|
||||
mins = int((t % 3600) // 60)
|
||||
secs = int(t % 60)
|
||||
millis = int((t % 1) * 1000)
|
||||
return f"{hours:02d}:{mins:02d}:{secs:02d},{millis:03d}"
|
||||
|
||||
lines.append(str(i))
|
||||
lines.append(f"{fmt_srt_time(seg.start)} --> {fmt_srt_time(seg.end)}")
|
||||
lines.append(seg.text)
|
||||
lines.append("") # Empty line between entries
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def result_to_vtt(result: LyricTranscriptionResult) -> str:
|
||||
"""Convert transcription result to WebVTT subtitle format."""
|
||||
lines = ["WEBVTT", ""]
|
||||
|
||||
for seg in result.segments:
|
||||
def fmt_vtt_time(t: float) -> str:
|
||||
hours = int(t // 3600)
|
||||
mins = int((t % 3600) // 60)
|
||||
secs = int(t % 60)
|
||||
millis = int((t % 1) * 1000)
|
||||
return f"{hours:02d}:{mins:02d}:{secs:02d}.{millis:03d}"
|
||||
|
||||
lines.append(f"{fmt_vtt_time(seg.start)} --> {fmt_vtt_time(seg.end)}")
|
||||
lines.append(seg.text)
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def run_lyric_transcription(
|
||||
job_id: str,
|
||||
audio_path: str,
|
||||
model: str = DEFAULT_WHISPER_MODEL,
|
||||
language: Optional[str] = None,
|
||||
task: str = "transcribe",
|
||||
output_formats: Optional[List[str]] = None,
|
||||
initial_prompt: Optional[str] = None,
|
||||
temperature: float = 0.0,
|
||||
condition_on_previous_text: bool = False,
|
||||
word_timestamps: bool = True,
|
||||
beam_size: Optional[int] = None,
|
||||
output_dir: Optional[str] = None,
|
||||
engine: str = "whisperx",
|
||||
diarize: bool = False,
|
||||
):
|
||||
"""
|
||||
Background task: transcribe vocals/lyrics from audio.
|
||||
|
||||
Args:
|
||||
job_id: Job identifier
|
||||
audio_path: Path to audio file
|
||||
model: Whisper model name
|
||||
language: Language code (auto-detect if None)
|
||||
task: 'transcribe' or 'translate'
|
||||
output_formats: List of output formats ('lrc', 'elrc', 'json', 'txt', 'srt', 'vtt')
|
||||
initial_prompt: Optional prompt text
|
||||
temperature: Sampling temperature
|
||||
output_dir: Custom output directory
|
||||
engine: 'whisperx' (faster-whisper + phoneme alignment)
|
||||
diarize: Run speaker diarization after alignment (WhisperX only)
|
||||
"""
|
||||
if output_formats is None:
|
||||
output_formats = ["elrc", "json", "txt"]
|
||||
|
||||
if output_dir:
|
||||
out_path = Path(output_dir)
|
||||
else:
|
||||
out_path = LRC_SYNC_DIR / job_id
|
||||
out_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
if engine == "whisperx":
|
||||
# ── WhisperX path: faster-whisper backend + phoneme-level alignment ──
|
||||
try:
|
||||
import torch
|
||||
import whisperx
|
||||
except ImportError as _imp_err:
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.ERROR,
|
||||
error=f"WhisperX is not installed ({_imp_err}). "
|
||||
"Install with: pip install whisperx",
|
||||
progress="Error.",
|
||||
)
|
||||
return
|
||||
|
||||
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress=f"Loading WhisperX model ({model})...")
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
compute_type = "float16" if device == "cuda" else "int8"
|
||||
wx_model = whisperx.load_model(model, device, compute_type=compute_type, language=language or None)
|
||||
audio_arr = whisperx.load_audio(audio_path)
|
||||
duration = float(len(audio_arr)) / 16000.0
|
||||
|
||||
jm.update_job(job_id, progress=f"Transcribing with WhisperX ({model})...")
|
||||
wx_result = wx_model.transcribe(audio_arr, batch_size=16)
|
||||
detected_lang = wx_result.get("language") or language or "unknown"
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
jm.update_job(job_id, progress="Aligning word timestamps (WhisperX)...")
|
||||
model_a, metadata = whisperx.load_align_model(language_code=detected_lang, device=device)
|
||||
wx_result = whisperx.align(
|
||||
wx_result["segments"], model_a, metadata, audio_arr, device,
|
||||
return_char_alignments=False,
|
||||
)
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
if diarize:
|
||||
jm.update_job(job_id, progress="Running speaker diarization...")
|
||||
try:
|
||||
import os
|
||||
hf_token = os.environ.get("HF_TOKEN") or None
|
||||
diarize_model = whisperx.DiarizationPipeline(use_auth_token=hf_token, device=device)
|
||||
diarize_segments = diarize_model(audio_arr)
|
||||
wx_result = whisperx.assign_word_speakers(diarize_segments, wx_result)
|
||||
except Exception as _diar_err:
|
||||
jm.update_job(job_id, progress=f"Diarization failed ({_diar_err}), continuing without speaker labels...")
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
# Convert WhisperX output → LyricTranscriptionResult
|
||||
segments: List[Segment] = []
|
||||
for seg in wx_result["segments"]:
|
||||
words = []
|
||||
for w in seg.get("words", []):
|
||||
words.append(WordTimestamp(
|
||||
word=w.get("word", ""),
|
||||
start=float(w.get("start") or seg["start"]),
|
||||
end=float(w.get("end") or seg["end"]),
|
||||
confidence=w.get("score"),
|
||||
))
|
||||
segments.append(Segment(
|
||||
text=seg["text"].strip(),
|
||||
start=float(seg["start"]),
|
||||
end=float(seg["end"]),
|
||||
words=words,
|
||||
speaker=seg.get("speaker"),
|
||||
))
|
||||
|
||||
result = LyricTranscriptionResult(
|
||||
text=" ".join(seg.text for seg in segments),
|
||||
segments=segments,
|
||||
language=detected_lang,
|
||||
duration=duration,
|
||||
model=f"whisperx-{model}",
|
||||
)
|
||||
|
||||
else:
|
||||
# ── Standard Whisper path ─────────────────────────────────────────────
|
||||
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress=f"Loading Whisper model ({model})...")
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
jm.update_job(job_id, progress=f"Transcribing lyrics with Whisper ({model})...")
|
||||
|
||||
# Accumulate Whisper's verbose segment output for live progress display.
|
||||
# Noisy lines (tqdm bars, empty lines) are filtered out.
|
||||
_log: List[str] = []
|
||||
_NOISE_TOKENS = ('%|', 'it/s', 'B/s', '100%')
|
||||
|
||||
def _on_line(line: str) -> None:
|
||||
if any(tok in line for tok in _NOISE_TOKENS):
|
||||
return
|
||||
_log.append(line)
|
||||
if len(_log) > 30:
|
||||
_log.pop(0)
|
||||
jm.update_job(job_id, progress='\n'.join(_log))
|
||||
|
||||
_TRANSCRIBE_TIMEOUT = 7200 # 2-hour hard cap
|
||||
_result_box: list = [None]
|
||||
_exc_box: list = [None]
|
||||
|
||||
def _do_transcribe():
|
||||
try:
|
||||
_result_box[0] = transcribe_lyrics(
|
||||
audio_path=audio_path,
|
||||
model_name=model,
|
||||
language=language,
|
||||
task=task, # type: ignore
|
||||
include_word_timestamps=word_timestamps,
|
||||
initial_prompt=initial_prompt,
|
||||
temperature=temperature,
|
||||
condition_on_previous_text=condition_on_previous_text,
|
||||
beam_size=beam_size,
|
||||
line_callback=_on_line,
|
||||
)
|
||||
except Exception as _e:
|
||||
_exc_box[0] = _e
|
||||
|
||||
_t = threading.Thread(target=_do_transcribe, daemon=True)
|
||||
_t.start()
|
||||
_t.join(timeout=_TRANSCRIBE_TIMEOUT)
|
||||
|
||||
if _t.is_alive():
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.ERROR,
|
||||
error=f"Transcription timed out after {_TRANSCRIBE_TIMEOUT // 3600}h. "
|
||||
"The model is still running in the background and will release "
|
||||
"the GPU when it finishes.",
|
||||
progress="Timed out.",
|
||||
)
|
||||
return
|
||||
|
||||
if _exc_box[0] is not None:
|
||||
raise _exc_box[0]
|
||||
|
||||
result = _result_box[0]
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
# Generate output files
|
||||
jm.update_job(job_id, progress="Generating output files...")
|
||||
|
||||
base_name = Path(audio_path).stem
|
||||
output_files = []
|
||||
|
||||
# JSON (full data)
|
||||
if "json" in output_formats:
|
||||
json_path = out_path / f"{base_name}_transcription.json"
|
||||
json_data = result_to_json(result)
|
||||
json_path.write_text(json.dumps(json_data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
output_files.append({"name": json_path.name, "path": str(json_path), "type": "json"})
|
||||
|
||||
# LRC (line-level)
|
||||
if "lrc" in output_formats:
|
||||
lrc_path = out_path / f"{base_name}.lrc"
|
||||
lrc_content = result_to_lrc(result)
|
||||
lrc_path.write_text(lrc_content, encoding="utf-8")
|
||||
output_files.append({"name": lrc_path.name, "path": str(lrc_path), "type": "lrc"})
|
||||
|
||||
# ELRC (word-level)
|
||||
if "elrc" in output_formats:
|
||||
elrc_path = out_path / f"{base_name}.elrc"
|
||||
elrc_content = result_to_elrc(result)
|
||||
elrc_path.write_text(elrc_content, encoding="utf-8")
|
||||
output_files.append({"name": elrc_path.name, "path": str(elrc_path), "type": "elrc"})
|
||||
|
||||
# Plain text
|
||||
if "txt" in output_formats:
|
||||
txt_path = out_path / f"{base_name}_lyrics.txt"
|
||||
txt_content = result_to_txt(result)
|
||||
txt_path.write_text(txt_content, encoding="utf-8")
|
||||
output_files.append({"name": txt_path.name, "path": str(txt_path), "type": "txt"})
|
||||
|
||||
# SRT subtitles
|
||||
if "srt" in output_formats:
|
||||
srt_path = out_path / f"{base_name}.srt"
|
||||
srt_content = result_to_srt(result)
|
||||
srt_path.write_text(srt_content, encoding="utf-8")
|
||||
output_files.append({"name": srt_path.name, "path": str(srt_path), "type": "srt"})
|
||||
|
||||
# VTT subtitles
|
||||
if "vtt" in output_formats:
|
||||
vtt_path = out_path / f"{base_name}.vtt"
|
||||
vtt_content = result_to_vtt(result)
|
||||
vtt_path.write_text(vtt_content, encoding="utf-8")
|
||||
output_files.append({"name": vtt_path.name, "path": str(vtt_path), "type": "vtt"})
|
||||
|
||||
# Update job with results
|
||||
result_summary = {
|
||||
"model": result.model,
|
||||
"language": result.language,
|
||||
"duration": result.duration,
|
||||
"segment_count": len(result.segments),
|
||||
"word_count": sum(len(seg.words) for seg in result.segments),
|
||||
}
|
||||
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.DONE,
|
||||
progress="Lyric transcription complete.",
|
||||
files=output_files,
|
||||
result=result_summary
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
|
||||
|
||||
|
||||
def run_lyric_transcription_with_diarization(
|
||||
job_id: str,
|
||||
audio_path: str,
|
||||
model: str = DEFAULT_WHISPER_MODEL,
|
||||
language: Optional[str] = None,
|
||||
num_speakers: Optional[int] = None,
|
||||
min_speakers: Optional[int] = None,
|
||||
max_speakers: Optional[int] = None,
|
||||
output_formats: Optional[List[str]] = None,
|
||||
output_dir: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
Background task: transcribe vocals with speaker diarization.
|
||||
|
||||
This uses WhisperX for diarization which requires additional setup:
|
||||
- pip install whisperx
|
||||
- HuggingFace token for pyannote models
|
||||
|
||||
Args:
|
||||
job_id: Job identifier
|
||||
audio_path: Path to audio file
|
||||
model: Whisper model name
|
||||
language: Language code
|
||||
num_speakers: Exact number of speakers (optional)
|
||||
min_speakers: Minimum number of speakers
|
||||
max_speakers: Maximum number of speakers
|
||||
output_formats: List of output formats
|
||||
output_dir: Custom output directory
|
||||
"""
|
||||
if output_formats is None:
|
||||
output_formats = ["json", "txt"]
|
||||
|
||||
if output_dir:
|
||||
out_path = Path(output_dir)
|
||||
else:
|
||||
out_path = LRC_SYNC_DIR / job_id
|
||||
out_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
import whisperx
|
||||
|
||||
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Loading WhisperX model...")
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
# Load WhisperX model
|
||||
device = "cuda" if jm.import_module("torch").cuda.is_available() else "cpu"
|
||||
compute_type = "float16" if device == "cuda" else "int8"
|
||||
|
||||
model_obj = whisperx.load_model(model, device, compute_type=compute_type)
|
||||
|
||||
# Load audio
|
||||
audio = whisperx.load_audio(audio_path)
|
||||
|
||||
jm.update_job(job_id, progress="Transcribing with WhisperX...")
|
||||
|
||||
# Transcribe
|
||||
result = model_obj.transcribe(audio, batch_size=16)
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
# Align timestamps
|
||||
jm.update_job(job_id, progress="Aligning word timestamps...")
|
||||
model_a, metadata = whisperx.load_align_model(language_code=result["language"], device=device)
|
||||
result = whisperx.align(result["segments"], model_a, metadata, audio, device)
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
# Diarization
|
||||
jm.update_job(job_id, progress="Running speaker diarization...")
|
||||
diarize_model = whisperx.DiarizationPipeline(device=device)
|
||||
diarize_segments = diarize_model(audio, num_speakers=num_speakers,
|
||||
min_speakers=min_speakers,
|
||||
max_speakers=max_speakers)
|
||||
|
||||
result = whisperx.assign_word_speakers(diarize_segments, result)
|
||||
|
||||
# Generate outputs
|
||||
jm.update_job(job_id, progress="Generating output files...")
|
||||
|
||||
base_name = Path(audio_path).stem
|
||||
output_files = []
|
||||
|
||||
if "json" in output_formats:
|
||||
json_path = out_path / f"{base_name}_diarized.json"
|
||||
json_path.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
output_files.append({"name": json_path.name, "path": str(json_path), "type": "json"})
|
||||
|
||||
if "txt" in output_formats:
|
||||
txt_path = out_path / f"{base_name}_diarized.txt"
|
||||
lines = [f"[{seg.get('speaker', 'UNKNOWN')}] {seg['text']}" for seg in result["segments"]]
|
||||
txt_path.write_text("\n".join(lines), encoding="utf-8")
|
||||
output_files.append({"name": txt_path.name, "path": str(txt_path), "type": "txt"})
|
||||
|
||||
result_summary = {
|
||||
"model": f"whisperx-{model}",
|
||||
"language": result.get("language", "unknown"),
|
||||
"segment_count": len(result["segments"]),
|
||||
"diarization": True,
|
||||
}
|
||||
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.DONE,
|
||||
progress="Lyric transcription with diarization complete.",
|
||||
files=output_files,
|
||||
result=result_summary
|
||||
)
|
||||
|
||||
except ImportError:
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.ERROR,
|
||||
error="WhisperX not installed. Install with: pip install whisperx",
|
||||
progress="Error."
|
||||
)
|
||||
except Exception as exc:
|
||||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
|
||||
|
||||
|
||||
def is_whisper_available() -> bool:
|
||||
"""Check if Whisper is available."""
|
||||
try:
|
||||
import whisper
|
||||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
|
||||
def is_whisperx_available() -> bool:
|
||||
"""Check if WhisperX is available."""
|
||||
try:
|
||||
import whisperx
|
||||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
+355
@@ -0,0 +1,355 @@
|
||||
"""
|
||||
MDX-Net separator using ONNX models from Ultimate Vocal Remover (UVR).
|
||||
Supports various MDX-Net models for vocals/instrumental separation.
|
||||
"""
|
||||
import gc
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional, Tuple
|
||||
from collections import namedtuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from tqdm import tqdm
|
||||
|
||||
# Try to import ONNX runtime
|
||||
try:
|
||||
import onnxruntime as ort
|
||||
ONNX_AVAILABLE = True
|
||||
except ImportError:
|
||||
ONNX_AVAILABLE = False
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
# Model configurations
|
||||
MDXNET_MODELS = {
|
||||
'mdxnet_voc_ft': {
|
||||
'repo_id': 'Politrees/UVR_resources',
|
||||
'path': 'models/MDXNet/UVR-MDX-NET-Voc_FT.onnx',
|
||||
'stems': ['vocals', 'instrumental'],
|
||||
'target_stem': 'vocals',
|
||||
},
|
||||
'mdxnet_inst_hq_1': {
|
||||
'repo_id': 'Politrees/UVR_resources',
|
||||
'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_1.onnx',
|
||||
'stems': ['instrumental', 'vocals'],
|
||||
'target_stem': 'instrumental',
|
||||
},
|
||||
'mdxnet_inst_hq_2': {
|
||||
'repo_id': 'Politrees/UVR_resources',
|
||||
'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_2.onnx',
|
||||
'stems': ['instrumental', 'vocals'],
|
||||
'target_stem': 'instrumental',
|
||||
},
|
||||
'mdxnet_inst_hq_3': {
|
||||
'repo_id': 'Politrees/UVR_resources',
|
||||
'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_3.onnx',
|
||||
'stems': ['instrumental', 'vocals'],
|
||||
'target_stem': 'instrumental',
|
||||
},
|
||||
'mdxnet_karaoke': {
|
||||
'repo_id': 'Politrees/UVR_resources',
|
||||
'path': 'models/MDXNet/UVR_MDXNET_KARA.onnx',
|
||||
'stems': ['lead', 'backing_instrumental'],
|
||||
'target_stem': 'lead',
|
||||
},
|
||||
# New models from MVSEP-Central-Backup
|
||||
'mdxnet_dereverb': {
|
||||
'repo_id': 'miercolesv/MVSEP-Central-Backup',
|
||||
'path': 'Dereverb-MelBand-Anvuew/dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt',
|
||||
'stems': ['dry_vocals', 'reverb'],
|
||||
'target_stem': 'dry_vocals',
|
||||
'use_melband': True, # Uses MelBand architecture, not pure ONNX
|
||||
},
|
||||
'mdxnet_drumsep': {
|
||||
'repo_id': 'miercolesv/MVSEP-Central-Backup',
|
||||
'path': 'Politrees/UVR_resources/models/MDX23C/MDX23C-DrumSep-aufr33-jarredou.ckpt',
|
||||
'stems': ['drums', 'no_drums'],
|
||||
'target_stem': 'drums',
|
||||
'use_melband': True,
|
||||
},
|
||||
}
|
||||
|
||||
# MDX-Net specific constants
|
||||
HOP_LENGTH = 1024
|
||||
N_FFT = 2048
|
||||
DIM_F = 3072
|
||||
DIM_T = 8
|
||||
|
||||
|
||||
def _load_audio(path: str, sample_rate: int = SAMPLE_RATE) -> np.ndarray:
|
||||
"""Load audio as (channels, samples) float32 array."""
|
||||
import soundfile as sf
|
||||
|
||||
try:
|
||||
data, sr = sf.read(path, dtype='float32', always_2d=True)
|
||||
data = data.T
|
||||
except Exception:
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', path, '-ar', str(sample_rate),
|
||||
'-ac', '2', '-f', 'wav', tmp_path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
data, sr = sf.read(tmp_path, dtype='float32', always_2d=True)
|
||||
data = data.T
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
if sr != sample_rate:
|
||||
import librosa
|
||||
data = np.array([
|
||||
librosa.resample(ch, orig_sr=sr, target_sr=sample_rate)
|
||||
for ch in data
|
||||
], dtype=np.float32)
|
||||
|
||||
# Ensure stereo
|
||||
if data.shape[0] == 1:
|
||||
data = np.vstack([data, data])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def _save_audio(path: str, data: np.ndarray, output_format: str, bitrate: str = '256k') -> None:
|
||||
"""Save (samples, channels) array to audio file."""
|
||||
import soundfile as sf
|
||||
|
||||
ext = output_format.lower()
|
||||
|
||||
if ext in ('wav', 'flac'):
|
||||
sf.write(path, data, SAMPLE_RATE, format=ext.upper(),
|
||||
subtype='PCM_24' if ext == 'wav' else 'PCM_16')
|
||||
else:
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
sf.write(tmp_path, data, SAMPLE_RATE, format='WAV', subtype='PCM_16')
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', tmp_path, '-b:a', bitrate, path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
|
||||
def _stft(wave: np.ndarray, n_fft: int = N_FFT, hop_length: int = HOP_LENGTH) -> np.ndarray:
|
||||
"""Compute STFT of audio."""
|
||||
# wave shape: (channels, samples)
|
||||
channels = wave.shape[0]
|
||||
specs = []
|
||||
for ch in range(channels):
|
||||
spec = np.stft(wave[ch], n_fft=n_fft, hop_length=hop_length, window='hann')
|
||||
specs.append(spec)
|
||||
return np.array(specs) # (channels, freq, time, 2) where last dim is real/imag
|
||||
|
||||
|
||||
def _istft(spec: np.ndarray, n_fft: int = N_FFT, hop_length: int = HOP_LENGTH) -> np.ndarray:
|
||||
"""Compute inverse STFT."""
|
||||
channels = spec.shape[0]
|
||||
waves = []
|
||||
for ch in range(channels):
|
||||
wave = np.istft(spec[ch], n_fft=n_fft, hop_length=hop_length, window='hann')
|
||||
waves.append(wave)
|
||||
return np.array(waves)
|
||||
|
||||
|
||||
def _spec_to_batch(spec: np.ndarray, dim_f: int = DIM_F, dim_t: int = DIM_T) -> Tuple[np.ndarray, dict]:
|
||||
"""Convert spectrogram to batch format for MDX-Net."""
|
||||
# spec shape: (channels, freq, time, 2)
|
||||
b, f, t, c = spec.shape
|
||||
|
||||
# Pad if necessary
|
||||
pad_f = dim_f - f if f < dim_f else 0
|
||||
pad_t = dim_t - (t % dim_t) if t % dim_t != 0 else 0
|
||||
|
||||
if pad_f > 0 or pad_t > 0:
|
||||
spec = np.pad(spec, ((0, 0), (0, pad_f), (0, pad_t), (0, 0)))
|
||||
|
||||
# Reshape to batches
|
||||
spec = spec.transpose(0, 3, 1, 2) # (channels, 2, freq, time)
|
||||
|
||||
# Split into time chunks
|
||||
t_chunks = spec.shape[-1] // dim_t
|
||||
spec = spec.reshape(b, c, dim_f, t_chunks, dim_t)
|
||||
spec = spec.transpose(0, 3, 1, 2, 4) # (b, t_chunks, c, f, t)
|
||||
spec = spec.reshape(-1, c, dim_f, dim_t)
|
||||
|
||||
metadata = {
|
||||
'original_shape': (b, f, t, c),
|
||||
'pad_f': pad_f,
|
||||
'pad_t': pad_t,
|
||||
't_chunks': t_chunks,
|
||||
}
|
||||
|
||||
return spec, metadata
|
||||
|
||||
|
||||
def _batch_to_spec(batch: np.ndarray, metadata: dict) -> np.ndarray:
|
||||
"""Convert batch format back to spectrogram."""
|
||||
b, f, t, c = metadata['original_shape']
|
||||
pad_f = metadata['pad_f']
|
||||
pad_t = metadata['pad_t']
|
||||
t_chunks = metadata['t_chunks']
|
||||
|
||||
# Reshape back
|
||||
batch = batch.reshape(b, t_chunks, c, DIM_F, DIM_T)
|
||||
batch = batch.transpose(0, 2, 3, 1, 4) # (b, c, f, t_chunks, t)
|
||||
batch = batch.reshape(b, c, DIM_F, t_chunks * DIM_T)
|
||||
|
||||
# Remove padding
|
||||
spec = batch.transpose(0, 2, 3, 1) # (b, f, t, c)
|
||||
if pad_f > 0:
|
||||
spec = spec[:, :f, :, :]
|
||||
if pad_t > 0:
|
||||
spec = spec[:, :, :t, :]
|
||||
|
||||
return spec
|
||||
|
||||
|
||||
class MDXNetSeparator:
|
||||
"""
|
||||
MDX-Net separator using ONNX models from Ultimate Vocal Remover.
|
||||
Supports various models for vocals/instrumental/karaoke separation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_id: str,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
):
|
||||
if not ONNX_AVAILABLE:
|
||||
raise ImportError("onnxruntime is required for MDX-Net models. Install with: pip install onnxruntime")
|
||||
|
||||
if model_id not in MDXNET_MODELS:
|
||||
raise ValueError(f"Unknown MDX-Net model: {model_id}. Available: {list(MDXNET_MODELS.keys())}")
|
||||
|
||||
self.model_id = model_id
|
||||
self.model_config = MDXNET_MODELS[model_id]
|
||||
self.model_dir = Path(model_dir) if model_dir else Path('/opt/mdxnet_models')
|
||||
self.model_path = self.model_dir / self.model_config['path'].split('/')[-1]
|
||||
self.device = 'cpu' if cpu_separation else 'cuda'
|
||||
self.output_format = output_format.lower()
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self.stems = self.model_config['stems']
|
||||
self.target_stem = self.model_config['target_stem']
|
||||
self._session = None
|
||||
|
||||
def _ensure_model(self):
|
||||
"""Download model if not present."""
|
||||
if self.model_path.exists():
|
||||
return
|
||||
|
||||
self.model_dir.mkdir(parents=True, exist_ok=True)
|
||||
from huggingface_hub import hf_hub_download
|
||||
import shutil
|
||||
|
||||
# Download to HF cache then copy flat into model_dir so model_path is correct
|
||||
cached = hf_hub_download(
|
||||
repo_id=self.model_config['repo_id'],
|
||||
filename=self.model_config['path'],
|
||||
)
|
||||
shutil.copy2(cached, str(self.model_path))
|
||||
|
||||
def _get_session(self):
|
||||
"""Get or create ONNX runtime session."""
|
||||
if self._session is None:
|
||||
self._ensure_model()
|
||||
|
||||
providers = ['CPUExecutionProvider']
|
||||
if not self.device == 'cpu' and 'CUDAExecutionProvider' in ort.get_available_providers():
|
||||
providers = ['CUDAExecutionProvider', 'CPUExecutionProvider']
|
||||
|
||||
self._session = ort.InferenceSession(
|
||||
str(self.model_path),
|
||||
providers=providers
|
||||
)
|
||||
return self._session
|
||||
|
||||
def _demix(self, mix: np.ndarray) -> Dict[str, np.ndarray]:
|
||||
"""Separate audio into stems using MDX-Net."""
|
||||
session = self._get_session()
|
||||
input_name = session.get_inputs()[0].name
|
||||
|
||||
# Compute STFT
|
||||
mix_spec = _stft(mix) # (channels, freq, time, 2)
|
||||
|
||||
# Convert to batch format
|
||||
mix_batch, metadata = _spec_to_batch(mix_spec)
|
||||
|
||||
# Run inference in chunks
|
||||
output_batches = []
|
||||
batch_size = 1 # Process one chunk at a time for memory efficiency
|
||||
|
||||
with tqdm(total=len(mix_batch), desc='Separating', unit='chunk', ncols=100) as pbar:
|
||||
for i in range(0, len(mix_batch), batch_size):
|
||||
batch = mix_batch[i:i + batch_size]
|
||||
# ONNX expects (batch, channels, freq, time)
|
||||
outputs = session.run(None, {input_name: batch.astype(np.float32)})
|
||||
output_batches.append(outputs[0])
|
||||
pbar.update(len(batch))
|
||||
|
||||
# Concatenate outputs
|
||||
target_spec = np.concatenate(output_batches, axis=0)
|
||||
|
||||
# Convert back to spectrogram format
|
||||
target_spec = _batch_to_spec(target_spec, metadata)
|
||||
|
||||
# Inverse STFT to get audio
|
||||
target_audio = _istft(target_spec)
|
||||
|
||||
# Calculate the other stem (residual)
|
||||
other_audio = mix - target_audio
|
||||
|
||||
# Create output dict based on stems configuration
|
||||
sources = {}
|
||||
if self.target_stem == 'vocals':
|
||||
sources['vocals'] = target_audio
|
||||
sources['instrumental'] = other_audio
|
||||
elif self.target_stem == 'instrumental':
|
||||
sources['instrumental'] = target_audio
|
||||
sources['vocals'] = other_audio
|
||||
elif self.target_stem == 'lead':
|
||||
sources['lead'] = target_audio
|
||||
sources['backing_instrumental'] = other_audio
|
||||
else:
|
||||
sources[self.target_stem] = target_audio
|
||||
sources['other'] = other_audio
|
||||
|
||||
return sources
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
||||
"""Separate audio and write stems."""
|
||||
output_dir = Path(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Load audio
|
||||
audio = _load_audio(input_path)
|
||||
|
||||
# Separate
|
||||
sources = self._demix(audio)
|
||||
|
||||
# Cleanup
|
||||
if self._session is not None:
|
||||
del self._session
|
||||
self._session = None
|
||||
gc.collect()
|
||||
|
||||
# Save outputs
|
||||
ext = self.output_format
|
||||
result = {}
|
||||
for name, data in sources.items():
|
||||
dest = str(output_dir / f'{name}.{ext}')
|
||||
_save_audio(dest, data.T, ext, self.mp3_bitrate)
|
||||
result[name] = dest
|
||||
|
||||
return result
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
"""
|
||||
MedleyVox separator - Multi-singer vocal separation
|
||||
HuggingFace model: Cyru5/MedleyVox
|
||||
Backup: miercolesv/MVSEP-Central-Backup
|
||||
"""
|
||||
import os
|
||||
import torch
|
||||
import torchaudio
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
import numpy as np
|
||||
|
||||
from ..config import MEDLEYVOX_MODEL_DIR, CPU_SEPARATION
|
||||
|
||||
|
||||
# Model configurations
|
||||
MEDLEYVOX_MODELS = {
|
||||
"medleyvox": {
|
||||
"model_path": "vocals 100.pth",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "MedleyVox-MultiSinger",
|
||||
"stems": ["vocals", "instrumental"],
|
||||
},
|
||||
"medleyvox_55ep": {
|
||||
"model_path": "vocals 55 epochs.pth",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "MedleyVox-MultiSinger",
|
||||
"stems": ["vocals", "instrumental"],
|
||||
},
|
||||
"medleyvox_66ep": {
|
||||
"model_path": "vocals 66.pth",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "MedleyVox-MultiSinger",
|
||||
"stems": ["vocals", "instrumental"],
|
||||
},
|
||||
"medleyvox_88ep": {
|
||||
"model_path": "vocals 88.pth",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "MedleyVox-MultiSinger",
|
||||
"stems": ["vocals", "instrumental"],
|
||||
},
|
||||
"medleyvox_100ep": {
|
||||
"model_path": "vocals 100.pth",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_subfolder": "MedleyVox-MultiSinger",
|
||||
"stems": ["vocals", "instrumental"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class MedleyVoxSeparator:
|
||||
"""
|
||||
MedleyVox separator for multi-singer vocal separation.
|
||||
Specialized for separating vocals when multiple singers are present.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_id: str = "medleyvox",
|
||||
cpu_separation: bool = CPU_SEPARATION,
|
||||
output_format: str = "mp3",
|
||||
mp3_bitrate: str = "256k",
|
||||
):
|
||||
if model_id not in MEDLEYVOX_MODELS:
|
||||
raise ValueError(f"Unknown MedleyVox model: {model_id}")
|
||||
|
||||
self.model_id = model_id
|
||||
self.model_config = MEDLEYVOX_MODELS[model_id]
|
||||
self.cpu_separation = cpu_separation
|
||||
self.output_format = output_format
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self._model = None
|
||||
self._device = None
|
||||
|
||||
def _ensure_model(self):
|
||||
"""Download model from HuggingFace if not present."""
|
||||
model_dir = MEDLEYVOX_MODEL_DIR / self.model_id
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
model_path = model_dir / self.model_config["model_path"]
|
||||
|
||||
if not model_path.exists():
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
print(f"Downloading MedleyVox model: {self.model_id}")
|
||||
|
||||
hf_hub_download(
|
||||
repo_id=self.model_config["hf_repo"],
|
||||
filename=f"{self.model_config['hf_subfolder']}/{self.model_config['model_path']}",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to download MedleyVox model: {e}")
|
||||
|
||||
return str(model_path)
|
||||
|
||||
def _load_model(self):
|
||||
"""Lazy-load the model."""
|
||||
if self._model is None:
|
||||
model_path = self._ensure_model()
|
||||
|
||||
# Set device
|
||||
self._device = "cpu" if self.cpu_separation else ("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
# Load model checkpoint
|
||||
checkpoint = torch.load(model_path, map_location=self._device)
|
||||
|
||||
# MedleyVox uses a custom architecture
|
||||
# This is a placeholder - actual implementation would need the model class
|
||||
# For now, we'll use audio-separator as a fallback
|
||||
self._model = None # Will use audio-separator fallback
|
||||
|
||||
return self._model
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
||||
"""
|
||||
Separate audio into vocals and instrumental.
|
||||
|
||||
Returns:
|
||||
Dict mapping stem names to output file paths
|
||||
"""
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# For now, use audio-separator with UVR as fallback
|
||||
# Full implementation would load the actual MedleyVox model
|
||||
try:
|
||||
from audio_separator.separator import Separator
|
||||
|
||||
# Use a generic 2-stem model as fallback
|
||||
sep = Separator(
|
||||
model_file_dir=str(MEDLEYVOX_MODEL_DIR),
|
||||
output_dir=str(output_path),
|
||||
output_format=self.output_format,
|
||||
output_bitrate=self.mp3_bitrate if self.output_format == "mp3" else None,
|
||||
cpu_separation=self.cpu_separation,
|
||||
)
|
||||
|
||||
# Use UVR-MDX-NET Inst HQ as fallback
|
||||
sep.load_model()
|
||||
result = sep.separate(input_path)
|
||||
|
||||
# Map outputs to stem names
|
||||
stems = self.model_config.get("stems", ["vocals", "instrumental"])
|
||||
output_paths = {}
|
||||
|
||||
if isinstance(result, list):
|
||||
for i, path in enumerate(result):
|
||||
if i < len(stems):
|
||||
stem_name = stems[i]
|
||||
final_path = output_path / f"{stem_name}.{self.output_format}"
|
||||
Path(path).rename(final_path)
|
||||
output_paths[stem_name] = str(final_path)
|
||||
else:
|
||||
final_path = output_path / f"vocals.{self.output_format}"
|
||||
Path(result).rename(final_path)
|
||||
output_paths["vocals"] = str(final_path)
|
||||
|
||||
return output_paths
|
||||
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"audio-separator is required for MedleyVox separation. "
|
||||
"Install with: pip install audio-separator"
|
||||
)
|
||||
+336
@@ -0,0 +1,336 @@
|
||||
"""
|
||||
MelBand RoFormer separator — standalone, no Django/Spleeter/TensorFlow dependency.
|
||||
|
||||
Audio I/O uses soundfile (WAV/FLAC) + subprocess ffmpeg (MP3 encoding).
|
||||
Model architecture is from https://github.com/lucidrains/BS-RoFormer
|
||||
|
||||
This module patches sys.modules to fix the import path for models.bs_roformer.attend
|
||||
which the melband files expect but doesn't exist in this project structure.
|
||||
"""
|
||||
import gc
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import yaml
|
||||
from omegaconf import OmegaConf
|
||||
from tqdm import tqdm
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Import path patching
|
||||
# The melband files import from models.bs_roformer.attend which doesn't exist
|
||||
# in this project structure. We patch sys.modules to redirect to our bs_roformer.
|
||||
# ---------------------------------------------------------------------------
|
||||
if 'models' not in sys.modules:
|
||||
sys.modules['models'] = type(sys)('models')
|
||||
if 'models.bs_roformer' not in sys.modules:
|
||||
import backend.services.bs_roformer as _bs_roformer
|
||||
sys.modules['models.bs_roformer'] = _bs_roformer
|
||||
if 'models.bs_roformer.attend' not in sys.modules:
|
||||
from backend.services.bs_roformer import attend as _attend
|
||||
sys.modules['models.bs_roformer.attend'] = _attend
|
||||
|
||||
# Import MelBandRoformer from the bs_roformer pip package (lucidrains/BS-RoFormer)
|
||||
from bs_roformer.mel_band_roformer import MelBandRoformer
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HuggingFace model coordinates
|
||||
# ---------------------------------------------------------------------------
|
||||
HF_REPO_ID = 'fabiankruger/mel-band-roformer-4-stem'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Default model paths (overridable via env var MEL_BAND_MODEL_DIR)
|
||||
# ---------------------------------------------------------------------------
|
||||
_MODEL_DIR_ENV = os.environ.get('MEL_BAND_MODEL_DIR', '/opt/mel_band_models')
|
||||
DEFAULT_MODEL_DIR = Path(_MODEL_DIR_ENV)
|
||||
DEFAULT_MODEL_PATH = DEFAULT_MODEL_DIR / 'mel_band_roformer_4_stem.ckpt'
|
||||
DEFAULT_CONFIG_PATH = DEFAULT_MODEL_DIR / 'config.yaml'
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _try_download_model(
|
||||
model_dir: Path = DEFAULT_MODEL_DIR,
|
||||
model_path: Path = DEFAULT_MODEL_PATH,
|
||||
config_path: Path = DEFAULT_CONFIG_PATH,
|
||||
) -> None:
|
||||
"""Download MelBand RoFormer model from HuggingFace if not already present."""
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
from huggingface_hub import snapshot_download
|
||||
snapshot_download(
|
||||
repo_id=HF_REPO_ID,
|
||||
revision=HF_REVISION,
|
||||
local_dir=str(model_dir),
|
||||
)
|
||||
|
||||
|
||||
def _load_config(config_path: Path):
|
||||
"""Load YAML config, supporting !!python/tuple tags."""
|
||||
with open(config_path, 'r') as f:
|
||||
config = yaml.load(f, Loader=yaml.UnsafeLoader)
|
||||
return OmegaConf.create(config)
|
||||
|
||||
|
||||
def _load_audio(path: str) -> np.ndarray:
|
||||
"""
|
||||
Load audio file and return as float32 numpy array of shape (channels, samples).
|
||||
Uses soundfile for WAV/FLAC; falls back to ffmpeg decode for MP3 and other formats.
|
||||
"""
|
||||
import soundfile as sf
|
||||
|
||||
try:
|
||||
data, sr = sf.read(path, dtype='float32', always_2d=True)
|
||||
# soundfile returns (samples, channels); transpose to (channels, samples)
|
||||
data = data.T
|
||||
except Exception:
|
||||
# Fallback: decode via ffmpeg to a temporary WAV
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', path, '-ar', str(SAMPLE_RATE),
|
||||
'-ac', '2', '-f', 'wav', tmp_path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
data, sr = sf.read(tmp_path, dtype='float32', always_2d=True)
|
||||
data = data.T
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
# Resample if needed
|
||||
if sr != SAMPLE_RATE:
|
||||
import librosa
|
||||
data = np.array([
|
||||
librosa.resample(ch, orig_sr=sr, target_sr=SAMPLE_RATE)
|
||||
for ch in data
|
||||
], dtype=np.float32)
|
||||
|
||||
# Ensure stereo
|
||||
if data.shape[0] == 1:
|
||||
data = np.vstack([data, data])
|
||||
|
||||
return data # (channels, samples)
|
||||
|
||||
|
||||
def _save_audio(path: str, data: np.ndarray, output_format: str, bitrate: str = '256k') -> None:
|
||||
"""
|
||||
Save (samples, channels) float32 array as audio file.
|
||||
WAV and FLAC use soundfile directly; MP3 encodes via ffmpeg.
|
||||
"""
|
||||
import soundfile as sf
|
||||
|
||||
ext = output_format.lower()
|
||||
|
||||
if ext in ('wav', 'flac'):
|
||||
sf.write(path, data, SAMPLE_RATE, format=ext.upper(),
|
||||
subtype='PCM_24' if ext == 'wav' else 'PCM_16')
|
||||
else:
|
||||
# Encode to MP3 via ffmpeg
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
sf.write(tmp_path, data, SAMPLE_RATE, format='WAV', subtype='PCM_16')
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', tmp_path,
|
||||
'-b:a', bitrate, path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Separator class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class MelBandRoformerSeparator:
|
||||
"""
|
||||
Performs source separation using the MelBand RoFormer model.
|
||||
Pure PyTorch — no TensorFlow / Spleeter dependency.
|
||||
"""
|
||||
|
||||
# MelBand RoFormer outputs 4 stems by default
|
||||
DEFAULT_STEMS = ['vocals', 'drums', 'bass', 'other']
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_path: Optional[str] = None,
|
||||
config_path: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3', # 'mp3' | 'flac' | 'wav'
|
||||
mp3_bitrate: str = '256k',
|
||||
):
|
||||
self.model_path = Path(model_path) if model_path else DEFAULT_MODEL_PATH
|
||||
self.config_path = Path(config_path) if config_path else DEFAULT_CONFIG_PATH
|
||||
self.device = 'cpu' if cpu_separation else 'cuda'
|
||||
self.output_format = output_format.lower()
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self._config = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Model loading
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _ensure_model(self):
|
||||
_try_download_model(DEFAULT_MODEL_DIR, self.model_path, self.config_path)
|
||||
|
||||
def _load_config(self):
|
||||
if self._config is None:
|
||||
self._config = _load_config(self.config_path)
|
||||
return self._config
|
||||
|
||||
def get_model(self) -> MelBandRoformer:
|
||||
self._ensure_model()
|
||||
config = self._load_config()
|
||||
model_cfg = OmegaConf.to_container(config.model, resolve=True)
|
||||
|
||||
# OmegaConf loads tuples as lists; the model requires tuples for certain fields
|
||||
if 'freqs_per_bands' in model_cfg and isinstance(model_cfg['freqs_per_bands'], list):
|
||||
model_cfg['freqs_per_bands'] = tuple(model_cfg['freqs_per_bands'])
|
||||
if 'multi_stft_resolutions_window_sizes' in model_cfg and isinstance(model_cfg['multi_stft_resolutions_window_sizes'], list):
|
||||
model_cfg['multi_stft_resolutions_window_sizes'] = tuple(model_cfg['multi_stft_resolutions_window_sizes'])
|
||||
|
||||
model = MelBandRoformer(**model_cfg)
|
||||
checkpoint = torch.load(str(self.model_path), map_location='cpu', weights_only=False)
|
||||
|
||||
# Handle different checkpoint formats
|
||||
if 'state_dict' in checkpoint:
|
||||
state_dict = checkpoint['state_dict']
|
||||
elif 'model' in checkpoint:
|
||||
state_dict = checkpoint['model']
|
||||
else:
|
||||
state_dict = checkpoint
|
||||
|
||||
model.load_state_dict(state_dict)
|
||||
model.to(self.device).eval()
|
||||
return model
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Inference
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _demix(self, mix: np.ndarray, model: MelBandRoformer,
|
||||
batch_size_override: Optional[int] = None,
|
||||
num_overlap_override: Optional[int] = None) -> Dict[str, np.ndarray]:
|
||||
"""Separate (channels, samples) array into a dict of stem arrays."""
|
||||
config = self._load_config()
|
||||
hop_length = config.model.stft_hop_length
|
||||
segment_size = getattr(config.inference, 'dim_t', 2000)
|
||||
C = hop_length * (segment_size - 1)
|
||||
N = num_overlap_override if num_overlap_override is not None else getattr(config.inference, 'num_overlap', 4)
|
||||
step = int(C // N)
|
||||
fade_size = C // 10
|
||||
batch_size = batch_size_override if batch_size_override is not None else getattr(config.inference, 'batch_size', 1)
|
||||
device = self.device
|
||||
|
||||
mix_t = torch.tensor(mix, dtype=torch.float32)
|
||||
length_init = mix_t.shape[-1]
|
||||
|
||||
if length_init > 2 * (C - step) and (C - step > 0):
|
||||
mix_t = nn.functional.pad(mix_t, (C - step, C - step), mode='reflect')
|
||||
|
||||
fadein = torch.linspace(0, 1, fade_size).to(device)
|
||||
fadeout = torch.linspace(1, 0, fade_size).to(device)
|
||||
window_start = torch.ones(C).to(device)
|
||||
window_start[-fade_size:] *= fadeout
|
||||
window_finish = torch.ones(C).to(device)
|
||||
window_finish[:fade_size] *= fadein
|
||||
window_middle = torch.ones(C).to(device)
|
||||
window_middle[:fade_size] *= fadein
|
||||
window_middle[-fade_size:] *= fadeout
|
||||
|
||||
# Get stems from config or use default
|
||||
stems = getattr(config.training, 'instruments', self.DEFAULT_STEMS)
|
||||
num_stems = len(stems)
|
||||
req_shape = (num_stems,) + tuple(mix_t.shape)
|
||||
|
||||
with torch.inference_mode():
|
||||
result = torch.zeros(req_shape, dtype=torch.float32, device=device)
|
||||
counter = torch.zeros(req_shape, dtype=torch.float32, device=device)
|
||||
batch_data, batch_locs = [], []
|
||||
i = 0
|
||||
|
||||
# Count total segments for tqdm
|
||||
total_seg = sum(1 for _ in range(0, mix_t.shape[1], step))
|
||||
|
||||
with tqdm(total=total_seg, desc='Separating', unit='segment', ncols=100) as pbar:
|
||||
while i < mix_t.shape[1]:
|
||||
part = mix_t[:, i:i + C].to(device)
|
||||
l = part.shape[-1]
|
||||
if l < C:
|
||||
part = nn.functional.pad(
|
||||
part,
|
||||
(0, C - l) if l > C // 2 + 1 else (0, C - l, 0, 0),
|
||||
mode='reflect' if l > C // 2 + 1 else 'constant',
|
||||
)
|
||||
batch_data.append(part)
|
||||
batch_locs.append((i, l))
|
||||
i += step
|
||||
|
||||
if len(batch_data) >= batch_size or i >= mix_t.shape[1]:
|
||||
arr = torch.stack(batch_data, dim=0)
|
||||
x = model(arr)
|
||||
for j, (start, ln) in enumerate(batch_locs):
|
||||
window = (window_start if start == 0
|
||||
else window_finish if i >= mix_t.shape[1]
|
||||
else window_middle)
|
||||
result[..., start:start + ln] += x[j][..., :ln] * window[..., :ln]
|
||||
counter[..., start:start + ln] += window[..., :ln]
|
||||
pbar.update(len(batch_data))
|
||||
batch_data, batch_locs = [], []
|
||||
|
||||
estimated = result / counter.clamp(min=1e-10)
|
||||
if length_init > 2 * (C - step) and (C - step > 0):
|
||||
estimated = estimated[..., (C - step):-(C - step)]
|
||||
|
||||
sources = {}
|
||||
for idx, name in enumerate(stems):
|
||||
sources[name] = estimated[idx].cpu().numpy()
|
||||
return sources
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str,
|
||||
batch_size_override: Optional[int] = None,
|
||||
num_overlap_override: Optional[int] = None) -> Dict[str, str]:
|
||||
"""
|
||||
Separate audio at input_path and write one file per stem into output_dir.
|
||||
Returns {stem_name: output_file_path}.
|
||||
"""
|
||||
output_dir = Path(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
audio = _load_audio(input_path) # (channels, samples)
|
||||
model = self.get_model()
|
||||
sources = self._demix(audio, model,
|
||||
batch_size_override=batch_size_override,
|
||||
num_overlap_override=num_overlap_override)
|
||||
|
||||
del model
|
||||
torch.cuda.empty_cache()
|
||||
gc.collect()
|
||||
|
||||
ext = self.output_format
|
||||
result = {}
|
||||
for name, data in sources.items():
|
||||
dest = str(output_dir / f'{name}.{ext}')
|
||||
_save_audio(dest, data.T, ext, self.mp3_bitrate) # T: (channels,samples)→(samples,channels)
|
||||
result[name] = dest
|
||||
return result
|
||||
@@ -0,0 +1,669 @@
|
||||
"""
|
||||
Specialized MelBand RoFormer separators for specific use cases:
|
||||
- Vocals/Instrumental separation (2-stem)
|
||||
- Dereverb (dry vocals vs reverb)
|
||||
- Karaoke (lead vocals vs backing+instrumental)
|
||||
|
||||
Uses sys.modules patching for import path fixes.
|
||||
"""
|
||||
import gc
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional, List
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import yaml
|
||||
from omegaconf import OmegaConf
|
||||
from tqdm import tqdm
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Import path patching
|
||||
# ---------------------------------------------------------------------------
|
||||
if 'models' not in sys.modules:
|
||||
sys.modules['models'] = type(sys)('models')
|
||||
if 'models.bs_roformer' not in sys.modules:
|
||||
import backend.services.bs_roformer as _bs_roformer
|
||||
sys.modules['models.bs_roformer'] = _bs_roformer
|
||||
if 'models.bs_roformer.attend' not in sys.modules:
|
||||
from backend.services.bs_roformer import attend as _attend
|
||||
sys.modules['models.bs_roformer.attend'] = _attend
|
||||
|
||||
from bs_roformer.mel_band_roformer import MelBandRoformer
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
|
||||
|
||||
def _load_audio(path: str) -> np.ndarray:
|
||||
"""Load audio as (channels, samples) float32 array."""
|
||||
import soundfile as sf
|
||||
|
||||
try:
|
||||
data, sr = sf.read(path, dtype='float32', always_2d=True)
|
||||
data = data.T
|
||||
except Exception:
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', path, '-ar', str(SAMPLE_RATE),
|
||||
'-ac', '2', '-f', 'wav', tmp_path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
data, sr = sf.read(tmp_path, dtype='float32', always_2d=True)
|
||||
data = data.T
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
if sr != SAMPLE_RATE:
|
||||
import librosa
|
||||
data = np.array([
|
||||
librosa.resample(ch, orig_sr=sr, target_sr=SAMPLE_RATE)
|
||||
for ch in data
|
||||
], dtype=np.float32)
|
||||
|
||||
if data.shape[0] == 1:
|
||||
data = np.vstack([data, data])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def _save_audio(path: str, data: np.ndarray, output_format: str, bitrate: str = '256k') -> None:
|
||||
"""Save (samples, channels) array to audio file."""
|
||||
import soundfile as sf
|
||||
|
||||
ext = output_format.lower()
|
||||
|
||||
if ext in ('wav', 'flac'):
|
||||
sf.write(path, data, SAMPLE_RATE, format=ext.upper(),
|
||||
subtype='PCM_24' if ext == 'wav' else 'PCM_16')
|
||||
else:
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
sf.write(tmp_path, data, SAMPLE_RATE, format='WAV', subtype='PCM_16')
|
||||
subprocess.run(
|
||||
['ffmpeg', '-y', '-i', tmp_path, '-b:a', bitrate, path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
|
||||
class MelBandSeparatorBase:
|
||||
"""Base class for MelBand RoFormer separators."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_path: Path,
|
||||
config_path: Path,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
stems: List[str] = None,
|
||||
):
|
||||
self.model_path = model_path
|
||||
self.config_path = config_path
|
||||
self.device = 'cpu' if cpu_separation else 'cuda'
|
||||
self.output_format = output_format.lower()
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self.stems = stems or ['vocals', 'instrumental']
|
||||
self._config = None
|
||||
|
||||
def _load_config(self):
|
||||
if self._config is None:
|
||||
with open(self.config_path, 'r') as f:
|
||||
config = yaml.load(f, Loader=yaml.UnsafeLoader)
|
||||
self._config = OmegaConf.create(config)
|
||||
return self._config
|
||||
|
||||
def get_model(self) -> MelBandRoformer:
|
||||
config = self._load_config()
|
||||
model_cfg = OmegaConf.to_container(config.model, resolve=True)
|
||||
|
||||
# Convert lists to tuples where needed
|
||||
if 'freqs_per_bands' in model_cfg and isinstance(model_cfg['freqs_per_bands'], list):
|
||||
model_cfg['freqs_per_bands'] = tuple(model_cfg['freqs_per_bands'])
|
||||
if 'multi_stft_resolutions_window_sizes' in model_cfg and isinstance(model_cfg['multi_stft_resolutions_window_sizes'], list):
|
||||
model_cfg['multi_stft_resolutions_window_sizes'] = tuple(model_cfg['multi_stft_resolutions_window_sizes'])
|
||||
|
||||
model = MelBandRoformer(**model_cfg)
|
||||
checkpoint = torch.load(str(self.model_path), map_location='cpu', weights_only=False)
|
||||
|
||||
if 'state_dict' in checkpoint:
|
||||
state_dict = checkpoint['state_dict']
|
||||
elif 'model' in checkpoint:
|
||||
state_dict = checkpoint['model']
|
||||
else:
|
||||
state_dict = checkpoint
|
||||
|
||||
model.load_state_dict(state_dict)
|
||||
model.to(self.device).eval()
|
||||
return model
|
||||
|
||||
def _demix(self, mix: np.ndarray, model: MelBandRoformer,
|
||||
batch_size_override: Optional[int] = None,
|
||||
num_overlap_override: Optional[int] = None) -> Dict[str, np.ndarray]:
|
||||
"""Separate audio into stems."""
|
||||
config = self._load_config()
|
||||
hop_length = config.model.stft_hop_length
|
||||
segment_size = getattr(config.inference, 'dim_t', 2000)
|
||||
C = hop_length * (segment_size - 1)
|
||||
N = num_overlap_override if num_overlap_override is not None else getattr(config.inference, 'num_overlap', 4)
|
||||
step = int(C // N)
|
||||
fade_size = C // 10
|
||||
batch_size = batch_size_override if batch_size_override is not None else getattr(config.inference, 'batch_size', 1)
|
||||
device = self.device
|
||||
|
||||
mix_t = torch.tensor(mix, dtype=torch.float32)
|
||||
length_init = mix_t.shape[-1]
|
||||
|
||||
if length_init > 2 * (C - step) and (C - step > 0):
|
||||
mix_t = nn.functional.pad(mix_t, (C - step, C - step), mode='reflect')
|
||||
|
||||
fadein = torch.linspace(0, 1, fade_size).to(device)
|
||||
fadeout = torch.linspace(1, 0, fade_size).to(device)
|
||||
window_start = torch.ones(C).to(device)
|
||||
window_start[-fade_size:] *= fadeout
|
||||
window_finish = torch.ones(C).to(device)
|
||||
window_finish[:fade_size] *= fadein
|
||||
window_middle = torch.ones(C).to(device)
|
||||
window_middle[:fade_size] *= fadein
|
||||
window_middle[-fade_size:] *= fadeout
|
||||
|
||||
num_stems = len(self.stems)
|
||||
req_shape = (num_stems,) + tuple(mix_t.shape)
|
||||
|
||||
with torch.inference_mode():
|
||||
result = torch.zeros(req_shape, dtype=torch.float32, device=device)
|
||||
counter = torch.zeros(req_shape, dtype=torch.float32, device=device)
|
||||
batch_data, batch_locs = [], []
|
||||
i = 0
|
||||
|
||||
total_seg = sum(1 for _ in range(0, mix_t.shape[1], step))
|
||||
|
||||
with tqdm(total=total_seg, desc='Separating', unit='segment', ncols=100) as pbar:
|
||||
while i < mix_t.shape[1]:
|
||||
part = mix_t[:, i:i + C].to(device)
|
||||
l = part.shape[-1]
|
||||
if l < C:
|
||||
part = nn.functional.pad(
|
||||
part,
|
||||
(0, C - l) if l > C // 2 + 1 else (0, C - l, 0, 0),
|
||||
mode='reflect' if l > C // 2 + 1 else 'constant',
|
||||
)
|
||||
batch_data.append(part)
|
||||
batch_locs.append((i, l))
|
||||
i += step
|
||||
|
||||
if len(batch_data) >= batch_size or i >= mix_t.shape[1]:
|
||||
arr = torch.stack(batch_data, dim=0)
|
||||
x = model(arr)
|
||||
for j, (start, ln) in enumerate(batch_locs):
|
||||
window = (window_start if start == 0
|
||||
else window_finish if i >= mix_t.shape[1]
|
||||
else window_middle)
|
||||
result[..., start:start + ln] += x[j][..., :ln] * window[..., :ln]
|
||||
counter[..., start:start + ln] += window[..., :ln]
|
||||
pbar.update(len(batch_data))
|
||||
batch_data, batch_locs = [], []
|
||||
|
||||
estimated = result / counter.clamp(min=1e-10)
|
||||
if length_init > 2 * (C - step) and (C - step > 0):
|
||||
estimated = estimated[..., (C - step):-(C - step)]
|
||||
|
||||
sources = {}
|
||||
for idx, name in enumerate(self.stems):
|
||||
sources[name] = estimated[idx].cpu().numpy()
|
||||
return sources
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str,
|
||||
batch_size_override: Optional[int] = None,
|
||||
num_overlap_override: Optional[int] = None) -> Dict[str, str]:
|
||||
"""Separate audio and write stems to output_dir."""
|
||||
output_dir = Path(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
audio = _load_audio(input_path)
|
||||
model = self.get_model()
|
||||
sources = self._demix(audio, model,
|
||||
batch_size_override=batch_size_override,
|
||||
num_overlap_override=num_overlap_override)
|
||||
|
||||
del model
|
||||
torch.cuda.empty_cache()
|
||||
gc.collect()
|
||||
|
||||
ext = self.output_format
|
||||
result = {}
|
||||
for name, data in sources.items():
|
||||
dest = str(output_dir / f'{name}.{ext}')
|
||||
_save_audio(dest, data.T, ext, self.mp3_bitrate)
|
||||
result[name] = dest
|
||||
return result
|
||||
|
||||
|
||||
class MelBandVocalsSeparator(MelBandSeparatorBase):
|
||||
"""MelBand RoFormer optimized for vocals/instrumental separation."""
|
||||
|
||||
HF_REPO_ID = 'becruily/mel-band-roformer-vocals'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
):
|
||||
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/vocals')
|
||||
model_path = model_dir / 'model.ckpt'
|
||||
config_path = model_dir / 'config.yaml'
|
||||
|
||||
# Ensure model is downloaded
|
||||
self._ensure_model(model_dir, model_path, config_path)
|
||||
|
||||
super().__init__(
|
||||
model_path=model_path,
|
||||
config_path=config_path,
|
||||
cpu_separation=cpu_separation,
|
||||
output_format=output_format,
|
||||
mp3_bitrate=mp3_bitrate,
|
||||
stems=['vocals', 'instrumental'],
|
||||
)
|
||||
|
||||
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
from huggingface_hub import snapshot_download
|
||||
snapshot_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
revision=self.HF_REVISION,
|
||||
local_dir=str(model_dir),
|
||||
)
|
||||
|
||||
|
||||
class MelBandDereverbSeparator(MelBandSeparatorBase):
|
||||
"""MelBand RoFormer for dereverb - separates dry vocals from reverb."""
|
||||
|
||||
HF_REPO_ID = 'anvuew/dereverb_mel_band_roformer'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
use_less_aggressive: bool = False,
|
||||
):
|
||||
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/dereverb')
|
||||
|
||||
# The dereverb repo has multiple checkpoints
|
||||
if use_less_aggressive:
|
||||
model_path = model_dir / 'dereverb_mel_band_roformer_less_aggressive_anvuew_sdr_18.8050.ckpt'
|
||||
else:
|
||||
model_path = model_dir / 'dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt'
|
||||
config_path = model_dir / 'config.yaml'
|
||||
|
||||
self._ensure_model(model_dir, model_path, config_path)
|
||||
|
||||
super().__init__(
|
||||
model_path=model_path,
|
||||
config_path=config_path,
|
||||
cpu_separation=cpu_separation,
|
||||
output_format=output_format,
|
||||
mp3_bitrate=mp3_bitrate,
|
||||
stems=['dry_vocals', 'reverb'],
|
||||
)
|
||||
|
||||
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
from huggingface_hub import snapshot_download
|
||||
snapshot_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
revision=self.HF_REVISION,
|
||||
local_dir=str(model_dir),
|
||||
)
|
||||
|
||||
|
||||
class MelBandKaraokeSeparator(MelBandSeparatorBase):
|
||||
"""MelBand RoFormer for karaoke - separates lead vocals from backing+instrumental."""
|
||||
|
||||
HF_REPO_ID = 'becruily/mel-band-roformer-karaoke'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
):
|
||||
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/karaoke')
|
||||
model_path = model_dir / 'model.ckpt'
|
||||
config_path = model_dir / 'config.yaml'
|
||||
|
||||
self._ensure_model(model_dir, model_path, config_path)
|
||||
|
||||
super().__init__(
|
||||
model_path=model_path,
|
||||
config_path=config_path,
|
||||
cpu_separation=cpu_separation,
|
||||
output_format=output_format,
|
||||
mp3_bitrate=mp3_bitrate,
|
||||
stems=['lead', 'backing_instrumental'],
|
||||
)
|
||||
|
||||
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
from huggingface_hub import snapshot_download
|
||||
snapshot_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
revision=self.HF_REVISION,
|
||||
local_dir=str(model_dir),
|
||||
)
|
||||
|
||||
|
||||
class MelBandDereverbRoomSeparator(MelBandSeparatorBase):
|
||||
"""MelBand RoFormer for room dereverb from MVSEP-Central-Backup."""
|
||||
|
||||
HF_REPO_ID = 'miercolesv/MVSEP-Central-Backup'
|
||||
HF_SUBFOLDER = 'Dereverb-Room-Anvuew'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
):
|
||||
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/dereverb_room')
|
||||
model_path = model_dir / 'dereverb_room_anvuew_sdr_13.7432.ckpt'
|
||||
config_path = model_dir / 'dereverb_room_anvuew.yaml'
|
||||
|
||||
self._ensure_model(model_dir, model_path, config_path)
|
||||
|
||||
super().__init__(
|
||||
model_path=model_path,
|
||||
config_path=config_path,
|
||||
cpu_separation=cpu_separation,
|
||||
output_format=output_format,
|
||||
mp3_bitrate=mp3_bitrate,
|
||||
stems=['dry_vocals', 'room_reverb'],
|
||||
)
|
||||
|
||||
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
if not model_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/dereverb_room_anvuew_sdr_13.7432.ckpt",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
if not config_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/dereverb_room_anvuew.yaml",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to download Room Dereverb model: {e}")
|
||||
|
||||
|
||||
class MelBandDereverbAnvuewSeparator(MelBandSeparatorBase):
|
||||
"""MelBand RoFormer Anvuew dereverb from MVSEP-Central-Backup."""
|
||||
|
||||
HF_REPO_ID = 'miercolesv/MVSEP-Central-Backup'
|
||||
HF_SUBFOLDER = 'Dereverb-MelBand-Anvuew'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
use_less_aggressive: bool = False,
|
||||
):
|
||||
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/dereverb_anvuew')
|
||||
|
||||
if use_less_aggressive:
|
||||
model_path = model_dir / 'dereverb_mel_band_roformer_less_aggressive_anvuew_sdr_18.8050.ckpt'
|
||||
else:
|
||||
model_path = model_dir / 'dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt'
|
||||
|
||||
config_path = model_dir / 'dereverb_mel_band_roformer_anvuew.yaml'
|
||||
|
||||
self._ensure_model(model_dir, model_path, config_path)
|
||||
|
||||
super().__init__(
|
||||
model_path=model_path,
|
||||
config_path=config_path,
|
||||
cpu_separation=cpu_separation,
|
||||
output_format=output_format,
|
||||
mp3_bitrate=mp3_bitrate,
|
||||
stems=['dry_vocals', 'reverb'],
|
||||
)
|
||||
|
||||
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
# Download the specific model variant
|
||||
model_filename = model_path.name
|
||||
if not model_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/{model_filename}",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
# Download config
|
||||
if not config_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/dereverb_mel_band_roformer_anvuew.yaml",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to download Anvuew Dereverb model: {e}")
|
||||
|
||||
|
||||
class MelBandVocalsAnvuewSeparator(MelBandSeparatorBase):
|
||||
"""MelBand RoFormer Anvuew vocals from MVSEP-Central-Backup."""
|
||||
|
||||
HF_REPO_ID = 'miercolesv/MVSEP-Central-Backup'
|
||||
HF_SUBFOLDER = 'BS-RoFormer-Anvuew'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
):
|
||||
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/vocals_anvuew')
|
||||
model_path = model_dir / 'bs_roformer_anvuew_sdr_12.45.ckpt'
|
||||
config_path = model_dir / 'config.yaml'
|
||||
|
||||
self._ensure_model(model_dir, model_path, config_path)
|
||||
|
||||
super().__init__(
|
||||
model_path=model_path,
|
||||
config_path=config_path,
|
||||
cpu_separation=cpu_separation,
|
||||
output_format=output_format,
|
||||
mp3_bitrate=mp3_bitrate,
|
||||
stems=['vocals', 'instrumental'],
|
||||
)
|
||||
|
||||
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
if not model_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/bs_roformer_anvuew_sdr_12.45.ckpt",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
if not config_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/config.yaml",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to download Anvuew Vocals model: {e}")
|
||||
|
||||
|
||||
class MelBandKaraokeGaboxSeparator(MelBandSeparatorBase):
|
||||
"""MelBand RoFormer Karaoke Gabox from MVSEP-Central-Backup."""
|
||||
|
||||
HF_REPO_ID = 'miercolesv/MVSEP-Central-Backup'
|
||||
HF_SUBFOLDER = 'MelBandRoformers/bsroformers'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
):
|
||||
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/karaoke_gabox')
|
||||
model_path = model_dir / 'bs_karaoke_gabox_IS.ckpt'
|
||||
config_path = model_dir / 'karaoke_bs_roformer.yaml'
|
||||
|
||||
self._ensure_model(model_dir, model_path, config_path)
|
||||
|
||||
super().__init__(
|
||||
model_path=model_path,
|
||||
config_path=config_path,
|
||||
cpu_separation=cpu_separation,
|
||||
output_format=output_format,
|
||||
mp3_bitrate=mp3_bitrate,
|
||||
stems=['vocals', 'karaoke'],
|
||||
)
|
||||
|
||||
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
if not model_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/bs_karaoke_gabox_IS.ckpt",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
if not config_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/karaoke_bs_roformer.yaml",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to download Karaoke Gabox model: {e}")
|
||||
|
||||
|
||||
class MelBandDeuxBecruilySeparator(MelBandSeparatorBase):
|
||||
"""MelBand RoFormer Deux Becruily - specialized 2-stem vocals model."""
|
||||
|
||||
HF_REPO_ID = 'miercolesv/MVSEP-Central-Backup'
|
||||
HF_SUBFOLDER = 'MelBand-Roformer-Deux-Becruily'
|
||||
HF_REVISION = 'main'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dir: Optional[str] = None,
|
||||
cpu_separation: bool = True,
|
||||
output_format: str = 'mp3',
|
||||
mp3_bitrate: str = '256k',
|
||||
):
|
||||
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/deux_becruily')
|
||||
model_path = model_dir / 'becruily_deux.ckpt'
|
||||
config_path = model_dir / 'config_deux_becruily.yaml'
|
||||
|
||||
self._ensure_model(model_dir, model_path, config_path)
|
||||
|
||||
super().__init__(
|
||||
model_path=model_path,
|
||||
config_path=config_path,
|
||||
cpu_separation=cpu_separation,
|
||||
output_format=output_format,
|
||||
mp3_bitrate=mp3_bitrate,
|
||||
stems=['vocals', 'instrumental'],
|
||||
)
|
||||
|
||||
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
|
||||
if model_path.exists() and config_path.exists():
|
||||
return
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
print(f"Downloading Deux Becruily model from {self.HF_REPO_ID}")
|
||||
|
||||
# Download model
|
||||
if not model_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/becruily_deux.ckpt",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
# Download config
|
||||
if not config_path.exists():
|
||||
hf_hub_download(
|
||||
repo_id=self.HF_REPO_ID,
|
||||
filename=f"{self.HF_SUBFOLDER}/config_deux_becruily.yaml",
|
||||
local_dir=model_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to download Deux Becruily model: {e}")
|
||||
+557
@@ -0,0 +1,557 @@
|
||||
"""
|
||||
Audio metadata (ID3 / Vorbis / EXIF) utilities for reading and writing tags.
|
||||
Uses mutagen for format-agnostic read/write.
|
||||
"""
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Dict, Tuple
|
||||
import re
|
||||
|
||||
|
||||
def _read_cover_art(source_path: str):
|
||||
"""
|
||||
Read cover art from an audio file.
|
||||
Returns (data_bytes, mime_type) or (None, None) if not found.
|
||||
"""
|
||||
try:
|
||||
ext = Path(source_path).suffix.lower()
|
||||
if ext == '.mp3':
|
||||
from mutagen.id3 import ID3, ID3NoHeaderError
|
||||
try:
|
||||
tags = ID3(source_path)
|
||||
apic_keys = [k for k in tags.keys() if k.startswith('APIC')]
|
||||
if apic_keys:
|
||||
apic = tags[apic_keys[0]]
|
||||
return apic.data, apic.mime
|
||||
except ID3NoHeaderError:
|
||||
pass
|
||||
elif ext == '.flac':
|
||||
from mutagen.flac import FLAC
|
||||
audio = FLAC(source_path)
|
||||
if audio.pictures:
|
||||
pic = audio.pictures[0]
|
||||
return pic.data, pic.mime
|
||||
elif ext == '.wav':
|
||||
from mutagen.wave import WAVE
|
||||
try:
|
||||
audio = WAVE(source_path)
|
||||
if audio.tags:
|
||||
apic_keys = [k for k in audio.tags.keys() if k.startswith('APIC')]
|
||||
if apic_keys:
|
||||
apic = audio.tags[apic_keys[0]]
|
||||
return apic.data, apic.mime
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
return None, None
|
||||
|
||||
|
||||
def copy_metadata(source_path: str, dest_path: str, stem_label: Optional[str] = None,
|
||||
copy_lyrics: bool = True) -> None:
|
||||
"""
|
||||
Copy all common tags (title, artist, album, track, date, genre, cover art, lyrics)
|
||||
from source_path to dest_path. Optionally appends stem_label to the title.
|
||||
Silently ignores unsupported formats — metadata is cosmetic, never fatal.
|
||||
"""
|
||||
try:
|
||||
import mutagen
|
||||
|
||||
src = mutagen.File(source_path, easy=True)
|
||||
if src is None:
|
||||
return
|
||||
|
||||
title = src.get("title", [""])[0] if src.get("title") else ""
|
||||
artist = src.get("artist", [""])[0] if src.get("artist") else ""
|
||||
album = src.get("album", [""])[0] if src.get("album") else ""
|
||||
tracknumber = src.get("tracknumber", [""])[0] if src.get("tracknumber") else ""
|
||||
date = src.get("date", [""])[0] if src.get("date") else ""
|
||||
genre = src.get("genre", [""])[0] if src.get("genre") else ""
|
||||
|
||||
if stem_label and title:
|
||||
title = f"{title} ({stem_label})"
|
||||
elif stem_label:
|
||||
title = stem_label
|
||||
|
||||
cover_data, cover_mime = _read_cover_art(source_path)
|
||||
|
||||
# Read lyrics from source
|
||||
lyrics = ""
|
||||
if copy_lyrics:
|
||||
lyrics = _read_lyrics_from_file(source_path)
|
||||
|
||||
ext = Path(dest_path).suffix.lower()
|
||||
if ext == ".mp3":
|
||||
_write_mp3_tags(dest_path, title, artist, album, tracknumber, date, genre,
|
||||
cover_data, cover_mime, lyrics)
|
||||
elif ext == ".flac":
|
||||
_write_flac_tags(dest_path, title, artist, album, tracknumber, date, genre,
|
||||
cover_data, cover_mime, lyrics)
|
||||
elif ext == ".wav":
|
||||
_write_wav_tags(dest_path, title, artist, album, tracknumber, date, genre,
|
||||
cover_data, cover_mime)
|
||||
|
||||
except Exception:
|
||||
pass # Never fail over metadata — it's cosmetic
|
||||
|
||||
|
||||
def _read_lyrics_from_file(path: str) -> str:
|
||||
"""Read lyrics from a file using the same logic as read_metadata."""
|
||||
try:
|
||||
ext = Path(path).suffix.lower()
|
||||
if ext == '.mp3':
|
||||
return _read_mp3_lyrics(path)
|
||||
elif ext in ('.flac', '.ogg', '.opus'):
|
||||
return _read_vorbis_lyrics(path)
|
||||
except Exception:
|
||||
pass
|
||||
return ''
|
||||
|
||||
|
||||
def _write_mp3_tags(path, title, artist, album, tracknumber, date, genre,
|
||||
cover_data=None, cover_mime=None, lyrics=None):
|
||||
from mutagen.id3 import ID3, ID3NoHeaderError, TIT2, TPE1, TALB, TRCK, TDRC, TCON, APIC, USLT
|
||||
try:
|
||||
tags = ID3(path)
|
||||
except ID3NoHeaderError:
|
||||
tags = ID3()
|
||||
if title: tags["TIT2"] = TIT2(encoding=3, text=title)
|
||||
if artist: tags["TPE1"] = TPE1(encoding=3, text=artist)
|
||||
if album: tags["TALB"] = TALB(encoding=3, text=album)
|
||||
if tracknumber: tags["TRCK"] = TRCK(encoding=3, text=tracknumber)
|
||||
if date: tags["TDRC"] = TDRC(encoding=3, text=date)
|
||||
if genre: tags["TCON"] = TCON(encoding=3, text=genre)
|
||||
if lyrics: tags["USLT::'eng'"] = USLT(encoding=3, lang='eng', desc='', text=lyrics)
|
||||
if cover_data:
|
||||
tags["APIC:"] = APIC(
|
||||
encoding=3, mime=cover_mime or "image/jpeg",
|
||||
type=3, desc="Cover", data=cover_data,
|
||||
)
|
||||
tags.save(path)
|
||||
|
||||
|
||||
def _write_flac_tags(path, title, artist, album, tracknumber, date, genre,
|
||||
cover_data=None, cover_mime=None, lyrics=None):
|
||||
from mutagen.flac import FLAC, Picture
|
||||
audio = FLAC(path)
|
||||
if title: audio["title"] = title
|
||||
if artist: audio["artist"] = artist
|
||||
if album: audio["album"] = album
|
||||
if tracknumber: audio["tracknumber"] = tracknumber
|
||||
if date: audio["date"] = date
|
||||
if genre: audio["genre"] = genre
|
||||
if lyrics: audio["lyrics"] = lyrics
|
||||
if cover_data:
|
||||
pic = Picture()
|
||||
pic.type = 3 # front cover
|
||||
pic.mime = cover_mime or "image/jpeg"
|
||||
pic.desc = "Cover"
|
||||
pic.data = cover_data
|
||||
audio.clear_pictures()
|
||||
audio.add_picture(pic)
|
||||
audio.save()
|
||||
|
||||
|
||||
def _write_wav_tags(path, title, artist, album, tracknumber, date, genre,
|
||||
cover_data=None, cover_mime=None):
|
||||
from mutagen.wave import WAVE
|
||||
from mutagen.id3 import TIT2, TPE1, TALB, TRCK, TDRC, TCON, APIC
|
||||
try:
|
||||
audio = WAVE(path)
|
||||
if audio.tags is None:
|
||||
audio.add_tags()
|
||||
if title: audio.tags["TIT2"] = TIT2(encoding=3, text=title)
|
||||
if artist: audio.tags["TPE1"] = TPE1(encoding=3, text=artist)
|
||||
if album: audio.tags["TALB"] = TALB(encoding=3, text=album)
|
||||
if tracknumber: audio.tags["TRCK"] = TRCK(encoding=3, text=tracknumber)
|
||||
if date: audio.tags["TDRC"] = TDRC(encoding=3, text=date)
|
||||
if genre: audio.tags["TCON"] = TCON(encoding=3, text=genre)
|
||||
if cover_data:
|
||||
audio.tags["APIC:"] = APIC(
|
||||
encoding=3, mime=cover_mime or "image/jpeg",
|
||||
type=3, desc="Cover", data=cover_data,
|
||||
)
|
||||
audio.save()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Metadata reading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def read_metadata(source_path: str) -> dict:
|
||||
"""
|
||||
Read basic metadata from an audio file.
|
||||
Returns dict with 'artist', 'title', 'album', 'lyrics' keys (all str).
|
||||
Lyrics are read in order of priority:
|
||||
1. Synced lyrics tags (SYLT for MP3, syncedlyrics for FLAC/OGG)
|
||||
2. Unsynced lyrics tags (USLT for MP3, lyrics/unsyncedlyrics for FLAC/OGG)
|
||||
3. External .lrc file with same base name
|
||||
Silently returns empty strings on failure or unsupported format.
|
||||
"""
|
||||
result = {'artist': '', 'title': '', 'album': '', 'lyrics': ''}
|
||||
try:
|
||||
import mutagen
|
||||
src = mutagen.File(source_path, easy=True)
|
||||
if src is None:
|
||||
return result
|
||||
result['artist'] = (src.get('artist') or [''])[0]
|
||||
result['title'] = (src.get('title') or [''])[0]
|
||||
result['album'] = (src.get('album') or [''])[0]
|
||||
|
||||
ext = Path(source_path).suffix.lower()
|
||||
lyrics = ''
|
||||
|
||||
# Try to read lyrics from tags first
|
||||
if ext == '.mp3':
|
||||
lyrics = _read_mp3_lyrics(source_path)
|
||||
elif ext in ('.flac', '.ogg', '.opus'):
|
||||
lyrics = _read_vorbis_lyrics(source_path)
|
||||
|
||||
# If no lyrics in tags, try external LRC file
|
||||
if not lyrics:
|
||||
lyrics = read_lrc_file(source_path)
|
||||
|
||||
result['lyrics'] = lyrics
|
||||
except Exception:
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def _read_mp3_lyrics(path: str) -> str:
|
||||
"""Read lyrics from ID3 tags: SYLT (synced) → USLT (unsynced) → TXXX:LYRICS.
|
||||
Also checks $-prefixed variants used by some tagging software.
|
||||
"""
|
||||
try:
|
||||
from mutagen.id3 import ID3, ID3NoHeaderError
|
||||
try:
|
||||
tags = ID3(path)
|
||||
except ID3NoHeaderError:
|
||||
return ''
|
||||
|
||||
# Synced lyrics: SYLT contains list of (text, time_ms) tuples
|
||||
sylt_keys = [k for k in tags.keys() if k.startswith('SYLT')]
|
||||
if sylt_keys:
|
||||
return '\n'.join(t for t, _ in tags[sylt_keys[0]].text)
|
||||
|
||||
# Unsynced lyrics
|
||||
uslt_keys = [k for k in tags.keys() if k.startswith('USLT')]
|
||||
if uslt_keys:
|
||||
return tags[uslt_keys[0]].text
|
||||
|
||||
# TXXX:LYRICS fallback (also check $-prefixed variants)
|
||||
for txxx_key in ('TXXX:LYRICS', 'TXXX:$LYRICS',
|
||||
'TXXX:SYNCEDLYRICS', 'TXXX:$SYNCEDLYRICS',
|
||||
'TXXX:UNSYNCEDLYRICS', 'TXXX:$UNSYNCEDLYRICS'):
|
||||
if txxx_key in tags:
|
||||
text = tags[txxx_key].text
|
||||
if text:
|
||||
return text[0] if isinstance(text, list) else str(text)
|
||||
except Exception:
|
||||
pass
|
||||
return ''
|
||||
|
||||
|
||||
def _read_vorbis_lyrics(path: str) -> str:
|
||||
"""Read lyrics from FLAC/OGG Vorbis comment tags.
|
||||
|
||||
Checks SYNCEDLYRICS, LYRICS, and UNSYNCEDLYRICS (case-insensitive).
|
||||
Also checks variants with $ prefix (e.g., $LYRICS) used by some tagging software.
|
||||
Returns the first non-empty value found, or empty string if all are empty.
|
||||
"""
|
||||
try:
|
||||
import mutagen
|
||||
f = mutagen.File(path)
|
||||
if f is None:
|
||||
return ''
|
||||
|
||||
# Build case-insensitive lookup: upper() -> original key
|
||||
key_map = {k.upper(): k for k in f.keys()}
|
||||
|
||||
# Check in priority order: SYNCEDLYRICS -> LYRICS -> UNSYNCEDLYRICS
|
||||
# Also check $-prefixed variants as fallback
|
||||
for key in ('SYNCEDLYRICS', '$SYNCEDLYRICS',
|
||||
'LYRICS', '$LYRICS',
|
||||
'UNSYNCEDLYRICS', '$UNSYNCEDLYRICS'):
|
||||
if key in key_map:
|
||||
val = f.get(key_map[key])
|
||||
if val:
|
||||
return val[0]
|
||||
except Exception:
|
||||
pass
|
||||
return ''
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LRC file utilities
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
LRC_LINE_REGEX = re.compile(
|
||||
r'^(\[\s*(\d+):(\d{1,2})(?:\.(\d{1,3}))?\s*\])' # timestamp [mm:ss.xx]
|
||||
r'\s*(.*?)$', # lyrics text
|
||||
re.MULTILINE
|
||||
)
|
||||
|
||||
ELRC_WORD_REGEX = re.compile(
|
||||
r'(<\s*(\d+):(\d{1,2})(?:\.(\d{1,3}))?\s*>)' # word timestamp <mm:ss.xx>
|
||||
r'\s*([^<\n]+)', # word text
|
||||
re.MULTILINE
|
||||
)
|
||||
|
||||
|
||||
def read_lrc_file(audio_path: str) -> str:
|
||||
"""
|
||||
Read lyrics from an external .lrc file with the same base name as the audio file.
|
||||
Returns the raw LRC content or empty string if not found.
|
||||
"""
|
||||
audio_path = Path(audio_path)
|
||||
lrc_path = audio_path.with_suffix('.lrc')
|
||||
|
||||
if not lrc_path.exists():
|
||||
# Try case-insensitive search (covers both .lrc and .LRC on Linux)
|
||||
for pattern in ('*.lrc', '*.LRC'):
|
||||
for f in audio_path.parent.glob(pattern):
|
||||
if f.stem.lower() == audio_path.stem.lower():
|
||||
lrc_path = f
|
||||
break
|
||||
if lrc_path.exists():
|
||||
break
|
||||
|
||||
if lrc_path.exists():
|
||||
try:
|
||||
return lrc_path.read_text(encoding='utf-8', errors='ignore')
|
||||
except Exception:
|
||||
pass
|
||||
return ''
|
||||
|
||||
|
||||
def parse_lrc(lrc_content: str) -> List[Tuple[float, str]]:
|
||||
"""
|
||||
Parse LRC content into list of (timestamp_seconds, line_text) tuples.
|
||||
|
||||
Args:
|
||||
lrc_content: Raw LRC file content
|
||||
|
||||
Returns:
|
||||
List of (timestamp, line) tuples sorted by timestamp
|
||||
"""
|
||||
lines = []
|
||||
|
||||
for match in LRC_LINE_REGEX.finditer(lrc_content):
|
||||
minutes = int(match.group(2))
|
||||
seconds = int(match.group(3))
|
||||
millis = match.group(4)
|
||||
text = match.group(5).strip()
|
||||
|
||||
if millis:
|
||||
# Handle different millisecond precisions
|
||||
millis = int(millis.ljust(3, '0')[:3])
|
||||
else:
|
||||
millis = 0
|
||||
|
||||
timestamp = minutes * 60 + seconds + millis / 1000.0
|
||||
|
||||
if text:
|
||||
lines.append((timestamp, text))
|
||||
|
||||
# Sort by timestamp and remove duplicates
|
||||
lines.sort(key=lambda x: x[0])
|
||||
|
||||
# Remove duplicate timestamps (keep first occurrence)
|
||||
seen = set()
|
||||
unique_lines = []
|
||||
for ts, text in lines:
|
||||
if ts not in seen:
|
||||
seen.add(ts)
|
||||
unique_lines.append((ts, text))
|
||||
|
||||
return unique_lines
|
||||
|
||||
|
||||
def parse_elrc(elrc_content: str) -> List[Tuple[float, str, List[Tuple[float, str]]]]:
|
||||
"""
|
||||
Parse Enhanced LRC (ELRC) content with word-level timestamps.
|
||||
|
||||
Args:
|
||||
elrc_content: Raw ELRC file content
|
||||
|
||||
Returns:
|
||||
List of (line_timestamp, line_text, [(word_timestamp, word), ...]) tuples
|
||||
"""
|
||||
lines = []
|
||||
|
||||
for match in LRC_LINE_REGEX.finditer(elrc_content):
|
||||
minutes = int(match.group(2))
|
||||
seconds = int(match.group(3))
|
||||
millis = match.group(4)
|
||||
full_line = match.group(5)
|
||||
|
||||
if millis:
|
||||
millis = int(millis.ljust(3, '0')[:3])
|
||||
else:
|
||||
millis = 0
|
||||
|
||||
line_timestamp = minutes * 60 + seconds + millis / 1000.0
|
||||
|
||||
# Parse word-level timestamps within this line
|
||||
words = []
|
||||
for word_match in ELRC_WORD_REGEX.finditer(full_line):
|
||||
w_minutes = int(word_match.group(2))
|
||||
w_seconds = int(word_match.group(3))
|
||||
w_millis = word_match.group(4)
|
||||
w_text = word_match.group(5).strip()
|
||||
|
||||
if w_millis:
|
||||
w_millis = int(w_millis.ljust(3, '0')[:3])
|
||||
else:
|
||||
w_millis = 0
|
||||
|
||||
word_timestamp = w_minutes * 60 + w_seconds + w_millis / 1000.0
|
||||
|
||||
if w_text:
|
||||
words.append((word_timestamp, w_text))
|
||||
|
||||
# Clean line text (remove word timestamps)
|
||||
line_text = ELRC_WORD_REGEX.sub(r'\5', full_line).strip()
|
||||
|
||||
if line_text or words:
|
||||
lines.append((line_timestamp, line_text, words))
|
||||
|
||||
lines.sort(key=lambda x: x[0])
|
||||
return lines
|
||||
|
||||
|
||||
def format_lrc(lines: List[Tuple[float, str]], include_empty: bool = False) -> str:
|
||||
"""
|
||||
Format lines back into LRC format.
|
||||
|
||||
Args:
|
||||
lines: List of (timestamp_seconds, line_text) tuples
|
||||
include_empty: Whether to include empty lines
|
||||
|
||||
Returns:
|
||||
LRC formatted string
|
||||
"""
|
||||
result = []
|
||||
for timestamp, text in lines:
|
||||
if not text and not include_empty:
|
||||
continue
|
||||
minutes = int(timestamp // 60)
|
||||
seconds = int(timestamp % 60)
|
||||
millis = int((timestamp % 1) * 100)
|
||||
result.append(f"[{minutes:02d}:{seconds:02d}.{millis:02d}]{text}")
|
||||
return '\n'.join(result)
|
||||
|
||||
|
||||
def format_elrc(lines: List[Tuple[float, str, List[Tuple[float, str]]]]) -> str:
|
||||
"""
|
||||
Format lines with word-level timestamps into ELRC format.
|
||||
|
||||
Args:
|
||||
lines: List of (line_timestamp, line_text, [(word_timestamp, word), ...]) tuples
|
||||
|
||||
Returns:
|
||||
ELRC formatted string
|
||||
"""
|
||||
result = []
|
||||
for line_ts, line_text, words in lines:
|
||||
minutes = int(line_ts // 60)
|
||||
seconds = int(line_ts % 60)
|
||||
millis = int((line_ts % 1) * 100)
|
||||
|
||||
if words:
|
||||
# Build line with word timestamps
|
||||
word_parts = []
|
||||
for word_ts, word in words:
|
||||
w_min = int(word_ts // 60)
|
||||
w_sec = int(word_ts % 60)
|
||||
w_mil = int((word_ts % 1) * 100)
|
||||
word_parts.append(f"<{w_min:02d}:{w_sec:02d}.{w_mil:02d}>{word}")
|
||||
line_content = ' '.join(word_parts)
|
||||
else:
|
||||
line_content = line_text
|
||||
|
||||
result.append(f"[{minutes:02d}:{seconds:02d}.{millis:02d}]{line_content}")
|
||||
return '\n'.join(result)
|
||||
|
||||
|
||||
def lyrics_to_plaintext(lyrics: str) -> str:
|
||||
"""
|
||||
Convert LRC/ELRC content to plain text (remove all timestamps).
|
||||
|
||||
Args:
|
||||
lyrics: LRC formatted content
|
||||
|
||||
Returns:
|
||||
Plain text lyrics
|
||||
"""
|
||||
# Remove line timestamps [mm:ss.xx]
|
||||
text = re.sub(r'\[\s*\d+:\d{2}(?:\.\d{1,3})?\s*\]', '', lyrics)
|
||||
# Remove word timestamps <mm:ss.xx>
|
||||
text = re.sub(r'<\s*\d+:\d{2}(?:\.\d{1,3})?\s*>', '', text)
|
||||
# Clean up whitespace
|
||||
lines = [line.strip() for line in text.split('\n') if line.strip()]
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
def is_lrc_synced(lyrics: str) -> bool:
|
||||
"""
|
||||
Check if lyrics content contains LRC timestamps (synced lyrics).
|
||||
|
||||
Args:
|
||||
lyrics: Lyrics content to check
|
||||
|
||||
Returns:
|
||||
True if content contains LRC timestamps
|
||||
"""
|
||||
return bool(LRC_LINE_REGEX.search(lyrics))
|
||||
|
||||
|
||||
def extract_lyrics_from_file(file_path: str) -> Dict[str, str]:
|
||||
"""
|
||||
Extract lyrics from an audio file or LRC file.
|
||||
|
||||
Args:
|
||||
file_path: Path to audio file or LRC file
|
||||
|
||||
Returns:
|
||||
Dict with 'source', 'raw', 'synced', 'plain' keys
|
||||
- source: 'tag', 'lrc_file', or 'none'
|
||||
- raw: Original lyrics content with timestamps if present
|
||||
- synced: True if lyrics are timestamped
|
||||
- plain: Plain text version without timestamps
|
||||
"""
|
||||
path = Path(file_path)
|
||||
result = {'source': 'none', 'raw': '', 'synced': False, 'plain': ''}
|
||||
|
||||
# If it's an LRC file, read it directly
|
||||
if path.suffix.lower() == '.lrc':
|
||||
try:
|
||||
content = path.read_text(encoding='utf-8', errors='ignore')
|
||||
result['source'] = 'lrc_file'
|
||||
result['raw'] = content
|
||||
result['synced'] = is_lrc_synced(content)
|
||||
result['plain'] = lyrics_to_plaintext(content)
|
||||
except Exception:
|
||||
pass
|
||||
return result
|
||||
|
||||
# Otherwise, try to read from audio file metadata
|
||||
metadata = read_metadata(file_path)
|
||||
lyrics = metadata.get('lyrics', '')
|
||||
|
||||
if lyrics:
|
||||
result['source'] = 'tag'
|
||||
result['raw'] = lyrics
|
||||
result['synced'] = is_lrc_synced(lyrics)
|
||||
result['plain'] = lyrics_to_plaintext(lyrics)
|
||||
else:
|
||||
# Try external LRC file as last resort
|
||||
lrc_content = read_lrc_file(file_path)
|
||||
if lrc_content:
|
||||
result['source'] = 'lrc_file'
|
||||
result['raw'] = lrc_content
|
||||
result['synced'] = is_lrc_synced(lrc_content)
|
||||
result['plain'] = lyrics_to_plaintext(lrc_content)
|
||||
|
||||
return result
|
||||
Executable
+56
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
Mix service: create static or dynamic mixes from separated stems.
|
||||
CPU-bound (pydub) — uses the general executor, not the GPU queue.
|
||||
"""
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from .. import job_manager as jm
|
||||
from .preset_service import make_static, make_dynamic
|
||||
|
||||
|
||||
def run_mix(
|
||||
job_id: str,
|
||||
stem_files: Dict[str, str], # stem_name (lowercase) → abs file path
|
||||
preset_type: str, # 'static' | 'dynamic'
|
||||
output_path: str,
|
||||
output_format: str,
|
||||
static_option: Optional[str] = None,
|
||||
dynamic_levels: Optional[Dict[str, float]] = None,
|
||||
):
|
||||
"""Background task: create a mix from separated stems."""
|
||||
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Creating mix...")
|
||||
try:
|
||||
out = Path(output_path)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if preset_type == "static":
|
||||
muted = _get_muted_stems(static_option)
|
||||
result_path = make_static(stem_files, muted, out, output_format)
|
||||
elif preset_type == "dynamic":
|
||||
volumes = dynamic_levels or {}
|
||||
result_path = make_dynamic(stem_files, volumes, out, output_format)
|
||||
else:
|
||||
raise ValueError(f"Unknown preset type: {preset_type}")
|
||||
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.DONE,
|
||||
progress="Mix complete.",
|
||||
files=[{"name": out.name, "path": result_path, "type": preset_type}],
|
||||
)
|
||||
except Exception as exc:
|
||||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
|
||||
|
||||
|
||||
def _get_muted_stems(static_option: Optional[str]) -> List[str]:
|
||||
if not static_option:
|
||||
return []
|
||||
option_map = {
|
||||
"no_vocals": ["vocals"],
|
||||
"no_drums": ["drums"],
|
||||
"no_bass": ["bass"],
|
||||
"no_vocals_no_bass": ["vocals", "bass"],
|
||||
"no_drums_no_bass": ["drums", "bass"],
|
||||
}
|
||||
return option_map.get(static_option, [])
|
||||
+310
@@ -0,0 +1,310 @@
|
||||
"""
|
||||
Full pipeline: separate → transcribe → abc-convert.
|
||||
Supports both Demucs (CLI) and BS-RoFormer (Python API) for the separation stage.
|
||||
Uses project-based folder structure for organized output.
|
||||
"""
|
||||
import sys
|
||||
import subprocess
|
||||
import time
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import PROJECTS_DIR, JOBS_DIR, CPU_SEPARATION, TRANSCRIBE_PRESETS
|
||||
from .metadata_util import copy_metadata, read_metadata
|
||||
from .project_namer import project_name, make_project_dirs
|
||||
from .separator_service import BSR_PREFIX, BSR_STEM_MODE
|
||||
from . import abc_generator
|
||||
|
||||
|
||||
def run_pipeline(
|
||||
job_id: str,
|
||||
input_path: str,
|
||||
sep_model: str,
|
||||
sep_format: str,
|
||||
transcribe_mode: str,
|
||||
song_title: str,
|
||||
project_name_override: str = None,
|
||||
):
|
||||
"""
|
||||
Full pipeline:
|
||||
1. Stem separation (Demucs CLI or BS-RoFormer Python API)
|
||||
2. MIDI transcription (basic-pitch) for each non-drum stem
|
||||
3. ABC notation conversion
|
||||
|
||||
Creates organized project folder structure under MEDIA_DIR.
|
||||
"""
|
||||
# Read metadata for project naming
|
||||
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Reading metadata...")
|
||||
metadata = read_metadata(input_path)
|
||||
|
||||
# Determine project name
|
||||
if project_name_override:
|
||||
proj_name = project_name_override
|
||||
else:
|
||||
filename_stem = Path(input_path).stem
|
||||
proj_name = project_name(metadata.get('artist', ''), metadata.get('title', ''), filename_stem)
|
||||
|
||||
# Create project directories
|
||||
jm.update_job(job_id, progress=f"Creating project: {proj_name}...")
|
||||
dirs = make_project_dirs(
|
||||
proj_name, PROJECTS_DIR,
|
||||
{'instrumental': False, 'static': False, 'dynamic': False}
|
||||
)
|
||||
|
||||
project_dir = dirs['project']
|
||||
stems_dir = dirs['stems']
|
||||
transcriptions_dir = project_dir / "Transcriptions"
|
||||
transcriptions_dir.mkdir(exist_ok=True)
|
||||
|
||||
all_files = []
|
||||
|
||||
# Normalise mp3_320 → mp3 with explicit bitrate
|
||||
mp3_bitrate = '320k' if sep_format == 'mp3_320' else '256k'
|
||||
normalised_format = 'mp3' if sep_format == 'mp3_320' else sep_format
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Stage 1: Separation
|
||||
# ------------------------------------------------------------------
|
||||
jm.update_job(job_id, progress="[1/3] Separating stems...")
|
||||
|
||||
try:
|
||||
if sep_model.startswith(BSR_PREFIX):
|
||||
stem_files = _separate_bsroformer(
|
||||
job_id, input_path, sep_model, normalised_format, stems_dir, mp3_bitrate
|
||||
)
|
||||
else:
|
||||
stem_files = _separate_demucs(
|
||||
job_id, input_path, sep_model, sep_format, stems_dir
|
||||
)
|
||||
except Exception as exc:
|
||||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Separation failed.")
|
||||
return
|
||||
|
||||
if stem_files is None:
|
||||
return # cancelled inside sub-function
|
||||
|
||||
# Record stems with proper metadata
|
||||
for stem_name, path in sorted(stem_files.items()):
|
||||
copy_metadata(input_path, path, stem_label=stem_name.capitalize())
|
||||
all_files.append({
|
||||
"name": Path(path).name,
|
||||
"path": path,
|
||||
"stage": "separation",
|
||||
"stem": stem_name
|
||||
})
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Stage 2: Transcription (skip drum stems)
|
||||
# ------------------------------------------------------------------
|
||||
from basic_pitch.inference import predict
|
||||
from basic_pitch import ICASSP_2022_MODEL_PATH
|
||||
|
||||
jm.update_job(job_id, progress="[2/3] Transcribing stems to MIDI...")
|
||||
|
||||
midi_files = []
|
||||
transcribe_paths = [
|
||||
(name, path) for name, path in stem_files.items()
|
||||
if 'drums' not in name.lower()
|
||||
]
|
||||
params = TRANSCRIBE_PRESETS.get(transcribe_mode, TRANSCRIBE_PRESETS["polyphonic"])
|
||||
|
||||
for stem_name, stem_path in transcribe_paths:
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
jm.update_job(job_id, progress=f"[2/3] Transcribing: {stem_name}...")
|
||||
|
||||
try:
|
||||
_, midi_data, _ = predict(stem_path, ICASSP_2022_MODEL_PATH, **params)
|
||||
|
||||
output_name = f"{proj_name} - {stem_name.capitalize()} ({transcribe_mode}).mid"
|
||||
dest = transcriptions_dir / output_name
|
||||
midi_data.write(str(dest))
|
||||
all_files.append({
|
||||
"name": dest.name,
|
||||
"path": str(dest),
|
||||
"stage": "transcription"
|
||||
})
|
||||
midi_files.append(str(dest))
|
||||
|
||||
except Exception as exc:
|
||||
jm.update_job(job_id, progress=f"[2/3] Error on {stem_name}: {exc}")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Stage 3: ABC conversion
|
||||
# ------------------------------------------------------------------
|
||||
if midi_files:
|
||||
jm.update_job(job_id, progress="[3/3] Converting MIDI to ABC notation...")
|
||||
try:
|
||||
abc_results = abc_generator.convert_midi_to_abc(midi_files, song_title or proj_name)
|
||||
if abc_results:
|
||||
for part_name, abc_text in abc_results.items():
|
||||
safe = part_name.replace("/", "_").replace("\\", "_")
|
||||
dest = transcriptions_dir / f"{safe}.abc"
|
||||
dest.write_text(abc_text, encoding="utf-8")
|
||||
all_files.append({
|
||||
"name": dest.name,
|
||||
"path": str(dest),
|
||||
"stage": "abc"
|
||||
})
|
||||
except Exception as exc:
|
||||
jm.update_job(job_id, progress=f"[3/3] ABC conversion error: {exc}")
|
||||
|
||||
jm.update_job(
|
||||
job_id, status=jm.JobStatus.DONE,
|
||||
progress="Pipeline complete.",
|
||||
files=all_files,
|
||||
result={
|
||||
"project_name": proj_name,
|
||||
"project_path": str(project_dir),
|
||||
"stems_dir": str(stems_dir),
|
||||
"transcriptions_dir": str(transcriptions_dir),
|
||||
}
|
||||
)
|
||||
jm.find_or_create_project(proj_name, job_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Separation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _separate_demucs(
|
||||
job_id: str, input_path: str, model: str, output_format: str, stems_dir: Path
|
||||
) -> dict | None:
|
||||
"""
|
||||
Run Demucs CLI; returns dict mapping stem names to file paths, or None if cancelled.
|
||||
"""
|
||||
ext = output_format.lower()
|
||||
fmt_flag = []
|
||||
if ext == "mp3_320":
|
||||
fmt_flag = ["--mp3", "--mp3-bitrate", "320"]
|
||||
ext = "mp3"
|
||||
elif ext == "mp3":
|
||||
fmt_flag = ["--mp3", "--mp3-bitrate", "256"]
|
||||
elif ext == "flac":
|
||||
fmt_flag = ["--flac"]
|
||||
|
||||
# Use temp directory for demucs output
|
||||
temp_out = JOBS_DIR / job_id / "demucs_out"
|
||||
temp_out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
cmd = [
|
||||
sys.executable, "-m", "demucs",
|
||||
"--name", model,
|
||||
"--out", str(temp_out),
|
||||
*fmt_flag,
|
||||
input_path,
|
||||
]
|
||||
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
|
||||
)
|
||||
while True:
|
||||
if jm.should_cancel(job_id):
|
||||
process.terminate()
|
||||
process.wait()
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return None
|
||||
line = process.stdout.readline()
|
||||
if line:
|
||||
jm.update_job(job_id, progress=f"[1/3] {line.rstrip()}")
|
||||
elif process.poll() is not None:
|
||||
break
|
||||
else:
|
||||
time.sleep(0.1)
|
||||
|
||||
if process.returncode != 0:
|
||||
raise RuntimeError(f"Demucs failed with code {process.returncode}")
|
||||
|
||||
input_stem = Path(input_path).stem
|
||||
stem_src_dir = temp_out / model / input_stem
|
||||
|
||||
stem_files = {}
|
||||
if stem_src_dir.exists():
|
||||
for f in sorted(stem_src_dir.iterdir()):
|
||||
if f.is_file():
|
||||
stem_name = f.stem.lower()
|
||||
for prefix in [input_stem.lower(), 'htdemucs', 'mdx']:
|
||||
stem_name = stem_name.replace(prefix, '').strip('_- ')
|
||||
if not stem_name:
|
||||
stem_name = f.stem
|
||||
|
||||
dest = stems_dir / f"{input_stem} - {stem_name.capitalize()}{f.suffix}"
|
||||
shutil.move(str(f), str(dest))
|
||||
stem_files[stem_name] = str(dest)
|
||||
|
||||
try:
|
||||
shutil.rmtree(temp_out, ignore_errors=True)
|
||||
except:
|
||||
pass
|
||||
|
||||
return stem_files
|
||||
|
||||
except Exception:
|
||||
try:
|
||||
shutil.rmtree(temp_out, ignore_errors=True)
|
||||
except:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def _separate_bsroformer(
|
||||
job_id: str, input_path: str, model: str, output_format: str, stems_dir: Path,
|
||||
mp3_bitrate: str = '256k'
|
||||
) -> dict | None:
|
||||
"""
|
||||
Run BS-RoFormer in-process; returns dict mapping stem names to file paths, or None if cancelled.
|
||||
"""
|
||||
from .bs_roformer_separator import BSRoformerSeparator
|
||||
|
||||
stem_mode = BSR_STEM_MODE.get(model, "4stem")
|
||||
jm.update_job(job_id, progress=f"[1/3] Loading BS-RoFormer ({stem_mode})...")
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return None
|
||||
|
||||
sep = BSRoformerSeparator(
|
||||
cpu_separation=CPU_SEPARATION,
|
||||
output_format=output_format,
|
||||
mp3_bitrate=mp3_bitrate,
|
||||
stem_mode=stem_mode,
|
||||
)
|
||||
|
||||
jm.update_job(job_id, progress="[1/3] Separating with BS-RoFormer...")
|
||||
|
||||
temp_out = JOBS_DIR / job_id / "bsr_out"
|
||||
temp_out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return None
|
||||
|
||||
input_stem = Path(input_path).stem
|
||||
stem_files = {}
|
||||
|
||||
for stem_name, path in sorted(stem_paths.items()):
|
||||
src = Path(path)
|
||||
dest = stems_dir / f"{input_stem} - {stem_name.capitalize()}{src.suffix}"
|
||||
shutil.move(str(src), str(dest))
|
||||
stem_files[stem_name.lower()] = str(dest)
|
||||
|
||||
try:
|
||||
shutil.rmtree(temp_out, ignore_errors=True)
|
||||
except:
|
||||
pass
|
||||
|
||||
return stem_files
|
||||
|
||||
except Exception:
|
||||
try:
|
||||
shutil.rmtree(temp_out, ignore_errors=True)
|
||||
except:
|
||||
pass
|
||||
raise
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
"""
|
||||
Preset mixing: combine separated stems into a single output file.
|
||||
|
||||
- Instrumental: all non-vocal stems mixed
|
||||
- Static: all stems except user-selected ones mixed
|
||||
- Dynamic: all stems mixed with per-stem volume multipliers (0.0–2.0)
|
||||
"""
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
from pydub import AudioSegment
|
||||
|
||||
|
||||
def _load(path: str) -> AudioSegment:
|
||||
return AudioSegment.from_file(path)
|
||||
|
||||
|
||||
def _apply_gain(audio: AudioSegment, multiplier: float) -> AudioSegment:
|
||||
"""Convert a linear multiplier to dB and apply as gain."""
|
||||
if multiplier <= 0.0:
|
||||
return AudioSegment.silent(duration=len(audio), frame_rate=audio.frame_rate)
|
||||
gain_db = 20 * math.log10(multiplier)
|
||||
return audio + gain_db
|
||||
|
||||
|
||||
def _export(audio: AudioSegment, dest: Path, output_format: str) -> None:
|
||||
if output_format == 'mp3_320':
|
||||
audio.export(str(dest), format='mp3', bitrate='320k')
|
||||
elif output_format == 'mp3':
|
||||
audio.export(str(dest), format='mp3', bitrate='256k')
|
||||
elif output_format == 'flac':
|
||||
audio.export(str(dest), format='flac')
|
||||
else:
|
||||
audio.export(str(dest), format='wav')
|
||||
|
||||
|
||||
def _mix(paths: List[str]) -> AudioSegment:
|
||||
"""Overlay all audio files into one mix."""
|
||||
if not paths:
|
||||
raise ValueError("No stems to mix")
|
||||
result = _load(paths[0])
|
||||
for p in paths[1:]:
|
||||
result = result.overlay(_load(p))
|
||||
return result
|
||||
|
||||
|
||||
def make_instrumental(
|
||||
stem_files: Dict[str, str],
|
||||
output_path: Path,
|
||||
output_format: str,
|
||||
) -> str:
|
||||
"""Mix all stems except those with 'vocal' in the stem name."""
|
||||
keep = [p for name, p in stem_files.items() if 'vocal' not in name.lower()]
|
||||
if not keep:
|
||||
raise RuntimeError("No non-vocal stems available for Instrumental preset")
|
||||
_export(_mix(keep), output_path, output_format)
|
||||
return str(output_path)
|
||||
|
||||
|
||||
def make_static(
|
||||
stem_files: Dict[str, str],
|
||||
muted_stems: List[str],
|
||||
output_path: Path,
|
||||
output_format: str,
|
||||
) -> str:
|
||||
"""Mix all stems except those listed in muted_stems."""
|
||||
muted_lower = {s.lower() for s in muted_stems}
|
||||
keep = [p for name, p in stem_files.items() if name.lower() not in muted_lower]
|
||||
if not keep:
|
||||
raise RuntimeError("All stems are muted — nothing to mix for Static preset")
|
||||
_export(_mix(keep), output_path, output_format)
|
||||
return str(output_path)
|
||||
|
||||
|
||||
def make_dynamic(
|
||||
stem_files: Dict[str, str],
|
||||
volumes: Dict[str, float],
|
||||
output_path: Path,
|
||||
output_format: str,
|
||||
) -> str:
|
||||
"""
|
||||
Mix stems with per-stem linear volume multipliers.
|
||||
volumes keys are stem names (lowercase); missing stems default to 1.0.
|
||||
"""
|
||||
adjusted: List[AudioSegment] = []
|
||||
for name, path in stem_files.items():
|
||||
vol = volumes.get(name.lower(), 1.0)
|
||||
adjusted.append(_apply_gain(_load(path), vol))
|
||||
if not adjusted:
|
||||
raise RuntimeError("No stems available for Dynamic preset")
|
||||
result = adjusted[0]
|
||||
for a in adjusted[1:]:
|
||||
result = result.overlay(a)
|
||||
_export(result, output_path, output_format)
|
||||
return str(output_path)
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
"""
|
||||
Project metadata file service.
|
||||
|
||||
Writes/reads a project.json snapshot in each project root folder.
|
||||
This file mirrors the DB state and can be used for browsing, repair,
|
||||
and offline project identification without a database query.
|
||||
"""
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
|
||||
_METADATA_FILENAME = "project.json"
|
||||
|
||||
|
||||
def write_project_metadata(
|
||||
project_dir: Path,
|
||||
*,
|
||||
project_id: str,
|
||||
name: str,
|
||||
artist: str = "",
|
||||
title: str = "",
|
||||
model: str = "",
|
||||
model_label: str = "",
|
||||
format: str = "",
|
||||
format_label: str = "",
|
||||
source_file: str = "",
|
||||
stems: dict,
|
||||
sep_job_id: str = "",
|
||||
trans_jobs: Optional[dict] = None,
|
||||
abc_jobs: Optional[dict] = None,
|
||||
lrc_jobs: Optional[dict] = None,
|
||||
mix_job_id: Optional[str] = None,
|
||||
created_at: Optional[str] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Write (or overwrite) project.json in *project_dir*.
|
||||
|
||||
If the file already exists its contents are merged so that keys not
|
||||
supplied by the caller (e.g. trans_jobs added later) are preserved.
|
||||
"""
|
||||
meta_path = project_dir / _METADATA_FILENAME
|
||||
|
||||
# Load existing data so we can merge incremental updates
|
||||
existing: dict = {}
|
||||
if meta_path.exists():
|
||||
try:
|
||||
existing = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
existing = {}
|
||||
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
updated = {
|
||||
**existing,
|
||||
"project_id": project_id,
|
||||
"name": name,
|
||||
"artist": artist,
|
||||
"title": title,
|
||||
"model": model,
|
||||
"model_label": model_label,
|
||||
"format": format,
|
||||
"format_label": format_label,
|
||||
"created_at": created_at or existing.get("created_at", now),
|
||||
"updated_at": now,
|
||||
"source_file": source_file or existing.get("source_file", ""),
|
||||
"stems": stems if stems is not None else existing.get("stems", {}),
|
||||
"sep_job_id": sep_job_id or existing.get("sep_job_id", ""),
|
||||
"trans_jobs": trans_jobs if trans_jobs is not None else existing.get("trans_jobs", {}),
|
||||
"abc_jobs": abc_jobs if abc_jobs is not None else existing.get("abc_jobs", {}),
|
||||
"lrc_jobs": lrc_jobs if lrc_jobs is not None else existing.get("lrc_jobs", {}),
|
||||
"mix_job_id": mix_job_id if mix_job_id is not None else existing.get("mix_job_id"),
|
||||
}
|
||||
|
||||
meta_path.write_text(
|
||||
json.dumps(updated, indent=2, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def read_project_metadata(project_dir: Path) -> Optional[dict]:
|
||||
"""
|
||||
Read project.json from *project_dir*.
|
||||
Returns None if the file does not exist or cannot be parsed.
|
||||
"""
|
||||
meta_path = project_dir / _METADATA_FILENAME
|
||||
if not meta_path.exists():
|
||||
return None
|
||||
try:
|
||||
return json.loads(meta_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return None
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
"""
|
||||
Project folder naming and structure creation.
|
||||
|
||||
Derives a human-readable project name from audio metadata and creates the
|
||||
subdirectory layout under MEDIA_DIR.
|
||||
"""
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
_ILLEGAL = re.compile(r'[<>:"/\\|?*\x00-\x1f]')
|
||||
|
||||
|
||||
def _sanitize(s: str) -> str:
|
||||
"""Remove filesystem-illegal characters and trim trailing dots/spaces."""
|
||||
s = _ILLEGAL.sub('', s).strip().rstrip('. ')
|
||||
return s or ''
|
||||
|
||||
|
||||
def project_name(artist: str, title: str, filename_stem: str = '') -> str:
|
||||
"""
|
||||
Return a human-readable project folder name.
|
||||
Format: "Artist - Title". Falls back to filename_stem if both are missing.
|
||||
"""
|
||||
a = _sanitize(artist)
|
||||
t = _sanitize(title)
|
||||
if a and t:
|
||||
return f"{a} - {t}"
|
||||
if t:
|
||||
return t
|
||||
if a:
|
||||
return a
|
||||
fs = _sanitize(filename_stem)
|
||||
return fs if fs else 'Unknown Project'
|
||||
|
||||
|
||||
def make_project_dirs(
|
||||
name: str,
|
||||
media_dir: Path,
|
||||
presets: Dict[str, bool],
|
||||
) -> Dict[str, Path]:
|
||||
"""
|
||||
Create project subdirectories under media_dir/name/.
|
||||
|
||||
Always creates Stems/. Creates Instrumental/, Static/, Dynamic/ only when
|
||||
the corresponding preset flag is True.
|
||||
|
||||
Returns a dict mapping role names ('project', 'stems', 'instrumental',
|
||||
'static', 'dynamic') to their Path objects.
|
||||
"""
|
||||
project_dir = media_dir / name
|
||||
stems_dir = project_dir / 'Stems'
|
||||
stems_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
dirs: Dict[str, Path] = {'project': project_dir, 'stems': stems_dir}
|
||||
|
||||
if presets.get('instrumental'):
|
||||
d = project_dir / 'Instrumental'
|
||||
d.mkdir(exist_ok=True)
|
||||
dirs['instrumental'] = d
|
||||
|
||||
if presets.get('static'):
|
||||
d = project_dir / 'Static'
|
||||
d.mkdir(exist_ok=True)
|
||||
dirs['static'] = d
|
||||
|
||||
if presets.get('dynamic'):
|
||||
d = project_dir / 'Dynamic'
|
||||
d.mkdir(exist_ok=True)
|
||||
dirs['dynamic'] = d
|
||||
|
||||
return dirs
|
||||
Executable
+321
@@ -0,0 +1,321 @@
|
||||
"""
|
||||
SCNet model architecture — vendored from:
|
||||
https://github.com/ZFTurbo/Music-Source-Separation-Training (MIT license)
|
||||
Paper: https://arxiv.org/abs/2401.13276
|
||||
|
||||
Contains SeparationNet and SCNet classes only; no training or I/O code.
|
||||
"""
|
||||
import math
|
||||
from collections import deque
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch.nn.modules.rnn import LSTM
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SeparationNet (dual-path RNN + FFT feature conversion)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class FeatureConversion(nn.Module):
|
||||
def __init__(self, channels, inverse):
|
||||
super().__init__()
|
||||
self.inverse = inverse
|
||||
self.channels = channels
|
||||
|
||||
def forward(self, x):
|
||||
if self.inverse:
|
||||
x = x.float()
|
||||
x_r = x[:, :self.channels // 2, :, :]
|
||||
x_i = x[:, self.channels // 2:, :, :]
|
||||
x = torch.complex(x_r, x_i)
|
||||
x = torch.fft.irfft(x, dim=3, norm="ortho")
|
||||
else:
|
||||
x = x.float()
|
||||
x = torch.fft.rfft(x, dim=3, norm="ortho")
|
||||
x = torch.cat([x.real, x.imag], dim=1)
|
||||
return x
|
||||
|
||||
|
||||
class DualPathRNN(nn.Module):
|
||||
def __init__(self, d_model, expand, bidirectional=True):
|
||||
super().__init__()
|
||||
self.d_model = d_model
|
||||
self.hidden_size = d_model * expand
|
||||
self.lstm_layers = nn.ModuleList([
|
||||
LSTM(d_model, self.hidden_size, num_layers=1, bidirectional=bidirectional, batch_first=True)
|
||||
for _ in range(2)
|
||||
])
|
||||
self.linear_layers = nn.ModuleList([nn.Linear(self.hidden_size * 2, d_model) for _ in range(2)])
|
||||
self.norm_layers = nn.ModuleList([nn.GroupNorm(1, d_model) for _ in range(2)])
|
||||
|
||||
def forward(self, x):
|
||||
B, C, F, T = x.shape
|
||||
original_x = x
|
||||
x = self.norm_layers[0](x)
|
||||
x = x.transpose(1, 3).contiguous().view(B * T, F, C)
|
||||
x, _ = self.lstm_layers[0](x)
|
||||
x = self.linear_layers[0](x)
|
||||
x = x.view(B, T, F, C).transpose(1, 3) + original_x
|
||||
|
||||
original_x = x
|
||||
x = self.norm_layers[1](x)
|
||||
x = x.transpose(1, 2).contiguous().view(B * F, C, T).transpose(1, 2)
|
||||
x, _ = self.lstm_layers[1](x)
|
||||
x = self.linear_layers[1](x)
|
||||
x = x.transpose(1, 2).contiguous().view(B, F, C, T).transpose(1, 2) + original_x
|
||||
return x
|
||||
|
||||
|
||||
class SeparationNet(nn.Module):
|
||||
def __init__(self, channels, expand=1, num_layers=6):
|
||||
super().__init__()
|
||||
self.num_layers = num_layers
|
||||
self.dp_modules = nn.ModuleList([
|
||||
DualPathRNN(channels * (2 if i % 2 == 1 else 1), expand)
|
||||
for i in range(num_layers)
|
||||
])
|
||||
self.feature_conversion = nn.ModuleList([
|
||||
FeatureConversion(channels * 2, inverse=(i % 2 == 1))
|
||||
for i in range(num_layers)
|
||||
])
|
||||
|
||||
def forward(self, x):
|
||||
for dp, fc in zip(self.dp_modules, self.feature_conversion):
|
||||
x = dp(x)
|
||||
x = fc(x)
|
||||
return x
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SCNet building blocks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class Swish(nn.Module):
|
||||
def forward(self, x):
|
||||
return x * x.sigmoid()
|
||||
|
||||
|
||||
class ConvolutionModule(nn.Module):
|
||||
def __init__(self, channels, depth=2, compress=4, kernel=3):
|
||||
super().__init__()
|
||||
assert kernel % 2 == 1
|
||||
hidden = int(channels / compress)
|
||||
norm = lambda d: nn.GroupNorm(1, d)
|
||||
self.layers = nn.ModuleList()
|
||||
for _ in range(abs(depth)):
|
||||
p = kernel // 2
|
||||
self.layers.append(nn.Sequential(
|
||||
norm(channels),
|
||||
nn.Conv1d(channels, hidden * 2, kernel, padding=p),
|
||||
nn.GLU(1),
|
||||
nn.Conv1d(hidden, hidden, kernel, padding=p, groups=hidden),
|
||||
norm(hidden),
|
||||
Swish(),
|
||||
nn.Conv1d(hidden, channels, 1),
|
||||
))
|
||||
|
||||
def forward(self, x):
|
||||
for layer in self.layers:
|
||||
x = x + layer(x)
|
||||
return x
|
||||
|
||||
|
||||
class FusionLayer(nn.Module):
|
||||
def __init__(self, channels, kernel_size=3, stride=1, padding=1):
|
||||
super().__init__()
|
||||
self.conv = nn.Conv2d(channels * 2, channels * 2, kernel_size, stride=stride, padding=padding)
|
||||
|
||||
def forward(self, x, skip=None):
|
||||
if skip is not None:
|
||||
x = x + skip
|
||||
x = self.conv(x.repeat(1, 2, 1, 1))
|
||||
return F.glu(x, dim=1)
|
||||
|
||||
|
||||
class SDlayer(nn.Module):
|
||||
def __init__(self, channels_in, channels_out, band_configs):
|
||||
super().__init__()
|
||||
self.convs = nn.ModuleList()
|
||||
self.strides, self.kernels = [], []
|
||||
for cfg in band_configs.values():
|
||||
self.convs.append(
|
||||
nn.Conv2d(channels_in, channels_out, (cfg['kernel'], 1), (cfg['stride'], 1))
|
||||
)
|
||||
self.strides.append(cfg['stride'])
|
||||
self.kernels.append(cfg['kernel'])
|
||||
self.SR_low = band_configs['low']['SR']
|
||||
self.SR_mid = band_configs['mid']['SR']
|
||||
|
||||
def forward(self, x):
|
||||
B, C, Fr, T = x.shape
|
||||
splits = [
|
||||
(0, math.ceil(Fr * self.SR_low)),
|
||||
(math.ceil(Fr * self.SR_low), math.ceil(Fr * (self.SR_low + self.SR_mid))),
|
||||
(math.ceil(Fr * (self.SR_low + self.SR_mid)), Fr),
|
||||
]
|
||||
outputs, original_lengths = [], []
|
||||
for conv, stride, kernel, (start, end) in zip(self.convs, self.strides, self.kernels, splits):
|
||||
extracted = x[:, :, start:end, :]
|
||||
original_lengths.append(end - start)
|
||||
L = extracted.shape[2]
|
||||
total_pad = (kernel - stride) if stride == 1 else (stride - L % stride) % stride
|
||||
pad_l = total_pad // 2
|
||||
outputs.append(conv(F.pad(extracted, (0, 0, pad_l, total_pad - pad_l))))
|
||||
return outputs, original_lengths
|
||||
|
||||
|
||||
class SUlayer(nn.Module):
|
||||
def __init__(self, channels_in, channels_out, band_configs):
|
||||
super().__init__()
|
||||
self.convtrs = nn.ModuleList([
|
||||
nn.ConvTranspose2d(channels_in, channels_out, [cfg['kernel'], 1], [cfg['stride'], 1])
|
||||
for cfg in band_configs.values()
|
||||
])
|
||||
|
||||
def forward(self, x, lengths, origin_lengths):
|
||||
splits = [
|
||||
(0, lengths[0]),
|
||||
(lengths[0], lengths[0] + lengths[1]),
|
||||
(lengths[0] + lengths[1], None),
|
||||
]
|
||||
outputs = []
|
||||
for convtr, (start, end), orig_len in zip(self.convtrs, splits, origin_lengths):
|
||||
out = convtr(x[:, :, start:end, :])
|
||||
dist = abs(orig_len - out.shape[2]) // 2
|
||||
outputs.append(out[:, :, dist:dist + orig_len, :])
|
||||
return torch.cat(outputs, dim=2)
|
||||
|
||||
|
||||
class SDblock(nn.Module):
|
||||
def __init__(self, channels_in, channels_out, band_configs={}, conv_config={}, depths=[3, 2, 1], kernel_size=3):
|
||||
super().__init__()
|
||||
self.SDlayer = SDlayer(channels_in, channels_out, band_configs)
|
||||
self.conv_modules = nn.ModuleList([
|
||||
ConvolutionModule(channels_out, depth, **conv_config) for depth in depths
|
||||
])
|
||||
self.globalconv = nn.Conv2d(channels_out, channels_out, kernel_size, 1, (kernel_size - 1) // 2)
|
||||
|
||||
def forward(self, x):
|
||||
bands, original_lengths = self.SDlayer(x)
|
||||
bands = [
|
||||
F.gelu(
|
||||
conv(band.permute(0, 2, 1, 3).reshape(-1, band.shape[1], band.shape[3]))
|
||||
.view(band.shape[0], band.shape[2], band.shape[1], band.shape[3])
|
||||
.permute(0, 2, 1, 3)
|
||||
)
|
||||
for conv, band in zip(self.conv_modules, bands)
|
||||
]
|
||||
lengths = [b.size(-2) for b in bands]
|
||||
full_band = torch.cat(bands, dim=2)
|
||||
return self.globalconv(full_band), full_band, lengths, original_lengths
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SCNet
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class SCNet(nn.Module):
|
||||
"""
|
||||
Sparse Compression Network for music source separation.
|
||||
Paper: https://arxiv.org/abs/2401.13276
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sources=['drums', 'bass', 'other', 'vocals'],
|
||||
audio_channels=2,
|
||||
dims=[4, 32, 64, 128],
|
||||
nfft=4096,
|
||||
hop_size=1024,
|
||||
win_size=4096,
|
||||
normalized=True,
|
||||
band_SR=[0.175, 0.392, 0.433],
|
||||
band_stride=[1, 4, 16],
|
||||
band_kernel=[3, 4, 16],
|
||||
conv_depths=[3, 2, 1],
|
||||
compress=4,
|
||||
conv_kernel=3,
|
||||
num_dplayer=6,
|
||||
expand=1,
|
||||
):
|
||||
super().__init__()
|
||||
self.sources = sources
|
||||
self.audio_channels = audio_channels
|
||||
self.dims = dims
|
||||
band_keys = ['low', 'mid', 'high']
|
||||
self.band_configs = {
|
||||
band_keys[i]: {'SR': band_SR[i], 'stride': band_stride[i], 'kernel': band_kernel[i]}
|
||||
for i in range(3)
|
||||
}
|
||||
self.hop_length = hop_size
|
||||
self.conv_config = {'compress': compress, 'kernel': conv_kernel}
|
||||
self.stft_config = {
|
||||
'n_fft': nfft, 'hop_length': hop_size, 'win_length': win_size,
|
||||
'center': True, 'normalized': normalized,
|
||||
}
|
||||
|
||||
self.encoder = nn.ModuleList()
|
||||
self.decoder = nn.ModuleList()
|
||||
|
||||
for i in range(len(dims) - 1):
|
||||
self.encoder.append(SDblock(
|
||||
channels_in=dims[i], channels_out=dims[i + 1],
|
||||
band_configs=self.band_configs, conv_config=self.conv_config,
|
||||
depths=conv_depths,
|
||||
))
|
||||
self.decoder.insert(0, nn.Sequential(
|
||||
FusionLayer(channels=dims[i + 1]),
|
||||
SUlayer(
|
||||
channels_in=dims[i + 1],
|
||||
channels_out=dims[i] if i != 0 else dims[i] * len(sources),
|
||||
band_configs=self.band_configs,
|
||||
),
|
||||
))
|
||||
|
||||
self.separation_net = SeparationNet(
|
||||
channels=dims[-1], expand=expand, num_layers=num_dplayer,
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
B = x.shape[0]
|
||||
padding = self.hop_length - x.shape[-1] % self.hop_length
|
||||
if (x.shape[-1] + padding) // self.hop_length % 2 == 0:
|
||||
padding += self.hop_length
|
||||
x = F.pad(x, (0, padding))
|
||||
|
||||
L = x.shape[-1]
|
||||
x = x.reshape(-1, L)
|
||||
x = torch.stft(x, **self.stft_config, return_complex=True)
|
||||
x = torch.view_as_real(x)
|
||||
x = x.permute(0, 3, 1, 2).reshape(
|
||||
x.shape[0] // self.audio_channels,
|
||||
x.shape[3] * self.audio_channels,
|
||||
x.shape[1], x.shape[2],
|
||||
)
|
||||
B, C, Fr, T = x.shape
|
||||
|
||||
save_skip = deque()
|
||||
save_lengths = deque()
|
||||
save_orig = deque()
|
||||
|
||||
for sd_layer in self.encoder:
|
||||
x, skip, lengths, orig_len = sd_layer(x)
|
||||
save_skip.append(skip)
|
||||
save_lengths.append(lengths)
|
||||
save_orig.append(orig_len)
|
||||
|
||||
x = self.separation_net(x)
|
||||
|
||||
for fusion_layer, su_layer in self.decoder:
|
||||
x = fusion_layer(x, save_skip.pop())
|
||||
x = su_layer(x, save_lengths.pop(), save_orig.pop())
|
||||
|
||||
n = self.dims[0]
|
||||
x = x.view(B, n, -1, Fr, T)
|
||||
x = x.reshape(-1, 2, Fr, T).permute(0, 2, 3, 1)
|
||||
x = torch.view_as_complex(x.contiguous())
|
||||
x = torch.istft(x, **self.stft_config)
|
||||
x = x.reshape(B, len(self.sources), self.audio_channels, -1)
|
||||
return x[:, :, :, :-padding]
|
||||
+299
@@ -0,0 +1,299 @@
|
||||
"""
|
||||
SCNet separator — native PyTorch inference.
|
||||
Models from MVSEP-Central-Backup (HuggingFace).
|
||||
Architecture vendored from MSST (MIT license):
|
||||
https://github.com/ZFTurbo/Music-Source-Separation-Training
|
||||
"""
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torchaudio
|
||||
import yaml
|
||||
|
||||
from .scnet_arch import SCNet
|
||||
from ..config import SCNET_MODEL_DIR, CPU_SEPARATION
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model registry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SCNET_MODELS = {
|
||||
"scnet_xl": {
|
||||
"model_file": "scnet_checkpoint_musdb18.ckpt",
|
||||
"config_file": "config_musdb18_scnet_xl.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_model_path": "Politrees/UVR_resources/models/SCnet/scnet_checkpoint_musdb18.ckpt",
|
||||
"hf_config_path": "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_xl.yaml",
|
||||
},
|
||||
"scnet_xl_ihf": {
|
||||
"model_file": "scnet_checkpoint_musdb18.ckpt",
|
||||
"config_file": "config_musdb18_scnet_xl.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_model_path": "Politrees/UVR_resources/models/SCnet/scnet_checkpoint_musdb18.ckpt",
|
||||
"hf_config_path": "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_xl.yaml",
|
||||
},
|
||||
"scnet_large": {
|
||||
"model_file": "SCNet-large_starrytong_fixed.ckpt",
|
||||
"config_file": "config_musdb18_scnet_large_starrytong.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_model_path": "Politrees/UVR_resources/models/SCnet/SCNet-large_starrytong_fixed.ckpt",
|
||||
"hf_config_path": "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_large_starrytong.yaml",
|
||||
},
|
||||
"scnet_large_starrytong": {
|
||||
"model_file": "SCNet-large_starrytong_fixed.ckpt",
|
||||
"config_file": "config_musdb18_scnet_large_starrytong.yaml",
|
||||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||||
"hf_model_path": "Politrees/UVR_resources/models/SCnet/SCNet-large_starrytong_fixed.ckpt",
|
||||
"hf_config_path": "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_large_starrytong.yaml",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Audio I/O helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_audio(path: str, sample_rate: int) -> np.ndarray:
|
||||
"""Load audio and return (2, samples) float32 array at target sample rate."""
|
||||
waveform, sr = torchaudio.load(path)
|
||||
if sr != sample_rate:
|
||||
waveform = torchaudio.functional.resample(waveform, sr, sample_rate)
|
||||
if waveform.shape[0] == 1:
|
||||
waveform = waveform.repeat(2, 1)
|
||||
elif waveform.shape[0] > 2:
|
||||
waveform = waveform[:2]
|
||||
return waveform.numpy().astype(np.float32)
|
||||
|
||||
|
||||
def _save_audio(path: str, audio: np.ndarray, sample_rate: int, output_format: str):
|
||||
"""Save (channels, samples) numpy array to file via torchaudio (ffmpeg backend)."""
|
||||
tensor = torch.tensor(audio, dtype=torch.float32)
|
||||
fmt = output_format.lower()
|
||||
if fmt == "mp3":
|
||||
torchaudio.save(path, tensor, sample_rate, format="mp3")
|
||||
elif fmt == "flac":
|
||||
torchaudio.save(path, tensor, sample_rate, format="flac")
|
||||
else:
|
||||
torchaudio.save(path, tensor, sample_rate)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Overlapping-window inference (MSST style)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _build_window(chunk_size: int, step: int) -> np.ndarray:
|
||||
"""
|
||||
Linear fade-in / flat / fade-out window of length chunk_size.
|
||||
Consecutive windows (offset by step) partition of unity: they sum to a
|
||||
constant, so the overlap-add correctly averages.
|
||||
"""
|
||||
fade = step
|
||||
fade_in = np.linspace(0.0, 1.0, fade, endpoint=False, dtype=np.float32)
|
||||
fade_out = np.linspace(1.0, 0.0, fade, endpoint=False, dtype=np.float32)
|
||||
flat_len = chunk_size - 2 * fade
|
||||
if flat_len > 0:
|
||||
return np.concatenate([fade_in, np.ones(flat_len, dtype=np.float32), fade_out])
|
||||
# num_overlap == 2: no flat region
|
||||
return np.concatenate([fade_in, fade_out])
|
||||
|
||||
|
||||
def _demix(
|
||||
model: SCNet,
|
||||
mix: np.ndarray,
|
||||
chunk_size: int,
|
||||
num_overlap: int,
|
||||
device: torch.device,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
MSST-style overlapping-window inference.
|
||||
|
||||
Args:
|
||||
mix: (channels, samples) float32
|
||||
Returns:
|
||||
(num_sources, channels, samples) float32
|
||||
"""
|
||||
C, N = mix.shape
|
||||
step = chunk_size // num_overlap
|
||||
border = chunk_size - step # == (num_overlap-1)*step
|
||||
|
||||
# Reflect-pad both sides so edge frames get proper context.
|
||||
mix_padded = np.pad(mix, ((0, 0), (border, border)), mode="reflect")
|
||||
|
||||
window = _build_window(chunk_size, step)
|
||||
sources_sum: Optional[np.ndarray] = None
|
||||
weight_sum = np.zeros(mix_padded.shape[-1], dtype=np.float32)
|
||||
|
||||
with torch.no_grad():
|
||||
for start in range(0, N, step):
|
||||
p_end = start + chunk_size
|
||||
if p_end > mix_padded.shape[-1]:
|
||||
break
|
||||
chunk = mix_padded[:, start:p_end] # (C, chunk_size)
|
||||
|
||||
# Per-chunk loudness normalisation
|
||||
mean = float(chunk.mean())
|
||||
std = max(float(chunk.std()), 1e-8)
|
||||
chunk_norm = (chunk - mean) / std
|
||||
|
||||
inp = torch.tensor(chunk_norm[np.newaxis], dtype=torch.float32, device=device)
|
||||
out = model(inp) # (1, num_sources, C, chunk_size)
|
||||
out_np = out.squeeze(0).cpu().numpy() # (num_sources, C, chunk_size)
|
||||
out_np = out_np * std + mean # denormalise
|
||||
|
||||
if sources_sum is None:
|
||||
num_sources = out_np.shape[0]
|
||||
sources_sum = np.zeros(
|
||||
(num_sources, C, mix_padded.shape[-1]), dtype=np.float32
|
||||
)
|
||||
|
||||
sources_sum[:, :, start:p_end] += out_np * window[np.newaxis, np.newaxis, :]
|
||||
weight_sum[start:p_end] += window
|
||||
|
||||
if sources_sum is None:
|
||||
raise RuntimeError("Audio is too short for chunked SCNet inference.")
|
||||
|
||||
weight_sum = np.maximum(weight_sum, 1e-8)
|
||||
sources = sources_sum / weight_sum[np.newaxis, np.newaxis, :]
|
||||
return sources[:, :, border : border + N]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Separator class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class SCNetSeparator:
|
||||
"""Native PyTorch inference wrapper for MSST SCNet models."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_id: str = "scnet_xl",
|
||||
cpu_separation: bool = CPU_SEPARATION,
|
||||
output_format: str = "mp3",
|
||||
mp3_bitrate: str = "256k",
|
||||
):
|
||||
if model_id not in SCNET_MODELS:
|
||||
raise ValueError(
|
||||
f"Unknown SCNet model: {model_id}. "
|
||||
f"Available: {list(SCNET_MODELS)}"
|
||||
)
|
||||
|
||||
self.model_id = model_id
|
||||
self.model_cfg = SCNET_MODELS[model_id]
|
||||
self.cpu_separation = cpu_separation
|
||||
self.output_format = output_format
|
||||
self.mp3_bitrate = mp3_bitrate
|
||||
self.device = torch.device(
|
||||
"cpu" if cpu_separation else ("cuda" if torch.cuda.is_available() else "cpu")
|
||||
)
|
||||
|
||||
self._model: Optional[SCNet] = None
|
||||
self._config: Optional[dict] = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Model management
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _ensure_model(self) -> tuple[Path, Path]:
|
||||
"""Download model and config from HuggingFace if not already cached."""
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
model_dir = SCNET_MODEL_DIR / self.model_id
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
model_path = model_dir / self.model_cfg["model_file"]
|
||||
config_path = model_dir / self.model_cfg["config_file"]
|
||||
|
||||
if not model_path.exists():
|
||||
print(f"[SCNet] Downloading {self.model_cfg['model_file']} …")
|
||||
cached = hf_hub_download(
|
||||
repo_id=self.model_cfg["hf_repo"],
|
||||
filename=self.model_cfg["hf_model_path"],
|
||||
)
|
||||
shutil.copy2(cached, model_path)
|
||||
|
||||
if not config_path.exists():
|
||||
print(f"[SCNet] Downloading {self.model_cfg['config_file']} …")
|
||||
cached = hf_hub_download(
|
||||
repo_id=self.model_cfg["hf_repo"],
|
||||
filename=self.model_cfg["hf_config_path"],
|
||||
)
|
||||
shutil.copy2(cached, config_path)
|
||||
|
||||
return model_path, config_path
|
||||
|
||||
def _load_model(self) -> None:
|
||||
"""Load YAML config and PyTorch checkpoint; instantiate SCNet."""
|
||||
model_path, config_path = self._ensure_model()
|
||||
|
||||
# Some config files use !!python/tuple (e.g. mixup_probs); add a
|
||||
# constructor so safe_load can handle them without crashing.
|
||||
loader = yaml.SafeLoader
|
||||
loader.add_constructor(
|
||||
"tag:yaml.org,2002:python/tuple",
|
||||
lambda ldr, node: tuple(ldr.construct_sequence(node)),
|
||||
)
|
||||
with open(config_path) as f:
|
||||
cfg = yaml.load(f, Loader=loader)
|
||||
self._config = cfg
|
||||
|
||||
model_params = cfg.get("model", {})
|
||||
model = SCNet(**model_params)
|
||||
|
||||
state = torch.load(str(model_path), map_location="cpu", weights_only=False)
|
||||
if "state_dict" in state:
|
||||
state = state["state_dict"]
|
||||
model.load_state_dict(state)
|
||||
model.eval()
|
||||
self._model = model.to(self.device)
|
||||
|
||||
def _get_model(self) -> SCNet:
|
||||
if self._model is None:
|
||||
self._load_model()
|
||||
return self._model
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
||||
"""
|
||||
Separate audio into stems.
|
||||
|
||||
Returns:
|
||||
Dict mapping stem name → output file path.
|
||||
"""
|
||||
model = self._get_model()
|
||||
cfg = self._config
|
||||
|
||||
sample_rate: int = cfg.get("audio", {}).get("sample_rate", 44100)
|
||||
inf_cfg: dict = cfg.get("inference", {})
|
||||
chunk_size: int = inf_cfg.get("chunk_size", 485100)
|
||||
num_overlap: int = inf_cfg.get("num_overlap", 2)
|
||||
|
||||
# Stem order from training config; fall back to model attribute
|
||||
instruments = cfg.get("training", {}).get("instruments") or list(model.sources)
|
||||
|
||||
mix = _load_audio(input_path, sample_rate) # (2, N)
|
||||
|
||||
print(f"[SCNet] Running {self.model_id} on {self.device} …")
|
||||
sources_np = _demix(model, mix, chunk_size, num_overlap, self.device)
|
||||
# sources_np: (num_sources, 2, N)
|
||||
|
||||
out_dir = Path(output_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_paths: Dict[str, str] = {}
|
||||
|
||||
for i, stem in enumerate(instruments):
|
||||
if i >= sources_np.shape[0]:
|
||||
break
|
||||
ext = self.output_format.lower()
|
||||
out_path = str(out_dir / f"{stem}.{ext}")
|
||||
_save_audio(out_path, sources_np[i], sample_rate, self.output_format)
|
||||
output_paths[stem] = out_path
|
||||
print(f"[SCNet] Saved {stem} → {out_path}")
|
||||
|
||||
return output_paths
|
||||
+1808
File diff suppressed because it is too large.
Load diff
+281
@@ -0,0 +1,281 @@
|
||||
"""
|
||||
Subsonic REST API client and local import service.
|
||||
|
||||
Auth: token-based (md5(password+salt)), salt generated per request.
|
||||
Config priority:
|
||||
1. MEDIA_DIR/subsonic.json (written by the Settings UI or seed_from_env())
|
||||
2. Environment variables SUBSONIC_URL / SUBSONIC_USERNAME / SUBSONIC_PASSWORD
|
||||
— on first use these are written to subsonic.json so the UI reflects them.
|
||||
|
||||
Credential storage:
|
||||
The password is never written to disk in plaintext. It is encrypted with
|
||||
Fernet (AES-128-CBC + HMAC-SHA256) using a key derived from SECRET_KEY.
|
||||
If SECRET_KEY is not set, credentials can still be supplied via environment
|
||||
variables but cannot be persisted to disk (the Settings UI will save only
|
||||
URL + username and prompt for the password on every restart).
|
||||
"""
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import requests as _requests
|
||||
from cryptography.fernet import Fernet, InvalidToken
|
||||
|
||||
from ..config import SUBSONIC_CONFIG_PATH, IMPORTS_DIR
|
||||
|
||||
_AUDIO_EXTS = {'.mp3', '.flac', '.wav', '.m4a', '.ogg', '.opus', '.aac'}
|
||||
_CLIENT_ID = 'StemEngine'
|
||||
_API_VERSION = '1.16.1'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Encryption helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_fernet: Optional[Fernet] = None
|
||||
_fernet_init = False # sentinel: False = derivation not yet attempted
|
||||
|
||||
|
||||
def _get_fernet() -> Optional[Fernet]:
|
||||
"""
|
||||
Return a Fernet instance keyed by SECRET_KEY, or None if SECRET_KEY is not set.
|
||||
Key is derived once via PBKDF2-SHA256 (200k rounds) and cached for the
|
||||
process lifetime.
|
||||
"""
|
||||
global _fernet, _fernet_init
|
||||
if _fernet_init:
|
||||
return _fernet
|
||||
_fernet_init = True
|
||||
|
||||
raw_key = os.environ.get('SECRET_KEY', '').strip()
|
||||
if not raw_key:
|
||||
_fernet = None
|
||||
return None
|
||||
|
||||
# Derive a 32-byte key from whatever the user provides
|
||||
derived = hashlib.pbkdf2_hmac(
|
||||
'sha256', raw_key.encode(), b'stemengine-subsonic', 200_000, dklen=32
|
||||
)
|
||||
_fernet = Fernet(base64.urlsafe_b64encode(derived))
|
||||
return _fernet
|
||||
|
||||
|
||||
def _encrypt(plaintext: str) -> str:
|
||||
f = _get_fernet()
|
||||
if f is None:
|
||||
raise RuntimeError(
|
||||
"SECRET_KEY is not set — cannot persist credentials to disk. "
|
||||
"Add SECRET_KEY to docker-compose.yml to enable Settings UI credential storage."
|
||||
)
|
||||
return f.encrypt(plaintext.encode()).decode()
|
||||
|
||||
|
||||
def _decrypt(ciphertext: str) -> str:
|
||||
f = _get_fernet()
|
||||
if f is None:
|
||||
raise RuntimeError(
|
||||
"SECRET_KEY is not set — cannot decrypt stored credentials. "
|
||||
"Set SECRET_KEY or re-enter credentials via the Settings UI."
|
||||
)
|
||||
try:
|
||||
return f.decrypt(ciphertext.encode()).decode()
|
||||
except InvalidToken as exc:
|
||||
raise RuntimeError(
|
||||
"Credential decryption failed — SECRET_KEY may have changed. "
|
||||
"Re-enter credentials via the Settings UI."
|
||||
) from exc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _config_from_env() -> Optional[dict]:
|
||||
"""Return config from environment variables, or None if not fully set."""
|
||||
url = os.environ.get('SUBSONIC_URL', '').strip()
|
||||
username = os.environ.get('SUBSONIC_USERNAME', '').strip()
|
||||
password = os.environ.get('SUBSONIC_PASSWORD', '').strip()
|
||||
if url and username and password:
|
||||
return {'url': url, 'username': username, 'password': password}
|
||||
return None
|
||||
|
||||
|
||||
def load_config() -> Optional[dict]:
|
||||
"""
|
||||
Return the active Subsonic config dict (plaintext password ready for use),
|
||||
or None if no credentials are available.
|
||||
|
||||
Handles encrypted-at-rest configs (_enc=True) and the legacy plaintext format.
|
||||
"""
|
||||
if SUBSONIC_CONFIG_PATH.exists():
|
||||
try:
|
||||
stored = json.loads(SUBSONIC_CONFIG_PATH.read_text())
|
||||
if stored.get('_enc'):
|
||||
pwd = _decrypt(stored['password'])
|
||||
return {'url': stored['url'], 'username': stored['username'], 'password': pwd}
|
||||
else:
|
||||
# Legacy plaintext — serve as-is; the UI can re-save to upgrade
|
||||
return {k: v for k, v in stored.items() if not k.startswith('_')}
|
||||
except RuntimeError:
|
||||
raise # Surface decryption / key errors to the caller
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
env_cfg = _config_from_env()
|
||||
if env_cfg:
|
||||
try:
|
||||
save_config(env_cfg) # Persist encrypted if SECRET_KEY is available
|
||||
except RuntimeError:
|
||||
pass # No key — serve from env, don't write to disk
|
||||
return env_cfg
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def save_config(config: dict) -> None:
|
||||
"""
|
||||
Persist Subsonic config with the password encrypted at rest.
|
||||
Raises RuntimeError if SECRET_KEY is not set.
|
||||
"""
|
||||
stored = {
|
||||
'url': config['url'],
|
||||
'username': config['username'],
|
||||
'password': _encrypt(config['password']),
|
||||
'_enc': True,
|
||||
}
|
||||
SUBSONIC_CONFIG_PATH.write_text(json.dumps(stored, indent=2))
|
||||
|
||||
|
||||
def delete_config() -> None:
|
||||
if SUBSONIC_CONFIG_PATH.exists():
|
||||
SUBSONIC_CONFIG_PATH.unlink()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth / URL helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_token(password: str) -> tuple[str, str]:
|
||||
"""Return (token, salt) for Subsonic token auth."""
|
||||
salt = secrets.token_hex(8)
|
||||
token = hashlib.md5((password + salt).encode()).hexdigest()
|
||||
return token, salt
|
||||
|
||||
|
||||
def _params(config: dict) -> dict:
|
||||
token, salt = _make_token(config['password'])
|
||||
return {
|
||||
'u': config['username'],
|
||||
't': token,
|
||||
's': salt,
|
||||
'v': _API_VERSION,
|
||||
'c': _CLIENT_ID,
|
||||
'f': 'json',
|
||||
}
|
||||
|
||||
|
||||
def _url(config: dict, endpoint: str) -> str:
|
||||
return f"{config['url'].rstrip('/')}/rest/{endpoint}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API methods
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_connection(config: dict) -> bool:
|
||||
"""Ping the Subsonic server; return True if auth succeeds."""
|
||||
try:
|
||||
r = _requests.get(_url(config, 'ping.view'), params=_params(config), timeout=10)
|
||||
data = r.json()
|
||||
return data.get('subsonic-response', {}).get('status') == 'ok'
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def search_songs(query: str, count: int = 20) -> list[dict]:
|
||||
"""Search for songs by keyword. Returns a list of song dicts."""
|
||||
config = load_config()
|
||||
if not config:
|
||||
raise RuntimeError("Subsonic server not configured")
|
||||
|
||||
p = _params(config)
|
||||
p.update({'query': query, 'songCount': count, 'artistCount': 0, 'albumCount': 0})
|
||||
|
||||
r = _requests.get(_url(config, 'search3.view'), params=p, timeout=15)
|
||||
r.raise_for_status()
|
||||
resp = r.json().get('subsonic-response', {})
|
||||
|
||||
if resp.get('status') != 'ok':
|
||||
msg = resp.get('error', {}).get('message', 'Subsonic API error')
|
||||
raise RuntimeError(msg)
|
||||
|
||||
return [
|
||||
{
|
||||
'id': s.get('id', ''),
|
||||
'title': s.get('title', 'Unknown'),
|
||||
'artist': s.get('artist', ''),
|
||||
'album': s.get('album', ''),
|
||||
'duration': s.get('duration', 0),
|
||||
'suffix': s.get('suffix', 'mp3'),
|
||||
'size': s.get('size', 0),
|
||||
'bitRate': s.get('bitRate', 0),
|
||||
}
|
||||
for s in resp.get('searchResult3', {}).get('song', [])
|
||||
]
|
||||
|
||||
|
||||
def import_song(song_id: str, title: str, artist: str = '', suffix: str = 'mp3') -> dict:
|
||||
"""Download a Subsonic song to IMPORTS_DIR."""
|
||||
config = load_config()
|
||||
if not config:
|
||||
raise RuntimeError("Subsonic server not configured")
|
||||
|
||||
def _safe(s: str) -> str:
|
||||
return "".join(c for c in s if c.isalnum() or c in ' -_().').strip()
|
||||
|
||||
safe_title = _safe(title) or 'Unknown'
|
||||
safe_artist = _safe(artist)
|
||||
base_name = f"{safe_artist} - {safe_title}" if safe_artist else safe_title
|
||||
dest = IMPORTS_DIR / f"{base_name}.{suffix}"
|
||||
|
||||
counter = 1
|
||||
while dest.exists():
|
||||
dest = IMPORTS_DIR / f"{base_name} ({counter}).{suffix}"
|
||||
counter += 1
|
||||
|
||||
p = _params(config)
|
||||
p['id'] = song_id
|
||||
|
||||
r = _requests.get(_url(config, 'stream.view'), params=p, stream=True, timeout=120)
|
||||
r.raise_for_status()
|
||||
|
||||
with open(dest, 'wb') as f:
|
||||
for chunk in r.iter_content(chunk_size=65536):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
|
||||
return {
|
||||
'filename': dest.name,
|
||||
'path': str(dest),
|
||||
'size': dest.stat().st_size,
|
||||
'title': title,
|
||||
'artist': artist,
|
||||
}
|
||||
|
||||
|
||||
def list_imports() -> list[dict]:
|
||||
"""Return metadata for all audio files in IMPORTS_DIR, newest first."""
|
||||
if not IMPORTS_DIR.exists():
|
||||
return []
|
||||
results = []
|
||||
for f in sorted(IMPORTS_DIR.iterdir(), key=lambda x: x.stat().st_mtime, reverse=True):
|
||||
if f.is_file() and f.suffix.lower() in _AUDIO_EXTS:
|
||||
results.append({
|
||||
'filename': f.name,
|
||||
'path': str(f),
|
||||
'size': f.stat().st_size,
|
||||
})
|
||||
return results
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
MIDI transcription via basic-pitch (in-process Python API).
|
||||
Supports two modes mapped to inference parameter presets:
|
||||
polyphonic — general multi-instrument transcription
|
||||
melody — single-voice / vocal melody (melodia trick, min frequency)
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import JOBS_DIR, TRANSCRIBE_PRESETS
|
||||
|
||||
|
||||
def run_transcription(job_id: str, input_path: str, mode: str):
|
||||
"""
|
||||
Background task: transcribe audio to MIDI using basic-pitch.
|
||||
mode: 'polyphonic' | 'melody'
|
||||
"""
|
||||
from basic_pitch.inference import predict
|
||||
from basic_pitch import ICASSP_2022_MODEL_PATH
|
||||
|
||||
output_dir = JOBS_DIR / job_id
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Loading basic-pitch model...")
|
||||
|
||||
try:
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
params = TRANSCRIBE_PRESETS.get(mode, TRANSCRIBE_PRESETS["polyphonic"])
|
||||
jm.update_job(job_id, progress=f"Transcribing with basic-pitch ({mode})...")
|
||||
|
||||
_, midi_data, _ = predict(
|
||||
input_path,
|
||||
ICASSP_2022_MODEL_PATH,
|
||||
**params,
|
||||
)
|
||||
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
output_name = f"{Path(input_path).stem}-{mode}.mid"
|
||||
dest = output_dir / output_name
|
||||
midi_data.write(str(dest))
|
||||
|
||||
jm.update_job(
|
||||
job_id,
|
||||
status=jm.JobStatus.DONE,
|
||||
progress="Transcription complete.",
|
||||
files=[{"name": dest.name, "path": str(dest)}],
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512">
|
||||
<defs>
|
||||
<linearGradient id="g" x1="0%" y1="0%" x2="100%" y2="100%">
|
||||
<stop offset="0%" stop-color="#3B82F6"/>
|
||||
<stop offset="100%" stop-color="#1D4ED8"/>
|
||||
</linearGradient>
|
||||
</defs>
|
||||
<!-- Full-bleed background (maskable safe zone = center 80%) -->
|
||||
<rect width="512" height="512" fill="url(#g)"/>
|
||||
<!-- Music2 icon from Lucide, original 24×24, scaled ×15, centered in 512×512 -->
|
||||
<!-- Content bounding box: ~3..21 in x, ~3..21 in y → 18×18 units → 270×270 px -->
|
||||
<!-- Translate so content center (12,12 orig) maps to (256,256) -->
|
||||
<!-- translateX = 256 - 12×15 = 76 translateY = 256 - 12×15 = 76 -->
|
||||
<g transform="translate(76,76) scale(15)"
|
||||
stroke="white" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" fill="none">
|
||||
<path d="M9 18V5l12-2v13"/>
|
||||
<circle cx="6" cy="18" r="3"/>
|
||||
<circle cx="18" cy="16" r="3"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 961 B |
@@ -0,0 +1,28 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>StemEngine</title>
|
||||
|
||||
<!-- PWA manifest -->
|
||||
<link rel="manifest" href="/manifest.webmanifest" />
|
||||
|
||||
<!-- Favicon -->
|
||||
<link rel="icon" href="/icon.svg" type="image/svg+xml" />
|
||||
|
||||
<!-- Theme color (browser chrome on Android, tab bar tint) -->
|
||||
<meta name="theme-color" content="#3B82F6" />
|
||||
|
||||
<!-- iOS PWA -->
|
||||
<meta name="apple-mobile-web-app-capable" content="yes" />
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
|
||||
<meta name="apple-mobile-web-app-title" content="StemEngine" />
|
||||
<link rel="apple-touch-icon" href="/icon.svg" />
|
||||
<script type="module" crossorigin src="/assets/index-DIxWD2yx.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-DGXAjPZc.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
</body>
|
||||
</html>
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"name": "StemEngine",
|
||||
"short_name": "StemEngine",
|
||||
"description": "Audio processing toolkit — stem separation, lyrics sync, MIDI transcription",
|
||||
"start_url": "/",
|
||||
"display": "standalone",
|
||||
"orientation": "any",
|
||||
"background_color": "#030712",
|
||||
"theme_color": "#3B82F6",
|
||||
"icons": [
|
||||
{
|
||||
"src": "/icon.svg",
|
||||
"sizes": "any",
|
||||
"type": "image/svg+xml",
|
||||
"purpose": "any maskable"
|
||||
}
|
||||
]
|
||||
}
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>StemEngine</title>
|
||||
|
||||
<!-- PWA manifest -->
|
||||
<link rel="manifest" href="/manifest.webmanifest" />
|
||||
|
||||
<!-- Favicon -->
|
||||
<link rel="icon" href="/icon.svg" type="image/svg+xml" />
|
||||
|
||||
<!-- Theme color (browser chrome on Android, tab bar tint) -->
|
||||
<meta name="theme-color" content="#3B82F6" />
|
||||
|
||||
<!-- iOS PWA -->
|
||||
<meta name="apple-mobile-web-app-capable" content="yes" />
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
|
||||
<meta name="apple-mobile-web-app-title" content="StemEngine" />
|
||||
<link rel="apple-touch-icon" href="/icon.svg" />
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
+2540
File diff suppressed because it is too large.
Load diff
Executable
+28
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "stemengine-frontend",
|
||||
"private": true,
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"@tanstack/react-query": "^5.0.0",
|
||||
"@tonejs/midi": "^2.0.28",
|
||||
"lucide-react": "^0.294.0",
|
||||
"react": "^18.2.0",
|
||||
"react-dom": "^18.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^18.2.37",
|
||||
"@types/react-dom": "^18.2.15",
|
||||
"@vitejs/plugin-react": "^4.2.0",
|
||||
"autoprefixer": "^10.4.16",
|
||||
"postcss": "^8.4.32",
|
||||
"tailwindcss": "^3.3.6",
|
||||
"typescript": "^5.9.3",
|
||||
"vite": "^5.0.0"
|
||||
}
|
||||
}
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
export default {
|
||||
plugins: {
|
||||
tailwindcss: {},
|
||||
autoprefixer: {},
|
||||
},
|
||||
}
|
||||
Loaded 100 of 146 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user