329 lines
11 KiB
Python
329 lines
11 KiB
Python
"""
|
|
SepACap separator — a cappella multi-singer vocal stem separation.
|
|
|
|
Architecture from ETH-DISCO/SepACap (MIT licence), cloned on first use.
|
|
Weights hosted at Tino3141/sepacap on HuggingFace (161 MB).
|
|
|
|
Separates a cappella recordings into 7 independent vocal stems:
|
|
alto, bass, finger_snap, lead_vocal, soprano, tenor, vocal_percussion
|
|
|
|
Sample rate: 24 kHz — input audio is automatically resampled.
|
|
"""
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import logging
|
|
from pathlib import Path
|
|
from typing import Dict, Optional
|
|
|
|
import torch
|
|
import torchaudio
|
|
import yaml
|
|
|
|
from ..config import SEPACAP_MODEL_DIR, CPU_SEPARATION
|
|
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
# Stems in the exact order the model outputs them (matches num_spks=7 in config)
|
|
SEPACAP_STEMS = [
|
|
"alto",
|
|
"bass",
|
|
"finger_snap",
|
|
"lead_vocal",
|
|
"soprano",
|
|
"tenor",
|
|
"vocal_percussion",
|
|
]
|
|
|
|
SAMPLE_RATE = 24_000
|
|
HF_REPO = "Tino3141/sepacap"
|
|
CHECKPOINT = "SepACap.pth"
|
|
CONFIG_FILE = "modelMusicSep.yaml"
|
|
GITHUB_REPO = "https://github.com/ETH-DISCO/SepACap.git"
|
|
_REPO_DIR = Path("/opt/sepacap_repo")
|
|
_DEPS = ["einops", "julius", "loguru"]
|
|
|
|
# Chunked inference settings — prevents VRAM OOM on long audio.
|
|
# 2 s chunks at 24 kHz with 0.25 s crossfade on each side.
|
|
# After the encoder (stride=8), 2 s = 6 000 time steps → ~288 MB attention
|
|
# per layer, safe for a 6 GB consumer GPU.
|
|
_CHUNK_SAMPLES = 48_000 # 2 seconds
|
|
_OVERLAP_SAMPLES = 6_000 # 0.25 second crossfade
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# One-time setup helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _ensure_deps() -> None:
|
|
"""Install lightweight runtime deps if not already available."""
|
|
missing = []
|
|
for pkg in _DEPS:
|
|
try:
|
|
__import__(pkg)
|
|
except ImportError:
|
|
missing.append(pkg)
|
|
if missing:
|
|
log.info("[SepACap] Installing missing packages: %s", missing)
|
|
subprocess.run(
|
|
[sys.executable, "-m", "pip", "install", *missing],
|
|
check=True,
|
|
)
|
|
|
|
|
|
def _ensure_repo() -> None:
|
|
"""Shallow-clone the SepACap GitHub repo for its model architecture code."""
|
|
if _REPO_DIR.exists():
|
|
return
|
|
_REPO_DIR.parent.mkdir(parents=True, exist_ok=True)
|
|
log.info("[SepACap] Cloning architecture code from %s …", GITHUB_REPO)
|
|
subprocess.run(
|
|
["git", "clone", "--depth=1", GITHUB_REPO, str(_REPO_DIR)],
|
|
check=True,
|
|
)
|
|
|
|
|
|
def _ensure_sys_path() -> None:
|
|
_ensure_deps()
|
|
_ensure_repo()
|
|
repo_str = str(_REPO_DIR)
|
|
if repo_str not in sys.path:
|
|
sys.path.insert(0, repo_str)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Audio helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _load_audio(path: str) -> torch.Tensor:
|
|
"""Load audio -> mono, resample to 24 kHz. Returns (1, N) float32."""
|
|
waveform, sr = torchaudio.load(path)
|
|
if sr != SAMPLE_RATE:
|
|
waveform = torchaudio.functional.resample(waveform, sr, SAMPLE_RATE)
|
|
if waveform.shape[0] > 1:
|
|
waveform = waveform.mean(dim=0, keepdim=True)
|
|
return waveform
|
|
|
|
|
|
def _save_audio(path: str, audio: torch.Tensor, fmt: str) -> None:
|
|
"""Save a (1, N) CPU tensor."""
|
|
audio = audio.cpu()
|
|
f = fmt.lower()
|
|
if f.startswith("mp3"):
|
|
torchaudio.save(path, audio, SAMPLE_RATE, format="mp3")
|
|
elif f == "flac":
|
|
torchaudio.save(path, audio, SAMPLE_RATE, format="flac")
|
|
else:
|
|
torchaudio.save(path, audio, SAMPLE_RATE)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Chunked inference
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _separate_chunked(
|
|
model,
|
|
audio: torch.Tensor,
|
|
device: torch.device,
|
|
num_stems: int,
|
|
chunk_samples: int = 48_000,
|
|
overlap_samples: int = 6_000,
|
|
) -> list:
|
|
"""
|
|
Overlap-add chunked inference.
|
|
|
|
Splits audio into chunk_samples-length pieces with overlap_samples
|
|
crossfade on each side, runs model on each piece, and reassembles with
|
|
a Hann window to avoid discontinuities. Frees CUDA cache between chunks.
|
|
"""
|
|
total_len = audio.shape[-1]
|
|
hop = chunk_samples - overlap_samples * 2
|
|
if hop <= 0:
|
|
hop = chunk_samples // 2
|
|
|
|
pad = overlap_samples
|
|
padded_len = total_len + pad * 2
|
|
audio_padded = torch.nn.functional.pad(audio, (pad, max(0, padded_len - audio.shape[-1])))
|
|
|
|
window = torch.hann_window(chunk_samples, periodic=False)
|
|
|
|
# Accumulators on CPU to avoid holding large VRAM buffers
|
|
out_buf = [torch.zeros(1, audio_padded.shape[-1]) for _ in range(num_stems)]
|
|
wgt_buf = torch.zeros(audio_padded.shape[-1])
|
|
|
|
pos = 0
|
|
chunk_idx = 0
|
|
while pos < total_len + pad:
|
|
end = pos + chunk_samples
|
|
chunk = audio_padded[:, pos:end]
|
|
if chunk.shape[-1] < chunk_samples:
|
|
chunk = torch.nn.functional.pad(chunk, (0, chunk_samples - chunk.shape[-1]))
|
|
|
|
chunk_dev = chunk.to(device)
|
|
with torch.no_grad():
|
|
separated, _ = model(chunk_dev)
|
|
del chunk_dev
|
|
|
|
if device.type == "cuda":
|
|
torch.cuda.empty_cache()
|
|
|
|
w = window
|
|
for i, stem_tensor in enumerate(separated):
|
|
stem_cpu = stem_tensor.detach().cpu()
|
|
if stem_cpu.shape[-1] < chunk_samples:
|
|
stem_cpu = torch.nn.functional.pad(
|
|
stem_cpu, (0, chunk_samples - stem_cpu.shape[-1])
|
|
)
|
|
win_stem = stem_cpu * w
|
|
actual = min(chunk_samples, out_buf[i].shape[-1] - pos)
|
|
if actual > 0:
|
|
out_buf[i][:, pos:pos + actual] += win_stem[:, :actual]
|
|
|
|
wgt_actual = min(chunk_samples, wgt_buf.shape[-1] - pos)
|
|
if wgt_actual > 0:
|
|
wgt_buf[pos:pos + wgt_actual] += w[:wgt_actual]
|
|
|
|
chunk_idx += 1
|
|
log.info(
|
|
"[SepACap] Chunk %d done (%.1fs / %.1fs)",
|
|
chunk_idx,
|
|
pos / SAMPLE_RATE,
|
|
total_len / SAMPLE_RATE,
|
|
)
|
|
pos += hop
|
|
|
|
# Normalise, strip padding, return list of (1, total_len) tensors
|
|
eps = 1e-8
|
|
result = []
|
|
for buf in out_buf:
|
|
normalised = buf / (wgt_buf + eps)
|
|
result.append(normalised[:, pad:pad + total_len])
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Separator
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class SepACapSeparator:
|
|
"""
|
|
Native PyTorch inference wrapper for SepACap a cappella vocal separation.
|
|
|
|
Separates mixed a cappella recordings into 7 independent vocal stems:
|
|
alto, bass, finger snap, lead vocal, soprano, tenor, and vocal percussion.
|
|
|
|
Architecture: adapted SepReformer with periodic activations and composite
|
|
loss -- state-of-the-art on the JaCappella benchmark (NeurIPS 2025).
|
|
Architecture code is cloned from GitHub on first use (~3 MB shallow clone).
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
cpu_separation: bool = CPU_SEPARATION,
|
|
output_format: str = "mp3",
|
|
mp3_bitrate: str = "256k",
|
|
chunk_samples: int = 48_000,
|
|
):
|
|
self.cpu_separation = cpu_separation
|
|
self.output_format = output_format
|
|
self.mp3_bitrate = mp3_bitrate
|
|
self.chunk_samples = chunk_samples
|
|
self.overlap_samples = max(1, chunk_samples // 8)
|
|
self.device = torch.device(
|
|
"cpu" if cpu_separation else ("cuda" if torch.cuda.is_available() else "cpu")
|
|
)
|
|
self._model = None
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
def _ensure_files(self) -> tuple[Path, Path]:
|
|
"""Download checkpoint and config from HuggingFace if missing."""
|
|
from huggingface_hub import hf_hub_download
|
|
|
|
model_dir = SEPACAP_MODEL_DIR
|
|
model_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
ckpt = model_dir / CHECKPOINT
|
|
cfg = model_dir / CONFIG_FILE
|
|
|
|
for fname, dest in [(CHECKPOINT, ckpt), (CONFIG_FILE, cfg)]:
|
|
if not dest.exists():
|
|
log.info("[SepACap] Downloading %s from %s ...", fname, HF_REPO)
|
|
cached = hf_hub_download(repo_id=HF_REPO, filename=fname)
|
|
shutil.copy2(cached, dest)
|
|
|
|
return ckpt, cfg
|
|
|
|
def _load_model(self) -> None:
|
|
_ensure_sys_path()
|
|
ckpt, cfg_path = self._ensure_files()
|
|
|
|
with open(cfg_path) as f:
|
|
raw = yaml.full_load(f)
|
|
model_cfg = raw["config"]["model"]
|
|
|
|
from src.model import Model # available after sys.path insert
|
|
|
|
model = Model(**model_cfg)
|
|
state = torch.load(str(ckpt), map_location="cpu", weights_only=False)
|
|
if isinstance(state, dict) and "model_state" in state:
|
|
state = state["model_state"]
|
|
elif isinstance(state, dict) and "state_dict" in state:
|
|
state = state["state_dict"]
|
|
model.load_state_dict(state, strict=False)
|
|
model.eval()
|
|
self._model = model.to(self.device)
|
|
|
|
def _get_model(self):
|
|
if self._model is None:
|
|
self._load_model()
|
|
return self._model
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
|
"""
|
|
Separate an a cappella recording into 7 vocal stems.
|
|
|
|
Returns:
|
|
Dict mapping stem name -> absolute output file path.
|
|
"""
|
|
model = self._get_model()
|
|
audio = _load_audio(input_path) # (1, N) on CPU
|
|
orig_len = audio.shape[-1]
|
|
duration = orig_len / SAMPLE_RATE
|
|
|
|
log.info(
|
|
"[SepACap] Running chunked inference on %s (audio=%.1fs, chunk=%.1fs) ...",
|
|
self.device,
|
|
duration,
|
|
self.chunk_samples / SAMPLE_RATE,
|
|
)
|
|
|
|
separated = _separate_chunked(
|
|
model, audio, self.device, len(SEPACAP_STEMS),
|
|
chunk_samples=self.chunk_samples,
|
|
overlap_samples=self.overlap_samples,
|
|
)
|
|
|
|
out_dir = Path(output_dir)
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
ext = self.output_format.lower()
|
|
if ext.startswith("mp3"):
|
|
ext = "mp3"
|
|
|
|
output_paths: Dict[str, str] = {}
|
|
for i, stem_name in enumerate(SEPACAP_STEMS):
|
|
if i >= len(separated):
|
|
break
|
|
stem_audio = separated[i]
|
|
out_path = str(out_dir / f"{stem_name}.{ext}")
|
|
_save_audio(out_path, stem_audio, ext)
|
|
output_paths[stem_name] = out_path
|
|
log.info("[SepACap] Saved %s -> %s", stem_name, out_path)
|
|
|
|
return output_paths
|