Files
2026-04-14 16:32:36 +09:00

329 lines
11 KiB
Python

"""
SepACap separator — a cappella multi-singer vocal stem separation.
Architecture from ETH-DISCO/SepACap (MIT licence), cloned on first use.
Weights hosted at Tino3141/sepacap on HuggingFace (161 MB).
Separates a cappella recordings into 7 independent vocal stems:
alto, bass, finger_snap, lead_vocal, soprano, tenor, vocal_percussion
Sample rate: 24 kHz — input audio is automatically resampled.
"""
import shutil
import subprocess
import sys
import logging
from pathlib import Path
from typing import Dict, Optional
import torch
import torchaudio
import yaml
from ..config import SEPACAP_MODEL_DIR, CPU_SEPARATION
log = logging.getLogger(__name__)
# Stems in the exact order the model outputs them (matches num_spks=7 in config)
SEPACAP_STEMS = [
"alto",
"bass",
"finger_snap",
"lead_vocal",
"soprano",
"tenor",
"vocal_percussion",
]
SAMPLE_RATE = 24_000
HF_REPO = "Tino3141/sepacap"
CHECKPOINT = "SepACap.pth"
CONFIG_FILE = "modelMusicSep.yaml"
GITHUB_REPO = "https://github.com/ETH-DISCO/SepACap.git"
_REPO_DIR = Path("/opt/sepacap_repo")
_DEPS = ["einops", "julius", "loguru"]
# Chunked inference settings — prevents VRAM OOM on long audio.
# 2 s chunks at 24 kHz with 0.25 s crossfade on each side.
# After the encoder (stride=8), 2 s = 6 000 time steps → ~288 MB attention
# per layer, safe for a 6 GB consumer GPU.
_CHUNK_SAMPLES = 48_000 # 2 seconds
_OVERLAP_SAMPLES = 6_000 # 0.25 second crossfade
# ---------------------------------------------------------------------------
# One-time setup helpers
# ---------------------------------------------------------------------------
def _ensure_deps() -> None:
"""Install lightweight runtime deps if not already available."""
missing = []
for pkg in _DEPS:
try:
__import__(pkg)
except ImportError:
missing.append(pkg)
if missing:
log.info("[SepACap] Installing missing packages: %s", missing)
subprocess.run(
[sys.executable, "-m", "pip", "install", *missing],
check=True,
)
def _ensure_repo() -> None:
"""Shallow-clone the SepACap GitHub repo for its model architecture code."""
if _REPO_DIR.exists():
return
_REPO_DIR.parent.mkdir(parents=True, exist_ok=True)
log.info("[SepACap] Cloning architecture code from %s …", GITHUB_REPO)
subprocess.run(
["git", "clone", "--depth=1", GITHUB_REPO, str(_REPO_DIR)],
check=True,
)
def _ensure_sys_path() -> None:
_ensure_deps()
_ensure_repo()
repo_str = str(_REPO_DIR)
if repo_str not in sys.path:
sys.path.insert(0, repo_str)
# ---------------------------------------------------------------------------
# Audio helpers
# ---------------------------------------------------------------------------
def _load_audio(path: str) -> torch.Tensor:
"""Load audio -> mono, resample to 24 kHz. Returns (1, N) float32."""
waveform, sr = torchaudio.load(path)
if sr != SAMPLE_RATE:
waveform = torchaudio.functional.resample(waveform, sr, SAMPLE_RATE)
if waveform.shape[0] > 1:
waveform = waveform.mean(dim=0, keepdim=True)
return waveform
def _save_audio(path: str, audio: torch.Tensor, fmt: str) -> None:
"""Save a (1, N) CPU tensor."""
audio = audio.cpu()
f = fmt.lower()
if f.startswith("mp3"):
torchaudio.save(path, audio, SAMPLE_RATE, format="mp3")
elif f == "flac":
torchaudio.save(path, audio, SAMPLE_RATE, format="flac")
else:
torchaudio.save(path, audio, SAMPLE_RATE)
# ---------------------------------------------------------------------------
# Chunked inference
# ---------------------------------------------------------------------------
def _separate_chunked(
model,
audio: torch.Tensor,
device: torch.device,
num_stems: int,
chunk_samples: int = 48_000,
overlap_samples: int = 6_000,
) -> list:
"""
Overlap-add chunked inference.
Splits audio into chunk_samples-length pieces with overlap_samples
crossfade on each side, runs model on each piece, and reassembles with
a Hann window to avoid discontinuities. Frees CUDA cache between chunks.
"""
total_len = audio.shape[-1]
hop = chunk_samples - overlap_samples * 2
if hop <= 0:
hop = chunk_samples // 2
pad = overlap_samples
padded_len = total_len + pad * 2
audio_padded = torch.nn.functional.pad(audio, (pad, max(0, padded_len - audio.shape[-1])))
window = torch.hann_window(chunk_samples, periodic=False)
# Accumulators on CPU to avoid holding large VRAM buffers
out_buf = [torch.zeros(1, audio_padded.shape[-1]) for _ in range(num_stems)]
wgt_buf = torch.zeros(audio_padded.shape[-1])
pos = 0
chunk_idx = 0
while pos < total_len + pad:
end = pos + chunk_samples
chunk = audio_padded[:, pos:end]
if chunk.shape[-1] < chunk_samples:
chunk = torch.nn.functional.pad(chunk, (0, chunk_samples - chunk.shape[-1]))
chunk_dev = chunk.to(device)
with torch.no_grad():
separated, _ = model(chunk_dev)
del chunk_dev
if device.type == "cuda":
torch.cuda.empty_cache()
w = window
for i, stem_tensor in enumerate(separated):
stem_cpu = stem_tensor.detach().cpu()
if stem_cpu.shape[-1] < chunk_samples:
stem_cpu = torch.nn.functional.pad(
stem_cpu, (0, chunk_samples - stem_cpu.shape[-1])
)
win_stem = stem_cpu * w
actual = min(chunk_samples, out_buf[i].shape[-1] - pos)
if actual > 0:
out_buf[i][:, pos:pos + actual] += win_stem[:, :actual]
wgt_actual = min(chunk_samples, wgt_buf.shape[-1] - pos)
if wgt_actual > 0:
wgt_buf[pos:pos + wgt_actual] += w[:wgt_actual]
chunk_idx += 1
log.info(
"[SepACap] Chunk %d done (%.1fs / %.1fs)",
chunk_idx,
pos / SAMPLE_RATE,
total_len / SAMPLE_RATE,
)
pos += hop
# Normalise, strip padding, return list of (1, total_len) tensors
eps = 1e-8
result = []
for buf in out_buf:
normalised = buf / (wgt_buf + eps)
result.append(normalised[:, pad:pad + total_len])
return result
# ---------------------------------------------------------------------------
# Separator
# ---------------------------------------------------------------------------
class SepACapSeparator:
"""
Native PyTorch inference wrapper for SepACap a cappella vocal separation.
Separates mixed a cappella recordings into 7 independent vocal stems:
alto, bass, finger snap, lead vocal, soprano, tenor, and vocal percussion.
Architecture: adapted SepReformer with periodic activations and composite
loss -- state-of-the-art on the JaCappella benchmark (NeurIPS 2025).
Architecture code is cloned from GitHub on first use (~3 MB shallow clone).
"""
def __init__(
self,
cpu_separation: bool = CPU_SEPARATION,
output_format: str = "mp3",
mp3_bitrate: str = "256k",
chunk_samples: int = 48_000,
):
self.cpu_separation = cpu_separation
self.output_format = output_format
self.mp3_bitrate = mp3_bitrate
self.chunk_samples = chunk_samples
self.overlap_samples = max(1, chunk_samples // 8)
self.device = torch.device(
"cpu" if cpu_separation else ("cuda" if torch.cuda.is_available() else "cpu")
)
self._model = None
# ------------------------------------------------------------------
def _ensure_files(self) -> tuple[Path, Path]:
"""Download checkpoint and config from HuggingFace if missing."""
from huggingface_hub import hf_hub_download
model_dir = SEPACAP_MODEL_DIR
model_dir.mkdir(parents=True, exist_ok=True)
ckpt = model_dir / CHECKPOINT
cfg = model_dir / CONFIG_FILE
for fname, dest in [(CHECKPOINT, ckpt), (CONFIG_FILE, cfg)]:
if not dest.exists():
log.info("[SepACap] Downloading %s from %s ...", fname, HF_REPO)
cached = hf_hub_download(repo_id=HF_REPO, filename=fname)
shutil.copy2(cached, dest)
return ckpt, cfg
def _load_model(self) -> None:
_ensure_sys_path()
ckpt, cfg_path = self._ensure_files()
with open(cfg_path) as f:
raw = yaml.full_load(f)
model_cfg = raw["config"]["model"]
from src.model import Model # available after sys.path insert
model = Model(**model_cfg)
state = torch.load(str(ckpt), map_location="cpu", weights_only=False)
if isinstance(state, dict) and "model_state" in state:
state = state["model_state"]
elif isinstance(state, dict) and "state_dict" in state:
state = state["state_dict"]
model.load_state_dict(state, strict=False)
model.eval()
self._model = model.to(self.device)
def _get_model(self):
if self._model is None:
self._load_model()
return self._model
# ------------------------------------------------------------------
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
"""
Separate an a cappella recording into 7 vocal stems.
Returns:
Dict mapping stem name -> absolute output file path.
"""
model = self._get_model()
audio = _load_audio(input_path) # (1, N) on CPU
orig_len = audio.shape[-1]
duration = orig_len / SAMPLE_RATE
log.info(
"[SepACap] Running chunked inference on %s (audio=%.1fs, chunk=%.1fs) ...",
self.device,
duration,
self.chunk_samples / SAMPLE_RATE,
)
separated = _separate_chunked(
model, audio, self.device, len(SEPACAP_STEMS),
chunk_samples=self.chunk_samples,
overlap_samples=self.overlap_samples,
)
out_dir = Path(output_dir)
out_dir.mkdir(parents=True, exist_ok=True)
ext = self.output_format.lower()
if ext.startswith("mp3"):
ext = "mp3"
output_paths: Dict[str, str] = {}
for i, stem_name in enumerate(SEPACAP_STEMS):
if i >= len(separated):
break
stem_audio = separated[i]
out_path = str(out_dir / f"{stem_name}.{ext}")
_save_audio(out_path, stem_audio, ext)
output_paths[stem_name] = out_path
log.info("[SepACap] Saved %s -> %s", stem_name, out_path)
return output_paths