Files
stemengine_old/backend/services/sepreformer_separator.py
2026-04-14 16:32:36 +09:00

335 lines
12 KiB
Python

"""
SepReformer separator — 2-speaker speech separation.
Architecture and weights from dmlguq456/SepReformer (NeurIPS 2024).
Weights are bundled in the repository via Git LFS.
Requires git-lfs to be installed in the container:
apt-get install git-lfs
Add to Dockerfile: RUN apt-get install -y git-lfs
Separates 2-speaker mixed audio into:
speaker_1, speaker_2
Sample rate: 8 kHz — input audio is automatically resampled.
Available model variants:
sepreformer_base — WSJ0-2Mix (clean speech)
sepreformer_large_wsj0 — WSJ0-2Mix large (clean speech)
sepreformer_large_wham — WHAM! (noisy speech)
sepreformer_large_whamr — WHAMR! (noisy + reverberant speech)
"""
import importlib.util
import shutil
import subprocess
import sys
import logging
from pathlib import Path
from typing import Dict, Optional
import torch
import torchaudio
import yaml
from ..config import SEPREFORMER_MODEL_DIR, CPU_SEPARATION
log = logging.getLogger(__name__)
SAMPLE_RATE = 8_000
GITHUB_REPO = "https://github.com/dmlguq456/SepReformer.git"
_REPO_DIR = Path("/opt/sepreformer_repo")
# Map our model IDs → repo directory names under models/
_MODEL_DIRS = {
"sepreformer_base": "SepReformer_Base_WSJ0",
"sepreformer_large_wsj0": "SepReformer_Large_DM_WSJ0",
"sepreformer_large_wham": "SepReformer_Large_DM_WHAM",
"sepreformer_large_whamr": "SepReformer_Large_DM_WHAMR",
}
_STEM_NAMES = ["speaker_1", "speaker_2"]
# ---------------------------------------------------------------------------
# One-time setup
# ---------------------------------------------------------------------------
def _check_git_lfs() -> None:
try:
subprocess.run(["git", "lfs", "version"], check=True, capture_output=True)
except (subprocess.CalledProcessError, FileNotFoundError):
raise RuntimeError(
"git-lfs is required for SepReformer but is not installed.\n"
"Install it with: apt-get install git-lfs\n"
"Then add the following line to your Dockerfile:\n"
" RUN apt-get install -y git-lfs"
)
def _ensure_repo() -> None:
"""Clone SepReformer repo with Git LFS (includes pretrained weights)."""
if _REPO_DIR.exists():
return
_check_git_lfs()
_REPO_DIR.parent.mkdir(parents=True, exist_ok=True)
log.info("[SepReformer] Cloning %s with LFS weights …", GITHUB_REPO)
subprocess.run(["git", "lfs", "install"], check=True)
subprocess.run(
["git", "clone", GITHUB_REPO, str(_REPO_DIR)],
check=True,
)
# ---------------------------------------------------------------------------
# Dynamic model loading — each variant has its own model.py
# ---------------------------------------------------------------------------
def _import_model_class(model_name: str):
"""Import the Model class from the specific variant's model.py."""
model_py = _REPO_DIR / "models" / model_name / "model.py"
if not model_py.exists():
raise FileNotFoundError(
f"SepReformer model file not found: {model_py}\n"
"Ensure the repository was cloned with git-lfs."
)
# Add both the repo root and the model-specific dir to sys.path for internal imports
for p in [str(_REPO_DIR), str(_REPO_DIR / "models" / model_name)]:
if p not in sys.path:
sys.path.insert(0, p)
spec = importlib.util.spec_from_file_location(
f"sepreformer_{model_name}_model", str(model_py)
)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod.Model
# ---------------------------------------------------------------------------
# Audio helpers
# ---------------------------------------------------------------------------
def _load_audio_mono(path: str) -> torch.Tensor:
"""Load audio, mix to mono, resample to 8 kHz. Returns (1, N) float32."""
waveform, sr = torchaudio.load(path)
if waveform.shape[0] > 1:
waveform = waveform.mean(dim=0, keepdim=True)
if sr != SAMPLE_RATE:
waveform = torchaudio.functional.resample(waveform, sr, SAMPLE_RATE)
return waveform
def _save_audio(path: str, audio: torch.Tensor, fmt: str) -> None:
audio = audio.cpu()
f = fmt.lower()
if f.startswith("mp3"):
torchaudio.save(path, audio, SAMPLE_RATE, format="mp3")
elif f == "flac":
torchaudio.save(path, audio, SAMPLE_RATE, format="flac")
else:
torchaudio.save(path, audio, SAMPLE_RATE)
# ---------------------------------------------------------------------------
# Separator
# ---------------------------------------------------------------------------
class SepReformerSeparator:
"""
Native PyTorch inference wrapper for SepReformer 2-speaker speech separation.
Separates a mixed audio signal containing two speakers into individual
speaker streams. Best suited for speech signals (8 kHz).
Architecture code and pretrained weights are cloned from GitHub on
first use (requires git-lfs).
"""
def __init__(
self,
model_id: str = "sepreformer_base",
cpu_separation: bool = CPU_SEPARATION,
output_format: str = "mp3",
mp3_bitrate: str = "256k",
chunk_samples: int = 48_000,
):
if model_id not in _MODEL_DIRS:
raise ValueError(
f"Unknown SepReformer model: {model_id}. "
f"Available: {list(_MODEL_DIRS)}"
)
self.model_id = model_id
self.model_name = _MODEL_DIRS[model_id]
self.cpu_separation = cpu_separation
self.output_format = output_format
self.mp3_bitrate = mp3_bitrate
self.chunk_samples = chunk_samples
self.overlap_samples = max(1, chunk_samples // 8)
self.device = torch.device(
"cpu" if cpu_separation else ("cuda" if torch.cuda.is_available() else "cpu")
)
self._model = None
self._config = None
# ------------------------------------------------------------------
def _load_model(self) -> None:
_ensure_repo()
cfg_path = _REPO_DIR / "models" / self.model_name / "configs.yaml"
if not cfg_path.exists():
raise FileNotFoundError(
f"SepReformer config not found: {cfg_path}\n"
"The repository may not have cloned correctly."
)
with open(cfg_path) as f:
config = yaml.safe_load(f)
self._config = config
Model = _import_model_class(self.model_name)
model = Model(**config["model"])
# Find the checkpoint file in log/scratch_weights/
weights_dir = _REPO_DIR / "models" / self.model_name / "log" / "scratch_weights"
ckpt_files = sorted(weights_dir.glob("*.pth")) + sorted(weights_dir.glob("*.ckpt"))
if not ckpt_files:
raise FileNotFoundError(
f"No checkpoint found in {weights_dir}.\n"
"Ensure the repo was cloned with git-lfs active (git lfs pull)."
)
ckpt_path = ckpt_files[-1] # Use the latest checkpoint
log.info("[SepReformer] Loading checkpoint: %s", ckpt_path.name)
state = torch.load(str(ckpt_path), map_location="cpu", weights_only=False)
if isinstance(state, dict) and "model" in state:
state = state["model"]
elif isinstance(state, dict) and "state_dict" in state:
state = state["state_dict"]
model.load_state_dict(state, strict=False)
model.eval()
self._model = model.to(self.device)
def _get_model(self):
if self._model is None:
self._load_model()
return self._model
# ------------------------------------------------------------------
def _separate_chunked(self, model, audio: torch.Tensor, num_spks: int, stride: int) -> list:
"""Overlap-add chunked inference. Frees CUDA cache between chunks."""
chunk_samples = self.chunk_samples
overlap_samples = self.overlap_samples
# Align chunk boundary to encoder stride
chunk_samples = (chunk_samples // stride) * stride
if chunk_samples == 0:
chunk_samples = stride
hop = chunk_samples - overlap_samples * 2
if hop <= 0:
hop = chunk_samples // 2
# Align hop to stride as well
hop = max(stride, (hop // stride) * stride)
total_len = audio.shape[-1]
pad = overlap_samples
audio_padded = torch.nn.functional.pad(audio, (pad, max(0, total_len + pad * 2 - audio.shape[-1])))
window = torch.hann_window(chunk_samples, periodic=False)
out_buf = [torch.zeros(1, audio_padded.shape[-1]) for _ in range(num_spks)]
wgt_buf = torch.zeros(audio_padded.shape[-1])
pos = 0
chunk_idx = 0
while pos < total_len + pad:
chunk = audio_padded[:, pos:pos + chunk_samples]
if chunk.shape[-1] < chunk_samples:
chunk = torch.nn.functional.pad(chunk, (0, chunk_samples - chunk.shape[-1]))
chunk_dev = chunk.to(self.device)
with torch.inference_mode():
estim_src, _ = model(chunk_dev)
del chunk_dev
if self.device.type == "cuda":
torch.cuda.empty_cache()
w = window
for i, stem_tensor in enumerate(estim_src[:num_spks]):
stem_cpu = stem_tensor.detach().cpu()
if stem_cpu.dim() == 1:
stem_cpu = stem_cpu.unsqueeze(0)
if stem_cpu.shape[-1] < chunk_samples:
stem_cpu = torch.nn.functional.pad(stem_cpu, (0, chunk_samples - stem_cpu.shape[-1]))
win_stem = stem_cpu * w
actual = min(chunk_samples, out_buf[i].shape[-1] - pos)
if actual > 0:
out_buf[i][:, pos:pos + actual] += win_stem[:, :actual]
wgt_actual = min(chunk_samples, wgt_buf.shape[-1] - pos)
if wgt_actual > 0:
wgt_buf[pos:pos + wgt_actual] += w[:wgt_actual]
chunk_idx += 1
log.info(
"[SepReformer] Chunk %d done (%.1fs / %.1fs)",
chunk_idx,
pos / SAMPLE_RATE,
total_len / SAMPLE_RATE,
)
pos += hop
eps = 1e-8
result = []
for buf in out_buf:
normalised = buf / (wgt_buf + eps)
result.append(normalised[:, pad:pad + total_len])
return result
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
"""
Separate 2-speaker mixed audio into individual speaker streams.
Returns:
Dict mapping stem name -> absolute output file path.
"""
model = self._get_model()
config = self._config
audio = _load_audio_mono(input_path) # (1, N) on CPU
orig_len = audio.shape[-1]
duration = orig_len / SAMPLE_RATE
stride = config["model"]["module_audio_enc"]["stride"]
num_spks = config["model"].get("num_spks", 2)
log.info(
"[SepReformer] Running %s on %s (audio=%.1fs, chunk=%.1fs) ...",
self.model_name,
self.device,
duration,
self.chunk_samples / SAMPLE_RATE,
)
estim_src = self._separate_chunked(model, audio, num_spks, stride)
out_dir = Path(output_dir)
out_dir.mkdir(parents=True, exist_ok=True)
ext = self.output_format.lower()
if ext.startswith("mp3"):
ext = "mp3"
output_paths: Dict[str, str] = {}
for i, stem_name in enumerate(_STEM_NAMES[:num_spks]):
if i >= len(estim_src):
break
src = estim_src[i]
out_path = str(out_dir / f"{stem_name}.{ext}")
_save_audio(out_path, src, ext)
output_paths[stem_name] = out_path
log.info("[SepReformer] Saved %s -> %s", stem_name, out_path)
return output_paths