335 lines
12 KiB
Python
335 lines
12 KiB
Python
"""
|
|
SepReformer separator — 2-speaker speech separation.
|
|
|
|
Architecture and weights from dmlguq456/SepReformer (NeurIPS 2024).
|
|
Weights are bundled in the repository via Git LFS.
|
|
|
|
Requires git-lfs to be installed in the container:
|
|
apt-get install git-lfs
|
|
Add to Dockerfile: RUN apt-get install -y git-lfs
|
|
|
|
Separates 2-speaker mixed audio into:
|
|
speaker_1, speaker_2
|
|
|
|
Sample rate: 8 kHz — input audio is automatically resampled.
|
|
|
|
Available model variants:
|
|
sepreformer_base — WSJ0-2Mix (clean speech)
|
|
sepreformer_large_wsj0 — WSJ0-2Mix large (clean speech)
|
|
sepreformer_large_wham — WHAM! (noisy speech)
|
|
sepreformer_large_whamr — WHAMR! (noisy + reverberant speech)
|
|
"""
|
|
import importlib.util
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import logging
|
|
from pathlib import Path
|
|
from typing import Dict, Optional
|
|
|
|
import torch
|
|
import torchaudio
|
|
import yaml
|
|
|
|
from ..config import SEPREFORMER_MODEL_DIR, CPU_SEPARATION
|
|
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
SAMPLE_RATE = 8_000
|
|
GITHUB_REPO = "https://github.com/dmlguq456/SepReformer.git"
|
|
_REPO_DIR = Path("/opt/sepreformer_repo")
|
|
|
|
# Map our model IDs → repo directory names under models/
|
|
_MODEL_DIRS = {
|
|
"sepreformer_base": "SepReformer_Base_WSJ0",
|
|
"sepreformer_large_wsj0": "SepReformer_Large_DM_WSJ0",
|
|
"sepreformer_large_wham": "SepReformer_Large_DM_WHAM",
|
|
"sepreformer_large_whamr": "SepReformer_Large_DM_WHAMR",
|
|
}
|
|
|
|
_STEM_NAMES = ["speaker_1", "speaker_2"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# One-time setup
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _check_git_lfs() -> None:
|
|
try:
|
|
subprocess.run(["git", "lfs", "version"], check=True, capture_output=True)
|
|
except (subprocess.CalledProcessError, FileNotFoundError):
|
|
raise RuntimeError(
|
|
"git-lfs is required for SepReformer but is not installed.\n"
|
|
"Install it with: apt-get install git-lfs\n"
|
|
"Then add the following line to your Dockerfile:\n"
|
|
" RUN apt-get install -y git-lfs"
|
|
)
|
|
|
|
|
|
def _ensure_repo() -> None:
|
|
"""Clone SepReformer repo with Git LFS (includes pretrained weights)."""
|
|
if _REPO_DIR.exists():
|
|
return
|
|
_check_git_lfs()
|
|
_REPO_DIR.parent.mkdir(parents=True, exist_ok=True)
|
|
log.info("[SepReformer] Cloning %s with LFS weights …", GITHUB_REPO)
|
|
subprocess.run(["git", "lfs", "install"], check=True)
|
|
subprocess.run(
|
|
["git", "clone", GITHUB_REPO, str(_REPO_DIR)],
|
|
check=True,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dynamic model loading — each variant has its own model.py
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _import_model_class(model_name: str):
|
|
"""Import the Model class from the specific variant's model.py."""
|
|
model_py = _REPO_DIR / "models" / model_name / "model.py"
|
|
if not model_py.exists():
|
|
raise FileNotFoundError(
|
|
f"SepReformer model file not found: {model_py}\n"
|
|
"Ensure the repository was cloned with git-lfs."
|
|
)
|
|
# Add both the repo root and the model-specific dir to sys.path for internal imports
|
|
for p in [str(_REPO_DIR), str(_REPO_DIR / "models" / model_name)]:
|
|
if p not in sys.path:
|
|
sys.path.insert(0, p)
|
|
|
|
spec = importlib.util.spec_from_file_location(
|
|
f"sepreformer_{model_name}_model", str(model_py)
|
|
)
|
|
mod = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(mod)
|
|
return mod.Model
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Audio helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _load_audio_mono(path: str) -> torch.Tensor:
|
|
"""Load audio, mix to mono, resample to 8 kHz. Returns (1, N) float32."""
|
|
waveform, sr = torchaudio.load(path)
|
|
if waveform.shape[0] > 1:
|
|
waveform = waveform.mean(dim=0, keepdim=True)
|
|
if sr != SAMPLE_RATE:
|
|
waveform = torchaudio.functional.resample(waveform, sr, SAMPLE_RATE)
|
|
return waveform
|
|
|
|
|
|
def _save_audio(path: str, audio: torch.Tensor, fmt: str) -> None:
|
|
audio = audio.cpu()
|
|
f = fmt.lower()
|
|
if f.startswith("mp3"):
|
|
torchaudio.save(path, audio, SAMPLE_RATE, format="mp3")
|
|
elif f == "flac":
|
|
torchaudio.save(path, audio, SAMPLE_RATE, format="flac")
|
|
else:
|
|
torchaudio.save(path, audio, SAMPLE_RATE)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Separator
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class SepReformerSeparator:
|
|
"""
|
|
Native PyTorch inference wrapper for SepReformer 2-speaker speech separation.
|
|
|
|
Separates a mixed audio signal containing two speakers into individual
|
|
speaker streams. Best suited for speech signals (8 kHz).
|
|
|
|
Architecture code and pretrained weights are cloned from GitHub on
|
|
first use (requires git-lfs).
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
model_id: str = "sepreformer_base",
|
|
cpu_separation: bool = CPU_SEPARATION,
|
|
output_format: str = "mp3",
|
|
mp3_bitrate: str = "256k",
|
|
chunk_samples: int = 48_000,
|
|
):
|
|
if model_id not in _MODEL_DIRS:
|
|
raise ValueError(
|
|
f"Unknown SepReformer model: {model_id}. "
|
|
f"Available: {list(_MODEL_DIRS)}"
|
|
)
|
|
self.model_id = model_id
|
|
self.model_name = _MODEL_DIRS[model_id]
|
|
self.cpu_separation = cpu_separation
|
|
self.output_format = output_format
|
|
self.mp3_bitrate = mp3_bitrate
|
|
self.chunk_samples = chunk_samples
|
|
self.overlap_samples = max(1, chunk_samples // 8)
|
|
self.device = torch.device(
|
|
"cpu" if cpu_separation else ("cuda" if torch.cuda.is_available() else "cpu")
|
|
)
|
|
self._model = None
|
|
self._config = None
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
def _load_model(self) -> None:
|
|
_ensure_repo()
|
|
cfg_path = _REPO_DIR / "models" / self.model_name / "configs.yaml"
|
|
if not cfg_path.exists():
|
|
raise FileNotFoundError(
|
|
f"SepReformer config not found: {cfg_path}\n"
|
|
"The repository may not have cloned correctly."
|
|
)
|
|
with open(cfg_path) as f:
|
|
config = yaml.safe_load(f)
|
|
self._config = config
|
|
|
|
Model = _import_model_class(self.model_name)
|
|
model = Model(**config["model"])
|
|
|
|
# Find the checkpoint file in log/scratch_weights/
|
|
weights_dir = _REPO_DIR / "models" / self.model_name / "log" / "scratch_weights"
|
|
ckpt_files = sorted(weights_dir.glob("*.pth")) + sorted(weights_dir.glob("*.ckpt"))
|
|
if not ckpt_files:
|
|
raise FileNotFoundError(
|
|
f"No checkpoint found in {weights_dir}.\n"
|
|
"Ensure the repo was cloned with git-lfs active (git lfs pull)."
|
|
)
|
|
ckpt_path = ckpt_files[-1] # Use the latest checkpoint
|
|
log.info("[SepReformer] Loading checkpoint: %s", ckpt_path.name)
|
|
|
|
state = torch.load(str(ckpt_path), map_location="cpu", weights_only=False)
|
|
if isinstance(state, dict) and "model" in state:
|
|
state = state["model"]
|
|
elif isinstance(state, dict) and "state_dict" in state:
|
|
state = state["state_dict"]
|
|
model.load_state_dict(state, strict=False)
|
|
model.eval()
|
|
self._model = model.to(self.device)
|
|
|
|
def _get_model(self):
|
|
if self._model is None:
|
|
self._load_model()
|
|
return self._model
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
def _separate_chunked(self, model, audio: torch.Tensor, num_spks: int, stride: int) -> list:
|
|
"""Overlap-add chunked inference. Frees CUDA cache between chunks."""
|
|
chunk_samples = self.chunk_samples
|
|
overlap_samples = self.overlap_samples
|
|
|
|
# Align chunk boundary to encoder stride
|
|
chunk_samples = (chunk_samples // stride) * stride
|
|
if chunk_samples == 0:
|
|
chunk_samples = stride
|
|
|
|
hop = chunk_samples - overlap_samples * 2
|
|
if hop <= 0:
|
|
hop = chunk_samples // 2
|
|
# Align hop to stride as well
|
|
hop = max(stride, (hop // stride) * stride)
|
|
|
|
total_len = audio.shape[-1]
|
|
pad = overlap_samples
|
|
audio_padded = torch.nn.functional.pad(audio, (pad, max(0, total_len + pad * 2 - audio.shape[-1])))
|
|
|
|
window = torch.hann_window(chunk_samples, periodic=False)
|
|
out_buf = [torch.zeros(1, audio_padded.shape[-1]) for _ in range(num_spks)]
|
|
wgt_buf = torch.zeros(audio_padded.shape[-1])
|
|
|
|
pos = 0
|
|
chunk_idx = 0
|
|
while pos < total_len + pad:
|
|
chunk = audio_padded[:, pos:pos + chunk_samples]
|
|
if chunk.shape[-1] < chunk_samples:
|
|
chunk = torch.nn.functional.pad(chunk, (0, chunk_samples - chunk.shape[-1]))
|
|
|
|
chunk_dev = chunk.to(self.device)
|
|
with torch.inference_mode():
|
|
estim_src, _ = model(chunk_dev)
|
|
del chunk_dev
|
|
|
|
if self.device.type == "cuda":
|
|
torch.cuda.empty_cache()
|
|
|
|
w = window
|
|
for i, stem_tensor in enumerate(estim_src[:num_spks]):
|
|
stem_cpu = stem_tensor.detach().cpu()
|
|
if stem_cpu.dim() == 1:
|
|
stem_cpu = stem_cpu.unsqueeze(0)
|
|
if stem_cpu.shape[-1] < chunk_samples:
|
|
stem_cpu = torch.nn.functional.pad(stem_cpu, (0, chunk_samples - stem_cpu.shape[-1]))
|
|
win_stem = stem_cpu * w
|
|
actual = min(chunk_samples, out_buf[i].shape[-1] - pos)
|
|
if actual > 0:
|
|
out_buf[i][:, pos:pos + actual] += win_stem[:, :actual]
|
|
|
|
wgt_actual = min(chunk_samples, wgt_buf.shape[-1] - pos)
|
|
if wgt_actual > 0:
|
|
wgt_buf[pos:pos + wgt_actual] += w[:wgt_actual]
|
|
|
|
chunk_idx += 1
|
|
log.info(
|
|
"[SepReformer] Chunk %d done (%.1fs / %.1fs)",
|
|
chunk_idx,
|
|
pos / SAMPLE_RATE,
|
|
total_len / SAMPLE_RATE,
|
|
)
|
|
pos += hop
|
|
|
|
eps = 1e-8
|
|
result = []
|
|
for buf in out_buf:
|
|
normalised = buf / (wgt_buf + eps)
|
|
result.append(normalised[:, pad:pad + total_len])
|
|
return result
|
|
|
|
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
|
"""
|
|
Separate 2-speaker mixed audio into individual speaker streams.
|
|
|
|
Returns:
|
|
Dict mapping stem name -> absolute output file path.
|
|
"""
|
|
model = self._get_model()
|
|
config = self._config
|
|
audio = _load_audio_mono(input_path) # (1, N) on CPU
|
|
orig_len = audio.shape[-1]
|
|
duration = orig_len / SAMPLE_RATE
|
|
|
|
stride = config["model"]["module_audio_enc"]["stride"]
|
|
num_spks = config["model"].get("num_spks", 2)
|
|
|
|
log.info(
|
|
"[SepReformer] Running %s on %s (audio=%.1fs, chunk=%.1fs) ...",
|
|
self.model_name,
|
|
self.device,
|
|
duration,
|
|
self.chunk_samples / SAMPLE_RATE,
|
|
)
|
|
|
|
estim_src = self._separate_chunked(model, audio, num_spks, stride)
|
|
|
|
out_dir = Path(output_dir)
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
ext = self.output_format.lower()
|
|
if ext.startswith("mp3"):
|
|
ext = "mp3"
|
|
|
|
output_paths: Dict[str, str] = {}
|
|
for i, stem_name in enumerate(_STEM_NAMES[:num_spks]):
|
|
if i >= len(estim_src):
|
|
break
|
|
src = estim_src[i]
|
|
out_path = str(out_dir / f"{stem_name}.{ext}")
|
|
_save_audio(out_path, src, ext)
|
|
output_paths[stem_name] = out_path
|
|
log.info("[SepReformer] Saved %s -> %s", stem_name, out_path)
|
|
|
|
return output_paths
|