mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
feat(worker): voice-pool separate-only GPU job (v0.283.1) (#1217)
## Summary - Adds `voice-pool-separate-job`, a Cloud Run GPU (L4) job for karaoke-decide's voice session. decide writes a JSONL manifest (`voice-pool/manifests/<run>.jsonl`) and starts the job with a per-run shard; each task separates its share of songs with the `instrumental_clean` Separator and writes `voice-pool/stems/<id>/vocals.flac` (skipped if already present). Results are written back to the manifest area every 10 songs. - No Firestore or gen job state. Model-load failure or an all-songs-failed task exits non-zero with a single ERROR; partial failures exit 0. - Task timeout is 3600s (Cloud Run's GPU maximum, found by the local CodeRabbit review), so decide must size the shard count to keep each task under 1h. Re-runs skip finished songs. ## Changes - `backend/workers/voice_pool_separate_worker.py` + `backend/tests/test_voice_pool_separate_worker.py` (9 tests, 91% coverage of the worker) - `infrastructure/modules/cloud_run.py`, `infrastructure/__main__.py`: Pulumi job definition - `.github/workflows/ci.yml`: CI guard step (tolerates a missing job) - `docs/ARCHITECTURE.md`: paragraph on the job - Version 0.283.0 -> 0.283.1 ## Deploy notes **Andrew: run `pulumi up` in gen infrastructure/ BEFORE merging (creates voice-pool-separate-job; the CI guard tolerates a missing job).** decide's Pulumi (later PR) will create the voice-pool SA and grant it `roles/run.jobsExecutorWithOverrides` on this job. ## Testing - [x] `pytest backend/tests/ --ignore=backend/tests/emulator`: 6691 passed, 40 skipped - [x] Local CodeRabbit review completed (2 cycles): the critical timeout finding fixed; the remaining minor (results flush inside `finally`) is a known deferred item @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_0188txBF47GpTo7hym1FoifX --------- Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c4c4f9265c
commit
286e604b75
7 files changed
+516
-1
No files matched your search
@@ -0,0 +1,209 @@
|
||||
"""Tests for the voice-pool separate-only worker (synthetic data only)."""
|
||||
import json
|
||||
import os
|
||||
|
||||
from backend.workers import voice_pool_separate_worker as w
|
||||
|
||||
BUCKET = "karaoke-gen-storage-nomadkaraoke"
|
||||
MANIFEST = "gs://karaoke-gen-storage-nomadkaraoke/voice-pool/manifests/run1.jsonl"
|
||||
|
||||
|
||||
class FakeStorage:
|
||||
def __init__(self, ids, existing=()):
|
||||
lines = "".join(
|
||||
json.dumps({"id": i, "audio": f"gs://{BUCKET}/voice-pool/audio/{i}/original.flac"}) + "\n" for i in ids
|
||||
)
|
||||
self.blobs = {"voice-pool/manifests/run1.jsonl": lines.encode()}
|
||||
for i in existing:
|
||||
self.blobs[f"voice-pool/stems/{i}/vocals.flac"] = b"x"
|
||||
self.downloads = []
|
||||
|
||||
def download_bytes(self, p):
|
||||
return self.blobs[p]
|
||||
|
||||
def file_exists(self, p):
|
||||
return p in self.blobs
|
||||
|
||||
def download_file(self, src, dest):
|
||||
self.downloads.append(src)
|
||||
open(dest, "wb").write(b"audio")
|
||||
return dest
|
||||
|
||||
def upload_file(self, local, dest):
|
||||
self.blobs[dest] = open(local, "rb").read()
|
||||
return dest
|
||||
|
||||
def upload_text(self, dest, text, content_type="text/plain"):
|
||||
self.blobs[dest] = text.encode()
|
||||
return dest
|
||||
|
||||
|
||||
class FakeSeparator:
|
||||
def __init__(self, out_dir, fail_ids=()):
|
||||
self.out_dir, self.fail_ids, self.calls = out_dir, fail_ids, 0
|
||||
|
||||
def separate(self, path):
|
||||
self.calls += 1
|
||||
if any(f in str(self.calls) for f in self.fail_ids):
|
||||
raise RuntimeError("boom")
|
||||
paths = []
|
||||
for tag in ("Vocals", "Instrumental"):
|
||||
p = os.path.join(self.out_dir, f"{self.calls}_({tag})_preset_instrumental_clean.flac")
|
||||
open(p, "wb").write(tag.encode())
|
||||
paths.append(p)
|
||||
return paths
|
||||
|
||||
|
||||
def _results(storage, idx=0):
|
||||
raw = storage.blobs[f"voice-pool/manifests/run1.results/part-{idx}.jsonl"].decode()
|
||||
return [json.loads(x) for x in raw.splitlines()]
|
||||
|
||||
|
||||
def test_shard_complete_and_disjoint():
|
||||
lines = [{"id": str(i)} for i in range(23)]
|
||||
parts = [w.shard(lines, i, 4) for i in range(4)]
|
||||
flat = [x["id"] for p in parts for x in p]
|
||||
assert sorted(flat) == sorted(x["id"] for x in lines)
|
||||
assert len(set(flat)) == 23
|
||||
assert [x["id"] for x in parts[1]] == ["1", "5", "9", "13", "17", "21"]
|
||||
|
||||
|
||||
def test_vocals_path_from_outputs():
|
||||
paths = [
|
||||
"/o/x_(Instrumental)_preset_instrumental_clean.flac",
|
||||
"/o/x_(Vocals)_preset_instrumental_clean.flac",
|
||||
]
|
||||
assert w.vocals_path_from_outputs(paths) == paths[1]
|
||||
assert w.vocals_path_from_outputs(paths[:1]) is None
|
||||
assert w.vocals_path_from_outputs(["/o/x_(No Vocal)_preset_instrumental_clean.flac"]) is None
|
||||
|
||||
|
||||
def test_run_manifest_skips_existing_and_records_failures(tmp_path):
|
||||
storage = FakeStorage(["a", "b", "c"], existing=["a"])
|
||||
sep = FakeSeparator(str(tmp_path), fail_ids=("1",)) # first real separation raises
|
||||
built = []
|
||||
|
||||
def factory():
|
||||
built.append(1)
|
||||
return sep
|
||||
|
||||
results = w.run_manifest(storage, factory, MANIFEST, 0, 1, bucket=BUCKET)
|
||||
by_id = {r["id"]: r for r in results}
|
||||
assert by_id["a"]["ok"] and by_id["a"]["seconds"] == 0
|
||||
assert not by_id["b"]["ok"] and "boom" in by_id["b"]["error"]
|
||||
assert by_id["c"]["ok"] and by_id["c"]["vocals"] == "voice-pool/stems/c/vocals.flac"
|
||||
assert storage.blobs["voice-pool/stems/c/vocals.flac"] == b"Vocals"
|
||||
assert "voice-pool/stems/b/vocals.flac" not in storage.blobs
|
||||
assert "voice-pool/audio/a/original.flac" not in storage.downloads
|
||||
assert len(built) == 1 # one separator reused
|
||||
assert _results(storage) == results
|
||||
assert os.listdir(tmp_path) == [] # stems cleaned up
|
||||
|
||||
|
||||
def test_results_written_every_10(tmp_path, monkeypatch):
|
||||
ids = [str(i) for i in range(12)]
|
||||
storage = FakeStorage(ids)
|
||||
writes = []
|
||||
orig = w._write_results
|
||||
monkeypatch.setattr(w, "_write_results", lambda s, m, i, r: (writes.append(len(r)), orig(s, m, i, r)))
|
||||
w.run_manifest(storage, lambda: FakeSeparator(str(tmp_path)), MANIFEST, 0, 1, bucket=BUCKET)
|
||||
assert writes == [10, 12]
|
||||
|
||||
|
||||
def test_sharded_task_only_processes_its_share(tmp_path):
|
||||
storage = FakeStorage([str(i) for i in range(6)])
|
||||
w.run_manifest(storage, lambda: FakeSeparator(str(tmp_path)), MANIFEST, 1, 3, bucket=BUCKET)
|
||||
assert [r["id"] for r in _results(storage, 1)] == ["1", "4"]
|
||||
|
||||
|
||||
def test_separator_load_failure_fails_fast(tmp_path):
|
||||
import pytest
|
||||
|
||||
storage = FakeStorage(["a", "b"])
|
||||
calls = []
|
||||
|
||||
def factory():
|
||||
calls.append(1)
|
||||
raise RuntimeError("no gpu")
|
||||
|
||||
with pytest.raises(w.SeparatorLoadError):
|
||||
w.run_manifest(storage, factory, MANIFEST, 0, 1, bucket=BUCKET)
|
||||
assert len(calls) == 1 # no per-song retry of the model load
|
||||
|
||||
|
||||
def _patch_main(monkeypatch, run):
|
||||
import backend.services.storage_service as ss
|
||||
|
||||
monkeypatch.setattr(ss, "StorageService", lambda: object())
|
||||
monkeypatch.setattr(w, "run_manifest", run)
|
||||
|
||||
|
||||
def test_main_all_failed_logs_one_error_and_exits_nonzero(monkeypatch, caplog):
|
||||
fail = {"id": "a", "ok": False, "vocals": None, "seconds": 0, "error": "oom"}
|
||||
_patch_main(monkeypatch, lambda *a, **k: [fail, dict(fail, id="b")])
|
||||
assert w.main(["--manifest", MANIFEST]) == 1
|
||||
assert len([r for r in caplog.records if r.levelname == "ERROR"]) == 1
|
||||
|
||||
|
||||
def test_main_partial_failure_exits_zero_without_error(monkeypatch, caplog):
|
||||
ok = {"id": "a", "ok": True, "vocals": "v", "seconds": 1, "error": None}
|
||||
_patch_main(monkeypatch, lambda *a, **k: [ok, dict(ok, id="b", ok=False, error="x")])
|
||||
assert w.main(["--manifest", MANIFEST]) == 0
|
||||
assert not [r for r in caplog.records if r.levelname == "ERROR"]
|
||||
|
||||
|
||||
def test_main_load_failure_exits_nonzero_with_one_error(monkeypatch, caplog):
|
||||
def boom(*a, **k):
|
||||
raise w.SeparatorLoadError("x")
|
||||
|
||||
_patch_main(monkeypatch, boom)
|
||||
assert w.main(["--manifest", MANIFEST]) == 1
|
||||
assert len([r for r in caplog.records if r.levelname == "ERROR"]) == 1
|
||||
|
||||
|
||||
def test_periodic_flush_failure_is_tolerated_final_is_strict(tmp_path, monkeypatch):
|
||||
import pytest
|
||||
|
||||
storage = FakeStorage([str(i) for i in range(11)])
|
||||
orig = w._write_results
|
||||
n = []
|
||||
|
||||
def flaky(s, m, i, r):
|
||||
n.append(len(r))
|
||||
if len(n) == 1:
|
||||
raise OSError("transient gcs")
|
||||
return orig(s, m, i, r)
|
||||
|
||||
monkeypatch.setattr(w, "_write_results", flaky)
|
||||
results = w.run_manifest(storage, lambda: FakeSeparator(str(tmp_path)), MANIFEST, 0, 1, bucket=BUCKET)
|
||||
assert len(results) == 11 and n == [10, 11]
|
||||
|
||||
monkeypatch.setattr(w, "_write_results", lambda *a: (_ for _ in ()).throw(OSError("down")))
|
||||
with pytest.raises(OSError):
|
||||
w.run_manifest(FakeStorage(["a"]), lambda: FakeSeparator(str(tmp_path)), MANIFEST, 0, 1, bucket=BUCKET)
|
||||
|
||||
|
||||
def test_wrong_bucket_song_recorded_and_manifest_fails(tmp_path):
|
||||
import pytest
|
||||
|
||||
storage = FakeStorage(["a", "b"])
|
||||
storage.blobs["voice-pool/manifests/run1.jsonl"] = (
|
||||
json.dumps({"id": "a", "audio": "gs://other-bucket/voice-pool/audio/a/original.flac"}) + "\n"
|
||||
+ json.dumps({"id": "b", "audio": f"gs://{BUCKET}/voice-pool/audio/b/original.flac"}) + "\n"
|
||||
).encode()
|
||||
results = w.run_manifest(storage, lambda: FakeSeparator(str(tmp_path)), MANIFEST, 0, 1, bucket=BUCKET)
|
||||
assert results[0]["ok"] is False and results[0]["error"] == "wrong_bucket"
|
||||
assert results[1]["ok"] is True
|
||||
|
||||
with pytest.raises(w.SystemicTaskError):
|
||||
w.run_manifest(storage, lambda: FakeSeparator(str(tmp_path)),
|
||||
"gs://other-bucket/voice-pool/manifests/run1.jsonl", 0, 1, bucket=BUCKET)
|
||||
|
||||
|
||||
def test_systemic_error_messages_are_task_independent(monkeypatch, caplog):
|
||||
fail = {"id": "a", "ok": False, "vocals": None, "seconds": 0, "error": "oom"}
|
||||
_patch_main(monkeypatch, lambda *a, **k: [fail])
|
||||
monkeypatch.setenv("CLOUD_RUN_TASK_INDEX", "3")
|
||||
w.main(["--manifest", MANIFEST])
|
||||
errs = [r for r in caplog.records if r.levelname == "ERROR"]
|
||||
assert len(errs) == 1 and "3" not in errs[0].getMessage().split("(first")[0]
|
||||
@@ -0,0 +1,222 @@
|
||||
"""
|
||||
Voice-pool separate-only GPU worker (Cloud Run Job ``voice-pool-separate-job``).
|
||||
|
||||
Reads a JSONL manifest (written by karaoke-decide) of original-audio objects in
|
||||
the gen bucket, separates the vocals stem with the ``instrumental_clean``
|
||||
ensemble preset and writes ``voice-pool/stems/<id>/vocals.flac`` plus per-task
|
||||
results. It touches no Firestore and no gen job state.
|
||||
|
||||
Manifest line: {"id": pool_id, "audio": "gs://<bucket>/voice-pool/audio/<id>/original.flac"}
|
||||
Results: <manifest without .jsonl>.results/part-<task_index>.jsonl, one line per
|
||||
song: {"id", "ok", "vocals", "seconds", "error"}
|
||||
|
||||
Cloud Run shards the manifest across tasks (CLOUD_RUN_TASK_INDEX / CLOUD_RUN_TASK_COUNT).
|
||||
|
||||
Usage: python -m backend.workers.voice_pool_separate_worker --manifest gs://.../<run>.jsonl
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
INSTRUMENTAL_PRESET = "instrumental_clean"
|
||||
STEMS_PREFIX = "voice-pool/stems"
|
||||
RESULTS_FLUSH_EVERY = 10
|
||||
|
||||
|
||||
class SystemicTaskError(RuntimeError):
|
||||
"""Failure that affects every song: the task fails fast with one ERROR."""
|
||||
|
||||
|
||||
class SeparatorLoadError(SystemicTaskError):
|
||||
"""Model load failed: systemic, so the task fails fast instead of retrying per song."""
|
||||
|
||||
|
||||
class WrongBucketError(ValueError):
|
||||
"""A gs:// URI points outside the gen bucket."""
|
||||
|
||||
|
||||
def shard(lines: List[dict], index: int, count: int) -> List[dict]:
|
||||
"""Lines owned by task ``index`` of ``count`` (round-robin over manifest order)."""
|
||||
return [line for i, line in enumerate(lines) if i % count == index]
|
||||
|
||||
|
||||
def vocals_path_from_outputs(paths: List[str]) -> Optional[str]:
|
||||
"""Pick the vocals stem from ensemble outputs.
|
||||
|
||||
Filenames look like ``{base}_(Vocals)_preset_instrumental_clean.flac``; match the
|
||||
stem tag, never the preset name (which itself contains "instrumental").
|
||||
"""
|
||||
for p in paths:
|
||||
if "_(vocals)_" in os.path.basename(p).lower():
|
||||
return p
|
||||
return None
|
||||
|
||||
|
||||
def _blob_path(uri: str, bucket: Optional[str] = None) -> str:
|
||||
"""``gs://bucket/path`` -> ``path`` (StorageService is bound to the gen bucket).
|
||||
|
||||
Raises WrongBucketError when ``bucket`` is given and the URI names another bucket.
|
||||
"""
|
||||
if uri.startswith("gs://"):
|
||||
uri_bucket, _, path = uri[5:].partition("/")
|
||||
if bucket is not None and uri_bucket != bucket:
|
||||
raise WrongBucketError("wrong_bucket")
|
||||
return path
|
||||
return uri
|
||||
|
||||
|
||||
def _results_path(manifest_uri: str, index: int) -> str:
|
||||
base = _blob_path(manifest_uri)
|
||||
if base.endswith(".jsonl"):
|
||||
base = base[: -len(".jsonl")]
|
||||
return f"{base}.results/part-{index}.jsonl"
|
||||
|
||||
|
||||
def _write_results(storage, manifest_uri: str, index: int, results: List[dict]) -> None:
|
||||
body = "".join(json.dumps(r) + "\n" for r in results)
|
||||
storage.upload_text(_results_path(manifest_uri, index), body, content_type="application/x-ndjson")
|
||||
|
||||
|
||||
def _read_manifest(storage, manifest_uri: str, bucket: Optional[str] = None) -> List[dict]:
|
||||
raw = storage.download_bytes(_blob_path(manifest_uri, bucket)).decode("utf-8")
|
||||
return [json.loads(line) for line in raw.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def run_manifest(
|
||||
storage,
|
||||
separator_factory: Callable[[], Any],
|
||||
manifest_uri: str,
|
||||
index: int,
|
||||
count: int,
|
||||
bucket: Optional[str] = None,
|
||||
) -> List[dict]:
|
||||
"""Separate this task's share of the manifest; never aborts on a per-song failure."""
|
||||
if bucket is None:
|
||||
from backend.config import get_settings
|
||||
|
||||
bucket = get_settings().gcs_bucket_name
|
||||
try:
|
||||
manifest = _read_manifest(storage, manifest_uri, bucket)
|
||||
except WrongBucketError:
|
||||
raise SystemicTaskError(f"manifest bucket is not {bucket} (wrong_bucket)")
|
||||
lines = shard(manifest, index, count)
|
||||
logger.info(f"Task {index}/{count}: {len(lines)} songs")
|
||||
results: List[dict] = []
|
||||
separator = None # built lazily so an all-skipped shard never loads models
|
||||
|
||||
for n, line in enumerate(lines, 1):
|
||||
pool_id = line.get("id")
|
||||
vocals_blob = f"{STEMS_PREFIX}/{pool_id}/vocals.flac"
|
||||
started = time.time()
|
||||
tmp = tempfile.mkdtemp(prefix="voicepool_")
|
||||
try:
|
||||
if storage.file_exists(vocals_blob):
|
||||
results.append({"id": pool_id, "ok": True, "vocals": vocals_blob, "seconds": 0, "error": None})
|
||||
continue
|
||||
if separator is None:
|
||||
try:
|
||||
separator = separator_factory()
|
||||
except Exception as e:
|
||||
raise SeparatorLoadError(f"separator failed to load: {e}") from e
|
||||
local = os.path.join(tmp, "original.flac")
|
||||
storage.download_file(_blob_path(line["audio"], bucket), local)
|
||||
outputs = separator.separate(local)
|
||||
# Separator may return bare filenames relative to its output_dir
|
||||
out_dir = getattr(separator, "output_dir", None)
|
||||
outputs = [o if os.path.isabs(o) or not out_dir else os.path.join(out_dir, o) for o in outputs]
|
||||
vocals = vocals_path_from_outputs(outputs)
|
||||
if not vocals or not os.path.exists(vocals):
|
||||
raise RuntimeError(f"no vocals stem in separator outputs: {[os.path.basename(o) for o in outputs]}")
|
||||
storage.upload_file(vocals, vocals_blob)
|
||||
for o in outputs: # all stems live in the shared output dir; free it per song
|
||||
try:
|
||||
os.remove(o)
|
||||
except OSError:
|
||||
pass
|
||||
results.append({
|
||||
"id": pool_id, "ok": True, "vocals": vocals_blob,
|
||||
"seconds": round(time.time() - started, 1), "error": None,
|
||||
})
|
||||
except SystemicTaskError:
|
||||
_write_results(storage, manifest_uri, index, results)
|
||||
raise
|
||||
except Exception as e: # per-song failure must not abort the task
|
||||
logger.warning(f"voice-pool separate failed for {pool_id}: {e}")
|
||||
results.append({
|
||||
"id": pool_id, "ok": False, "vocals": None,
|
||||
"seconds": round(time.time() - started, 1), "error": str(e)[:500],
|
||||
})
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
if n % RESULTS_FLUSH_EVERY == 0:
|
||||
try: # periodic flush is best-effort; the final write stays strict
|
||||
_write_results(storage, manifest_uri, index, results)
|
||||
except Exception as e:
|
||||
logger.warning(f"periodic results flush failed (will retry): {e}")
|
||||
|
||||
_write_results(storage, manifest_uri, index, results)
|
||||
return results
|
||||
|
||||
|
||||
def make_separator_factory(output_dir: str) -> Callable[[], Any]:
|
||||
def factory():
|
||||
from audio_separator.separator import Separator
|
||||
|
||||
from karaoke_gen.audio_processor import _separator_kwargs
|
||||
|
||||
separator = Separator(
|
||||
model_file_dir=os.environ["MODEL_DIR"],
|
||||
output_format="FLAC",
|
||||
output_dir=output_dir,
|
||||
ensemble_preset=INSTRUMENTAL_PRESET,
|
||||
**_separator_kwargs(),
|
||||
)
|
||||
separator.load_model()
|
||||
return separator
|
||||
|
||||
return factory
|
||||
|
||||
|
||||
def main(argv: Optional[List[str]] = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--manifest", required=True, help="gs://bucket/voice-pool/manifests/<run>.jsonl")
|
||||
args = parser.parse_args(argv)
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
|
||||
index = int(os.environ.get("CLOUD_RUN_TASK_INDEX", "0"))
|
||||
count = int(os.environ.get("CLOUD_RUN_TASK_COUNT", "1"))
|
||||
|
||||
from backend.services.storage_service import StorageService
|
||||
|
||||
out_dir = tempfile.mkdtemp(prefix="voicepool_out_")
|
||||
try:
|
||||
results = run_manifest(StorageService(), make_separator_factory(out_dir), args.manifest, index, count)
|
||||
except SystemicTaskError as e:
|
||||
# task-independent text so every failing task groups as one alert pattern
|
||||
logger.info(f"Task {index}/{count} aborting")
|
||||
logger.error(f"voice-pool separate task aborted: {e}", extra={"task_index": index})
|
||||
return 1
|
||||
finally:
|
||||
shutil.rmtree(out_dir, ignore_errors=True)
|
||||
failed = sum(1 for r in results if not r["ok"])
|
||||
if results and failed == len(results):
|
||||
logger.info(f"Task {index}/{count}: all {failed} songs failed")
|
||||
logger.error(
|
||||
f"voice-pool separate task failed: every song failed (first error: {results[0]['error']})",
|
||||
extra={"task_index": index, "failed": failed},
|
||||
)
|
||||
return 1
|
||||
logger.info(f"Task {index}/{count} done: {len(results) - failed} ok, {failed} failed")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user