mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary - Adds `voice-pool-separate-job`, a Cloud Run GPU (L4) job for karaoke-decide's voice session. decide writes a JSONL manifest (`voice-pool/manifests/<run>.jsonl`) and starts the job with a per-run shard; each task separates its share of songs with the `instrumental_clean` Separator and writes `voice-pool/stems/<id>/vocals.flac` (skipped if already present). Results are written back to the manifest area every 10 songs. - No Firestore or gen job state. Model-load failure or an all-songs-failed task exits non-zero with a single ERROR; partial failures exit 0. - Task timeout is 3600s (Cloud Run's GPU maximum, found by the local CodeRabbit review), so decide must size the shard count to keep each task under 1h. Re-runs skip finished songs. ## Changes - `backend/workers/voice_pool_separate_worker.py` + `backend/tests/test_voice_pool_separate_worker.py` (9 tests, 91% coverage of the worker) - `infrastructure/modules/cloud_run.py`, `infrastructure/__main__.py`: Pulumi job definition - `.github/workflows/ci.yml`: CI guard step (tolerates a missing job) - `docs/ARCHITECTURE.md`: paragraph on the job - Version 0.283.0 -> 0.283.1 ## Deploy notes **Andrew: run `pulumi up` in gen infrastructure/ BEFORE merging (creates voice-pool-separate-job; the CI guard tolerates a missing job).** decide's Pulumi (later PR) will create the voice-pool SA and grant it `roles/run.jobsExecutorWithOverrides` on this job. ## Testing - [x] `pytest backend/tests/ --ignore=backend/tests/emulator`: 6691 passed, 40 skipped - [x] Local CodeRabbit review completed (2 cycles): the critical timeout finding fixed; the remaining minor (results flush inside `finally`) is a known deferred item @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_0188txBF47GpTo7hym1FoifX --------- Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
223 lines
8.7 KiB
Python
223 lines
8.7 KiB
Python
"""
|
|
Voice-pool separate-only GPU worker (Cloud Run Job ``voice-pool-separate-job``).
|
|
|
|
Reads a JSONL manifest (written by karaoke-decide) of original-audio objects in
|
|
the gen bucket, separates the vocals stem with the ``instrumental_clean``
|
|
ensemble preset and writes ``voice-pool/stems/<id>/vocals.flac`` plus per-task
|
|
results. It touches no Firestore and no gen job state.
|
|
|
|
Manifest line: {"id": pool_id, "audio": "gs://<bucket>/voice-pool/audio/<id>/original.flac"}
|
|
Results: <manifest without .jsonl>.results/part-<task_index>.jsonl, one line per
|
|
song: {"id", "ok", "vocals", "seconds", "error"}
|
|
|
|
Cloud Run shards the manifest across tasks (CLOUD_RUN_TASK_INDEX / CLOUD_RUN_TASK_COUNT).
|
|
|
|
Usage: python -m backend.workers.voice_pool_separate_worker --manifest gs://.../<run>.jsonl
|
|
"""
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from typing import Any, Callable, Dict, List, Optional
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
INSTRUMENTAL_PRESET = "instrumental_clean"
|
|
STEMS_PREFIX = "voice-pool/stems"
|
|
RESULTS_FLUSH_EVERY = 10
|
|
|
|
|
|
class SystemicTaskError(RuntimeError):
|
|
"""Failure that affects every song: the task fails fast with one ERROR."""
|
|
|
|
|
|
class SeparatorLoadError(SystemicTaskError):
|
|
"""Model load failed: systemic, so the task fails fast instead of retrying per song."""
|
|
|
|
|
|
class WrongBucketError(ValueError):
|
|
"""A gs:// URI points outside the gen bucket."""
|
|
|
|
|
|
def shard(lines: List[dict], index: int, count: int) -> List[dict]:
|
|
"""Lines owned by task ``index`` of ``count`` (round-robin over manifest order)."""
|
|
return [line for i, line in enumerate(lines) if i % count == index]
|
|
|
|
|
|
def vocals_path_from_outputs(paths: List[str]) -> Optional[str]:
|
|
"""Pick the vocals stem from ensemble outputs.
|
|
|
|
Filenames look like ``{base}_(Vocals)_preset_instrumental_clean.flac``; match the
|
|
stem tag, never the preset name (which itself contains "instrumental").
|
|
"""
|
|
for p in paths:
|
|
if "_(vocals)_" in os.path.basename(p).lower():
|
|
return p
|
|
return None
|
|
|
|
|
|
def _blob_path(uri: str, bucket: Optional[str] = None) -> str:
|
|
"""``gs://bucket/path`` -> ``path`` (StorageService is bound to the gen bucket).
|
|
|
|
Raises WrongBucketError when ``bucket`` is given and the URI names another bucket.
|
|
"""
|
|
if uri.startswith("gs://"):
|
|
uri_bucket, _, path = uri[5:].partition("/")
|
|
if bucket is not None and uri_bucket != bucket:
|
|
raise WrongBucketError("wrong_bucket")
|
|
return path
|
|
return uri
|
|
|
|
|
|
def _results_path(manifest_uri: str, index: int) -> str:
|
|
base = _blob_path(manifest_uri)
|
|
if base.endswith(".jsonl"):
|
|
base = base[: -len(".jsonl")]
|
|
return f"{base}.results/part-{index}.jsonl"
|
|
|
|
|
|
def _write_results(storage, manifest_uri: str, index: int, results: List[dict]) -> None:
|
|
body = "".join(json.dumps(r) + "\n" for r in results)
|
|
storage.upload_text(_results_path(manifest_uri, index), body, content_type="application/x-ndjson")
|
|
|
|
|
|
def _read_manifest(storage, manifest_uri: str, bucket: Optional[str] = None) -> List[dict]:
|
|
raw = storage.download_bytes(_blob_path(manifest_uri, bucket)).decode("utf-8")
|
|
return [json.loads(line) for line in raw.splitlines() if line.strip()]
|
|
|
|
|
|
def run_manifest(
|
|
storage,
|
|
separator_factory: Callable[[], Any],
|
|
manifest_uri: str,
|
|
index: int,
|
|
count: int,
|
|
bucket: Optional[str] = None,
|
|
) -> List[dict]:
|
|
"""Separate this task's share of the manifest; never aborts on a per-song failure."""
|
|
if bucket is None:
|
|
from backend.config import get_settings
|
|
|
|
bucket = get_settings().gcs_bucket_name
|
|
try:
|
|
manifest = _read_manifest(storage, manifest_uri, bucket)
|
|
except WrongBucketError:
|
|
raise SystemicTaskError(f"manifest bucket is not {bucket} (wrong_bucket)")
|
|
lines = shard(manifest, index, count)
|
|
logger.info(f"Task {index}/{count}: {len(lines)} songs")
|
|
results: List[dict] = []
|
|
separator = None # built lazily so an all-skipped shard never loads models
|
|
|
|
for n, line in enumerate(lines, 1):
|
|
pool_id = line.get("id")
|
|
vocals_blob = f"{STEMS_PREFIX}/{pool_id}/vocals.flac"
|
|
started = time.time()
|
|
tmp = tempfile.mkdtemp(prefix="voicepool_")
|
|
try:
|
|
if storage.file_exists(vocals_blob):
|
|
results.append({"id": pool_id, "ok": True, "vocals": vocals_blob, "seconds": 0, "error": None})
|
|
continue
|
|
if separator is None:
|
|
try:
|
|
separator = separator_factory()
|
|
except Exception as e:
|
|
raise SeparatorLoadError(f"separator failed to load: {e}") from e
|
|
local = os.path.join(tmp, "original.flac")
|
|
storage.download_file(_blob_path(line["audio"], bucket), local)
|
|
outputs = separator.separate(local)
|
|
# Separator may return bare filenames relative to its output_dir
|
|
out_dir = getattr(separator, "output_dir", None)
|
|
outputs = [o if os.path.isabs(o) or not out_dir else os.path.join(out_dir, o) for o in outputs]
|
|
vocals = vocals_path_from_outputs(outputs)
|
|
if not vocals or not os.path.exists(vocals):
|
|
raise RuntimeError(f"no vocals stem in separator outputs: {[os.path.basename(o) for o in outputs]}")
|
|
storage.upload_file(vocals, vocals_blob)
|
|
for o in outputs: # all stems live in the shared output dir; free it per song
|
|
try:
|
|
os.remove(o)
|
|
except OSError:
|
|
pass
|
|
results.append({
|
|
"id": pool_id, "ok": True, "vocals": vocals_blob,
|
|
"seconds": round(time.time() - started, 1), "error": None,
|
|
})
|
|
except SystemicTaskError:
|
|
_write_results(storage, manifest_uri, index, results)
|
|
raise
|
|
except Exception as e: # per-song failure must not abort the task
|
|
logger.warning(f"voice-pool separate failed for {pool_id}: {e}")
|
|
results.append({
|
|
"id": pool_id, "ok": False, "vocals": None,
|
|
"seconds": round(time.time() - started, 1), "error": str(e)[:500],
|
|
})
|
|
finally:
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
if n % RESULTS_FLUSH_EVERY == 0:
|
|
try: # periodic flush is best-effort; the final write stays strict
|
|
_write_results(storage, manifest_uri, index, results)
|
|
except Exception as e:
|
|
logger.warning(f"periodic results flush failed (will retry): {e}")
|
|
|
|
_write_results(storage, manifest_uri, index, results)
|
|
return results
|
|
|
|
|
|
def make_separator_factory(output_dir: str) -> Callable[[], Any]:
|
|
def factory():
|
|
from audio_separator.separator import Separator
|
|
|
|
from karaoke_gen.audio_processor import _separator_kwargs
|
|
|
|
separator = Separator(
|
|
model_file_dir=os.environ["MODEL_DIR"],
|
|
output_format="FLAC",
|
|
output_dir=output_dir,
|
|
ensemble_preset=INSTRUMENTAL_PRESET,
|
|
**_separator_kwargs(),
|
|
)
|
|
separator.load_model()
|
|
return separator
|
|
|
|
return factory
|
|
|
|
|
|
def main(argv: Optional[List[str]] = None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--manifest", required=True, help="gs://bucket/voice-pool/manifests/<run>.jsonl")
|
|
args = parser.parse_args(argv)
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
|
|
|
index = int(os.environ.get("CLOUD_RUN_TASK_INDEX", "0"))
|
|
count = int(os.environ.get("CLOUD_RUN_TASK_COUNT", "1"))
|
|
|
|
from backend.services.storage_service import StorageService
|
|
|
|
out_dir = tempfile.mkdtemp(prefix="voicepool_out_")
|
|
try:
|
|
results = run_manifest(StorageService(), make_separator_factory(out_dir), args.manifest, index, count)
|
|
except SystemicTaskError as e:
|
|
# task-independent text so every failing task groups as one alert pattern
|
|
logger.info(f"Task {index}/{count} aborting")
|
|
logger.error(f"voice-pool separate task aborted: {e}", extra={"task_index": index})
|
|
return 1
|
|
finally:
|
|
shutil.rmtree(out_dir, ignore_errors=True)
|
|
failed = sum(1 for r in results if not r["ok"])
|
|
if results and failed == len(results):
|
|
logger.info(f"Task {index}/{count}: all {failed} songs failed")
|
|
logger.error(
|
|
f"voice-pool separate task failed: every song failed (first error: {results[0]['error']})",
|
|
extra={"task_index": index, "failed": failed},
|
|
)
|
|
return 1
|
|
logger.info(f"Task {index}/{count} done: {len(results) - failed} ok, {failed} failed")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|