Files
karaoke-gen/backend/workers/voice_pool_separate_worker.py
Andrew Beveridge ☄️andClaude Sonnet 5.5 286e604b75 feat(worker): voice-pool separate-only GPU job (v0.283.1) (#1217)
## Summary
- Adds `voice-pool-separate-job`, a Cloud Run GPU (L4) job for
karaoke-decide's voice session. decide writes a JSONL manifest
(`voice-pool/manifests/<run>.jsonl`) and starts the job with a per-run
shard; each task separates its share of songs with the
`instrumental_clean` Separator and writes
`voice-pool/stems/<id>/vocals.flac` (skipped if already present).
Results are written back to the manifest area every 10 songs.
- No Firestore or gen job state. Model-load failure or an
all-songs-failed task exits non-zero with a single ERROR; partial
failures exit 0.
- Task timeout is 3600s (Cloud Run's GPU maximum, found by the local
CodeRabbit review), so decide must size the shard count to keep each
task under 1h. Re-runs skip finished songs.

## Changes
- `backend/workers/voice_pool_separate_worker.py` +
`backend/tests/test_voice_pool_separate_worker.py` (9 tests, 91%
coverage of the worker)
- `infrastructure/modules/cloud_run.py`, `infrastructure/__main__.py`:
Pulumi job definition
- `.github/workflows/ci.yml`: CI guard step (tolerates a missing job)
- `docs/ARCHITECTURE.md`: paragraph on the job
- Version 0.283.0 -> 0.283.1

## Deploy notes
**Andrew: run `pulumi up` in gen infrastructure/ BEFORE merging (creates
voice-pool-separate-job; the CI guard tolerates a missing job).**
decide's Pulumi (later PR) will create the voice-pool SA and grant it
`roles/run.jobsExecutorWithOverrides` on this job.

## Testing
- [x] `pytest backend/tests/ --ignore=backend/tests/emulator`: 6691
passed, 40 skipped
- [x] Local CodeRabbit review completed (2 cycles): the critical timeout
finding fixed; the remaining minor (results flush inside `finally`) is a
known deferred item

@coderabbitai ignore

🤖 Generated with [Claude Code](https://claude.com/claude-code)

https://claude.ai/code/session_0188txBF47GpTo7hym1FoifX

---------

Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-22 18:42:35 -04:00

223 lines
8.7 KiB
Python

"""
Voice-pool separate-only GPU worker (Cloud Run Job ``voice-pool-separate-job``).
Reads a JSONL manifest (written by karaoke-decide) of original-audio objects in
the gen bucket, separates the vocals stem with the ``instrumental_clean``
ensemble preset and writes ``voice-pool/stems/<id>/vocals.flac`` plus per-task
results. It touches no Firestore and no gen job state.
Manifest line: {"id": pool_id, "audio": "gs://<bucket>/voice-pool/audio/<id>/original.flac"}
Results: <manifest without .jsonl>.results/part-<task_index>.jsonl, one line per
song: {"id", "ok", "vocals", "seconds", "error"}
Cloud Run shards the manifest across tasks (CLOUD_RUN_TASK_INDEX / CLOUD_RUN_TASK_COUNT).
Usage: python -m backend.workers.voice_pool_separate_worker --manifest gs://.../<run>.jsonl
"""
import argparse
import json
import logging
import os
import shutil
import sys
import tempfile
import time
from typing import Any, Callable, Dict, List, Optional
logger = logging.getLogger(__name__)
INSTRUMENTAL_PRESET = "instrumental_clean"
STEMS_PREFIX = "voice-pool/stems"
RESULTS_FLUSH_EVERY = 10
class SystemicTaskError(RuntimeError):
"""Failure that affects every song: the task fails fast with one ERROR."""
class SeparatorLoadError(SystemicTaskError):
"""Model load failed: systemic, so the task fails fast instead of retrying per song."""
class WrongBucketError(ValueError):
"""A gs:// URI points outside the gen bucket."""
def shard(lines: List[dict], index: int, count: int) -> List[dict]:
"""Lines owned by task ``index`` of ``count`` (round-robin over manifest order)."""
return [line for i, line in enumerate(lines) if i % count == index]
def vocals_path_from_outputs(paths: List[str]) -> Optional[str]:
"""Pick the vocals stem from ensemble outputs.
Filenames look like ``{base}_(Vocals)_preset_instrumental_clean.flac``; match the
stem tag, never the preset name (which itself contains "instrumental").
"""
for p in paths:
if "_(vocals)_" in os.path.basename(p).lower():
return p
return None
def _blob_path(uri: str, bucket: Optional[str] = None) -> str:
"""``gs://bucket/path`` -> ``path`` (StorageService is bound to the gen bucket).
Raises WrongBucketError when ``bucket`` is given and the URI names another bucket.
"""
if uri.startswith("gs://"):
uri_bucket, _, path = uri[5:].partition("/")
if bucket is not None and uri_bucket != bucket:
raise WrongBucketError("wrong_bucket")
return path
return uri
def _results_path(manifest_uri: str, index: int) -> str:
base = _blob_path(manifest_uri)
if base.endswith(".jsonl"):
base = base[: -len(".jsonl")]
return f"{base}.results/part-{index}.jsonl"
def _write_results(storage, manifest_uri: str, index: int, results: List[dict]) -> None:
body = "".join(json.dumps(r) + "\n" for r in results)
storage.upload_text(_results_path(manifest_uri, index), body, content_type="application/x-ndjson")
def _read_manifest(storage, manifest_uri: str, bucket: Optional[str] = None) -> List[dict]:
raw = storage.download_bytes(_blob_path(manifest_uri, bucket)).decode("utf-8")
return [json.loads(line) for line in raw.splitlines() if line.strip()]
def run_manifest(
storage,
separator_factory: Callable[[], Any],
manifest_uri: str,
index: int,
count: int,
bucket: Optional[str] = None,
) -> List[dict]:
"""Separate this task's share of the manifest; never aborts on a per-song failure."""
if bucket is None:
from backend.config import get_settings
bucket = get_settings().gcs_bucket_name
try:
manifest = _read_manifest(storage, manifest_uri, bucket)
except WrongBucketError:
raise SystemicTaskError(f"manifest bucket is not {bucket} (wrong_bucket)")
lines = shard(manifest, index, count)
logger.info(f"Task {index}/{count}: {len(lines)} songs")
results: List[dict] = []
separator = None # built lazily so an all-skipped shard never loads models
for n, line in enumerate(lines, 1):
pool_id = line.get("id")
vocals_blob = f"{STEMS_PREFIX}/{pool_id}/vocals.flac"
started = time.time()
tmp = tempfile.mkdtemp(prefix="voicepool_")
try:
if storage.file_exists(vocals_blob):
results.append({"id": pool_id, "ok": True, "vocals": vocals_blob, "seconds": 0, "error": None})
continue
if separator is None:
try:
separator = separator_factory()
except Exception as e:
raise SeparatorLoadError(f"separator failed to load: {e}") from e
local = os.path.join(tmp, "original.flac")
storage.download_file(_blob_path(line["audio"], bucket), local)
outputs = separator.separate(local)
# Separator may return bare filenames relative to its output_dir
out_dir = getattr(separator, "output_dir", None)
outputs = [o if os.path.isabs(o) or not out_dir else os.path.join(out_dir, o) for o in outputs]
vocals = vocals_path_from_outputs(outputs)
if not vocals or not os.path.exists(vocals):
raise RuntimeError(f"no vocals stem in separator outputs: {[os.path.basename(o) for o in outputs]}")
storage.upload_file(vocals, vocals_blob)
for o in outputs: # all stems live in the shared output dir; free it per song
try:
os.remove(o)
except OSError:
pass
results.append({
"id": pool_id, "ok": True, "vocals": vocals_blob,
"seconds": round(time.time() - started, 1), "error": None,
})
except SystemicTaskError:
_write_results(storage, manifest_uri, index, results)
raise
except Exception as e: # per-song failure must not abort the task
logger.warning(f"voice-pool separate failed for {pool_id}: {e}")
results.append({
"id": pool_id, "ok": False, "vocals": None,
"seconds": round(time.time() - started, 1), "error": str(e)[:500],
})
finally:
shutil.rmtree(tmp, ignore_errors=True)
if n % RESULTS_FLUSH_EVERY == 0:
try: # periodic flush is best-effort; the final write stays strict
_write_results(storage, manifest_uri, index, results)
except Exception as e:
logger.warning(f"periodic results flush failed (will retry): {e}")
_write_results(storage, manifest_uri, index, results)
return results
def make_separator_factory(output_dir: str) -> Callable[[], Any]:
def factory():
from audio_separator.separator import Separator
from karaoke_gen.audio_processor import _separator_kwargs
separator = Separator(
model_file_dir=os.environ["MODEL_DIR"],
output_format="FLAC",
output_dir=output_dir,
ensemble_preset=INSTRUMENTAL_PRESET,
**_separator_kwargs(),
)
separator.load_model()
return separator
return factory
def main(argv: Optional[List[str]] = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--manifest", required=True, help="gs://bucket/voice-pool/manifests/<run>.jsonl")
args = parser.parse_args(argv)
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
index = int(os.environ.get("CLOUD_RUN_TASK_INDEX", "0"))
count = int(os.environ.get("CLOUD_RUN_TASK_COUNT", "1"))
from backend.services.storage_service import StorageService
out_dir = tempfile.mkdtemp(prefix="voicepool_out_")
try:
results = run_manifest(StorageService(), make_separator_factory(out_dir), args.manifest, index, count)
except SystemicTaskError as e:
# task-independent text so every failing task groups as one alert pattern
logger.info(f"Task {index}/{count} aborting")
logger.error(f"voice-pool separate task aborted: {e}", extra={"task_index": index})
return 1
finally:
shutil.rmtree(out_dir, ignore_errors=True)
failed = sum(1 for r in results if not r["ok"])
if results and failed == len(results):
logger.info(f"Task {index}/{count}: all {failed} songs failed")
logger.error(
f"voice-pool separate task failed: every song failed (first error: {results[0]['error']})",
extra={"task_index": index, "failed": failed},
)
return 1
logger.info(f"Task {index}/{count} done: {len(results) - failed} ok, {failed} failed")
return 0
if __name__ == "__main__":
sys.exit(main())