From 286e604b7571feba9e161ef856db2e2ee00d57d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrew=20Beveridge=20=E2=98=84=EF=B8=8F?= Date: Thu, 22 Oct 2026 18:42:35 -0400 Subject: [PATCH] feat(worker): voice-pool separate-only GPU job (v0.283.1) (#1217) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Adds `voice-pool-separate-job`, a Cloud Run GPU (L4) job for karaoke-decide's voice session. decide writes a JSONL manifest (`voice-pool/manifests/.jsonl`) and starts the job with a per-run shard; each task separates its share of songs with the `instrumental_clean` Separator and writes `voice-pool/stems//vocals.flac` (skipped if already present). Results are written back to the manifest area every 10 songs. - No Firestore or gen job state. Model-load failure or an all-songs-failed task exits non-zero with a single ERROR; partial failures exit 0. - Task timeout is 3600s (Cloud Run's GPU maximum, found by the local CodeRabbit review), so decide must size the shard count to keep each task under 1h. Re-runs skip finished songs. ## Changes - `backend/workers/voice_pool_separate_worker.py` + `backend/tests/test_voice_pool_separate_worker.py` (9 tests, 91% coverage of the worker) - `infrastructure/modules/cloud_run.py`, `infrastructure/__main__.py`: Pulumi job definition - `.github/workflows/ci.yml`: CI guard step (tolerates a missing job) - `docs/ARCHITECTURE.md`: paragraph on the job - Version 0.283.0 -> 0.283.1 ## Deploy notes **Andrew: run `pulumi up` in gen infrastructure/ BEFORE merging (creates voice-pool-separate-job; the CI guard tolerates a missing job).** decide's Pulumi (later PR) will create the voice-pool SA and grant it `roles/run.jobsExecutorWithOverrides` on this job. ## Testing - [x] `pytest backend/tests/ --ignore=backend/tests/emulator`: 6691 passed, 40 skipped - [x] Local CodeRabbit review completed (2 cycles): the critical timeout finding fixed; the remaining minor (results flush inside `finally`) is a known deferred item @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_0188txBF47GpTo7hym1FoifX --------- Co-authored-by: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 14 ++ .../tests/test_voice_pool_separate_worker.py | 209 +++++++++++++++++ backend/workers/voice_pool_separate_worker.py | 222 ++++++++++++++++++ docs/ARCHITECTURE.md | 10 + infrastructure/__main__.py | 4 + infrastructure/modules/cloud_run.py | 56 +++++ pyproject.toml | 2 +- 7 files changed, 516 insertions(+), 1 deletion(-) create mode 100644 backend/tests/test_voice_pool_separate_worker.py create mode 100644 backend/workers/voice_pool_separate_worker.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 634eb455..036b42d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2280,6 +2280,20 @@ jobs: --project ${{ env.PROJECT_ID }} \ --quiet + # Voice-pool separate-only job (GPU, us-east4) — same GPU image. Created by + # Pulumi; tolerate it not existing yet (pre `pulumi up`). + if gcloud run jobs describe voice-pool-separate-job \ + --region us-east4 \ + --project ${{ env.PROJECT_ID }} >/dev/null 2>&1; then + gcloud run jobs update voice-pool-separate-job \ + --image "$GPU_IMAGE" \ + --region us-east4 \ + --project ${{ env.PROJECT_ID }} \ + --quiet + else + echo "::warning::voice-pool-separate-job not found in us-east4 (run pulumi up); skipping image update" + fi + echo "✅ Cloud Run Jobs updated" - name: Verify deployment and version diff --git a/backend/tests/test_voice_pool_separate_worker.py b/backend/tests/test_voice_pool_separate_worker.py new file mode 100644 index 00000000..8952c52c --- /dev/null +++ b/backend/tests/test_voice_pool_separate_worker.py @@ -0,0 +1,209 @@ +"""Tests for the voice-pool separate-only worker (synthetic data only).""" +import json +import os + +from backend.workers import voice_pool_separate_worker as w + +BUCKET = "karaoke-gen-storage-nomadkaraoke" +MANIFEST = "gs://karaoke-gen-storage-nomadkaraoke/voice-pool/manifests/run1.jsonl" + + +class FakeStorage: + def __init__(self, ids, existing=()): + lines = "".join( + json.dumps({"id": i, "audio": f"gs://{BUCKET}/voice-pool/audio/{i}/original.flac"}) + "\n" for i in ids + ) + self.blobs = {"voice-pool/manifests/run1.jsonl": lines.encode()} + for i in existing: + self.blobs[f"voice-pool/stems/{i}/vocals.flac"] = b"x" + self.downloads = [] + + def download_bytes(self, p): + return self.blobs[p] + + def file_exists(self, p): + return p in self.blobs + + def download_file(self, src, dest): + self.downloads.append(src) + open(dest, "wb").write(b"audio") + return dest + + def upload_file(self, local, dest): + self.blobs[dest] = open(local, "rb").read() + return dest + + def upload_text(self, dest, text, content_type="text/plain"): + self.blobs[dest] = text.encode() + return dest + + +class FakeSeparator: + def __init__(self, out_dir, fail_ids=()): + self.out_dir, self.fail_ids, self.calls = out_dir, fail_ids, 0 + + def separate(self, path): + self.calls += 1 + if any(f in str(self.calls) for f in self.fail_ids): + raise RuntimeError("boom") + paths = [] + for tag in ("Vocals", "Instrumental"): + p = os.path.join(self.out_dir, f"{self.calls}_({tag})_preset_instrumental_clean.flac") + open(p, "wb").write(tag.encode()) + paths.append(p) + return paths + + +def _results(storage, idx=0): + raw = storage.blobs[f"voice-pool/manifests/run1.results/part-{idx}.jsonl"].decode() + return [json.loads(x) for x in raw.splitlines()] + + +def test_shard_complete_and_disjoint(): + lines = [{"id": str(i)} for i in range(23)] + parts = [w.shard(lines, i, 4) for i in range(4)] + flat = [x["id"] for p in parts for x in p] + assert sorted(flat) == sorted(x["id"] for x in lines) + assert len(set(flat)) == 23 + assert [x["id"] for x in parts[1]] == ["1", "5", "9", "13", "17", "21"] + + +def test_vocals_path_from_outputs(): + paths = [ + "/o/x_(Instrumental)_preset_instrumental_clean.flac", + "/o/x_(Vocals)_preset_instrumental_clean.flac", + ] + assert w.vocals_path_from_outputs(paths) == paths[1] + assert w.vocals_path_from_outputs(paths[:1]) is None + assert w.vocals_path_from_outputs(["/o/x_(No Vocal)_preset_instrumental_clean.flac"]) is None + + +def test_run_manifest_skips_existing_and_records_failures(tmp_path): + storage = FakeStorage(["a", "b", "c"], existing=["a"]) + sep = FakeSeparator(str(tmp_path), fail_ids=("1",)) # first real separation raises + built = [] + + def factory(): + built.append(1) + return sep + + results = w.run_manifest(storage, factory, MANIFEST, 0, 1, bucket=BUCKET) + by_id = {r["id"]: r for r in results} + assert by_id["a"]["ok"] and by_id["a"]["seconds"] == 0 + assert not by_id["b"]["ok"] and "boom" in by_id["b"]["error"] + assert by_id["c"]["ok"] and by_id["c"]["vocals"] == "voice-pool/stems/c/vocals.flac" + assert storage.blobs["voice-pool/stems/c/vocals.flac"] == b"Vocals" + assert "voice-pool/stems/b/vocals.flac" not in storage.blobs + assert "voice-pool/audio/a/original.flac" not in storage.downloads + assert len(built) == 1 # one separator reused + assert _results(storage) == results + assert os.listdir(tmp_path) == [] # stems cleaned up + + +def test_results_written_every_10(tmp_path, monkeypatch): + ids = [str(i) for i in range(12)] + storage = FakeStorage(ids) + writes = [] + orig = w._write_results + monkeypatch.setattr(w, "_write_results", lambda s, m, i, r: (writes.append(len(r)), orig(s, m, i, r))) + w.run_manifest(storage, lambda: FakeSeparator(str(tmp_path)), MANIFEST, 0, 1, bucket=BUCKET) + assert writes == [10, 12] + + +def test_sharded_task_only_processes_its_share(tmp_path): + storage = FakeStorage([str(i) for i in range(6)]) + w.run_manifest(storage, lambda: FakeSeparator(str(tmp_path)), MANIFEST, 1, 3, bucket=BUCKET) + assert [r["id"] for r in _results(storage, 1)] == ["1", "4"] + + +def test_separator_load_failure_fails_fast(tmp_path): + import pytest + + storage = FakeStorage(["a", "b"]) + calls = [] + + def factory(): + calls.append(1) + raise RuntimeError("no gpu") + + with pytest.raises(w.SeparatorLoadError): + w.run_manifest(storage, factory, MANIFEST, 0, 1, bucket=BUCKET) + assert len(calls) == 1 # no per-song retry of the model load + + +def _patch_main(monkeypatch, run): + import backend.services.storage_service as ss + + monkeypatch.setattr(ss, "StorageService", lambda: object()) + monkeypatch.setattr(w, "run_manifest", run) + + +def test_main_all_failed_logs_one_error_and_exits_nonzero(monkeypatch, caplog): + fail = {"id": "a", "ok": False, "vocals": None, "seconds": 0, "error": "oom"} + _patch_main(monkeypatch, lambda *a, **k: [fail, dict(fail, id="b")]) + assert w.main(["--manifest", MANIFEST]) == 1 + assert len([r for r in caplog.records if r.levelname == "ERROR"]) == 1 + + +def test_main_partial_failure_exits_zero_without_error(monkeypatch, caplog): + ok = {"id": "a", "ok": True, "vocals": "v", "seconds": 1, "error": None} + _patch_main(monkeypatch, lambda *a, **k: [ok, dict(ok, id="b", ok=False, error="x")]) + assert w.main(["--manifest", MANIFEST]) == 0 + assert not [r for r in caplog.records if r.levelname == "ERROR"] + + +def test_main_load_failure_exits_nonzero_with_one_error(monkeypatch, caplog): + def boom(*a, **k): + raise w.SeparatorLoadError("x") + + _patch_main(monkeypatch, boom) + assert w.main(["--manifest", MANIFEST]) == 1 + assert len([r for r in caplog.records if r.levelname == "ERROR"]) == 1 + + +def test_periodic_flush_failure_is_tolerated_final_is_strict(tmp_path, monkeypatch): + import pytest + + storage = FakeStorage([str(i) for i in range(11)]) + orig = w._write_results + n = [] + + def flaky(s, m, i, r): + n.append(len(r)) + if len(n) == 1: + raise OSError("transient gcs") + return orig(s, m, i, r) + + monkeypatch.setattr(w, "_write_results", flaky) + results = w.run_manifest(storage, lambda: FakeSeparator(str(tmp_path)), MANIFEST, 0, 1, bucket=BUCKET) + assert len(results) == 11 and n == [10, 11] + + monkeypatch.setattr(w, "_write_results", lambda *a: (_ for _ in ()).throw(OSError("down"))) + with pytest.raises(OSError): + w.run_manifest(FakeStorage(["a"]), lambda: FakeSeparator(str(tmp_path)), MANIFEST, 0, 1, bucket=BUCKET) + + +def test_wrong_bucket_song_recorded_and_manifest_fails(tmp_path): + import pytest + + storage = FakeStorage(["a", "b"]) + storage.blobs["voice-pool/manifests/run1.jsonl"] = ( + json.dumps({"id": "a", "audio": "gs://other-bucket/voice-pool/audio/a/original.flac"}) + "\n" + + json.dumps({"id": "b", "audio": f"gs://{BUCKET}/voice-pool/audio/b/original.flac"}) + "\n" + ).encode() + results = w.run_manifest(storage, lambda: FakeSeparator(str(tmp_path)), MANIFEST, 0, 1, bucket=BUCKET) + assert results[0]["ok"] is False and results[0]["error"] == "wrong_bucket" + assert results[1]["ok"] is True + + with pytest.raises(w.SystemicTaskError): + w.run_manifest(storage, lambda: FakeSeparator(str(tmp_path)), + "gs://other-bucket/voice-pool/manifests/run1.jsonl", 0, 1, bucket=BUCKET) + + +def test_systemic_error_messages_are_task_independent(monkeypatch, caplog): + fail = {"id": "a", "ok": False, "vocals": None, "seconds": 0, "error": "oom"} + _patch_main(monkeypatch, lambda *a, **k: [fail]) + monkeypatch.setenv("CLOUD_RUN_TASK_INDEX", "3") + w.main(["--manifest", MANIFEST]) + errs = [r for r in caplog.records if r.levelname == "ERROR"] + assert len(errs) == 1 and "3" not in errs[0].getMessage().split("(first")[0] diff --git a/backend/workers/voice_pool_separate_worker.py b/backend/workers/voice_pool_separate_worker.py new file mode 100644 index 00000000..8e69182b --- /dev/null +++ b/backend/workers/voice_pool_separate_worker.py @@ -0,0 +1,222 @@ +""" +Voice-pool separate-only GPU worker (Cloud Run Job ``voice-pool-separate-job``). + +Reads a JSONL manifest (written by karaoke-decide) of original-audio objects in +the gen bucket, separates the vocals stem with the ``instrumental_clean`` +ensemble preset and writes ``voice-pool/stems//vocals.flac`` plus per-task +results. It touches no Firestore and no gen job state. + +Manifest line: {"id": pool_id, "audio": "gs:///voice-pool/audio//original.flac"} +Results: .results/part-.jsonl, one line per + song: {"id", "ok", "vocals", "seconds", "error"} + +Cloud Run shards the manifest across tasks (CLOUD_RUN_TASK_INDEX / CLOUD_RUN_TASK_COUNT). + +Usage: python -m backend.workers.voice_pool_separate_worker --manifest gs://.../.jsonl +""" +import argparse +import json +import logging +import os +import shutil +import sys +import tempfile +import time +from typing import Any, Callable, Dict, List, Optional + +logger = logging.getLogger(__name__) + +INSTRUMENTAL_PRESET = "instrumental_clean" +STEMS_PREFIX = "voice-pool/stems" +RESULTS_FLUSH_EVERY = 10 + + +class SystemicTaskError(RuntimeError): + """Failure that affects every song: the task fails fast with one ERROR.""" + + +class SeparatorLoadError(SystemicTaskError): + """Model load failed: systemic, so the task fails fast instead of retrying per song.""" + + +class WrongBucketError(ValueError): + """A gs:// URI points outside the gen bucket.""" + + +def shard(lines: List[dict], index: int, count: int) -> List[dict]: + """Lines owned by task ``index`` of ``count`` (round-robin over manifest order).""" + return [line for i, line in enumerate(lines) if i % count == index] + + +def vocals_path_from_outputs(paths: List[str]) -> Optional[str]: + """Pick the vocals stem from ensemble outputs. + + Filenames look like ``{base}_(Vocals)_preset_instrumental_clean.flac``; match the + stem tag, never the preset name (which itself contains "instrumental"). + """ + for p in paths: + if "_(vocals)_" in os.path.basename(p).lower(): + return p + return None + + +def _blob_path(uri: str, bucket: Optional[str] = None) -> str: + """``gs://bucket/path`` -> ``path`` (StorageService is bound to the gen bucket). + + Raises WrongBucketError when ``bucket`` is given and the URI names another bucket. + """ + if uri.startswith("gs://"): + uri_bucket, _, path = uri[5:].partition("/") + if bucket is not None and uri_bucket != bucket: + raise WrongBucketError("wrong_bucket") + return path + return uri + + +def _results_path(manifest_uri: str, index: int) -> str: + base = _blob_path(manifest_uri) + if base.endswith(".jsonl"): + base = base[: -len(".jsonl")] + return f"{base}.results/part-{index}.jsonl" + + +def _write_results(storage, manifest_uri: str, index: int, results: List[dict]) -> None: + body = "".join(json.dumps(r) + "\n" for r in results) + storage.upload_text(_results_path(manifest_uri, index), body, content_type="application/x-ndjson") + + +def _read_manifest(storage, manifest_uri: str, bucket: Optional[str] = None) -> List[dict]: + raw = storage.download_bytes(_blob_path(manifest_uri, bucket)).decode("utf-8") + return [json.loads(line) for line in raw.splitlines() if line.strip()] + + +def run_manifest( + storage, + separator_factory: Callable[[], Any], + manifest_uri: str, + index: int, + count: int, + bucket: Optional[str] = None, +) -> List[dict]: + """Separate this task's share of the manifest; never aborts on a per-song failure.""" + if bucket is None: + from backend.config import get_settings + + bucket = get_settings().gcs_bucket_name + try: + manifest = _read_manifest(storage, manifest_uri, bucket) + except WrongBucketError: + raise SystemicTaskError(f"manifest bucket is not {bucket} (wrong_bucket)") + lines = shard(manifest, index, count) + logger.info(f"Task {index}/{count}: {len(lines)} songs") + results: List[dict] = [] + separator = None # built lazily so an all-skipped shard never loads models + + for n, line in enumerate(lines, 1): + pool_id = line.get("id") + vocals_blob = f"{STEMS_PREFIX}/{pool_id}/vocals.flac" + started = time.time() + tmp = tempfile.mkdtemp(prefix="voicepool_") + try: + if storage.file_exists(vocals_blob): + results.append({"id": pool_id, "ok": True, "vocals": vocals_blob, "seconds": 0, "error": None}) + continue + if separator is None: + try: + separator = separator_factory() + except Exception as e: + raise SeparatorLoadError(f"separator failed to load: {e}") from e + local = os.path.join(tmp, "original.flac") + storage.download_file(_blob_path(line["audio"], bucket), local) + outputs = separator.separate(local) + # Separator may return bare filenames relative to its output_dir + out_dir = getattr(separator, "output_dir", None) + outputs = [o if os.path.isabs(o) or not out_dir else os.path.join(out_dir, o) for o in outputs] + vocals = vocals_path_from_outputs(outputs) + if not vocals or not os.path.exists(vocals): + raise RuntimeError(f"no vocals stem in separator outputs: {[os.path.basename(o) for o in outputs]}") + storage.upload_file(vocals, vocals_blob) + for o in outputs: # all stems live in the shared output dir; free it per song + try: + os.remove(o) + except OSError: + pass + results.append({ + "id": pool_id, "ok": True, "vocals": vocals_blob, + "seconds": round(time.time() - started, 1), "error": None, + }) + except SystemicTaskError: + _write_results(storage, manifest_uri, index, results) + raise + except Exception as e: # per-song failure must not abort the task + logger.warning(f"voice-pool separate failed for {pool_id}: {e}") + results.append({ + "id": pool_id, "ok": False, "vocals": None, + "seconds": round(time.time() - started, 1), "error": str(e)[:500], + }) + finally: + shutil.rmtree(tmp, ignore_errors=True) + if n % RESULTS_FLUSH_EVERY == 0: + try: # periodic flush is best-effort; the final write stays strict + _write_results(storage, manifest_uri, index, results) + except Exception as e: + logger.warning(f"periodic results flush failed (will retry): {e}") + + _write_results(storage, manifest_uri, index, results) + return results + + +def make_separator_factory(output_dir: str) -> Callable[[], Any]: + def factory(): + from audio_separator.separator import Separator + + from karaoke_gen.audio_processor import _separator_kwargs + + separator = Separator( + model_file_dir=os.environ["MODEL_DIR"], + output_format="FLAC", + output_dir=output_dir, + ensemble_preset=INSTRUMENTAL_PRESET, + **_separator_kwargs(), + ) + separator.load_model() + return separator + + return factory + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", required=True, help="gs://bucket/voice-pool/manifests/.jsonl") + args = parser.parse_args(argv) + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") + + index = int(os.environ.get("CLOUD_RUN_TASK_INDEX", "0")) + count = int(os.environ.get("CLOUD_RUN_TASK_COUNT", "1")) + + from backend.services.storage_service import StorageService + + out_dir = tempfile.mkdtemp(prefix="voicepool_out_") + try: + results = run_manifest(StorageService(), make_separator_factory(out_dir), args.manifest, index, count) + except SystemicTaskError as e: + # task-independent text so every failing task groups as one alert pattern + logger.info(f"Task {index}/{count} aborting") + logger.error(f"voice-pool separate task aborted: {e}", extra={"task_index": index}) + return 1 + finally: + shutil.rmtree(out_dir, ignore_errors=True) + failed = sum(1 for r in results if not r["ok"]) + if results and failed == len(results): + logger.info(f"Task {index}/{count}: all {failed} songs failed") + logger.error( + f"voice-pool separate task failed: every song failed (first error: {results[0]['error']})", + extra={"task_index": index, "failed": failed}, + ) + return 1 + logger.info(f"Task {index}/{count} done: {len(results) - failed} ok, {failed} failed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 4db77942..de4938e7 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -749,6 +749,16 @@ is still defined in Pulumi behind the `audioSeparatorServiceEnabled` flag — se [GCP-COST-OPTIMIZATION.md](GCP-COST-OPTIMIZATION.md#audio-separator-service-shut-down-on-demand-redeploy) to bring it back on demand. +### Voice-pool separate-only job + +`voice-pool-separate-job` (Cloud Run Job, L4 GPU, us-east4, same GPU image) serves karaoke-decide's +voice session. decide writes a JSONL manifest (`voice-pool/manifests/.jsonl`, lines +`{"id", "audio": "gs://…/voice-pool/audio//original.flac"}`) and starts the job with a per-run +`task_count`. Each task (`CLOUD_RUN_TASK_INDEX`/`COUNT`, round-robin over manifest order) loads one +`instrumental_clean` Separator, writes `voice-pool/stems//vocals.flac` (skipped if present) and +`.results/part-.jsonl` (`{id, ok, vocals, seconds, error}`, flushed every 10 songs). +No Firestore or gen job state. gen's Pulumi only creates the job; decide's Pulumi creates the `voice-pool` SA and grants it `roles/run.jobsExecutorWithOverrides` on it. The job has `parallelism=1` (decide cannot override it), so tasks run one after another and a pool run never uses more than one L4, leaving GPU capacity to customers' `audio-separation-job`. Task timeout is 3600s (Cloud Run's GPU maximum), so each task must finish in under 1h: decide chunks manifests accordingly and treats a timed-out execution as "re-run" (re-runs skip finished songs). URIs outside the gen bucket fail as `wrong_bucket`. A model-load failure or an all-songs-failed task exits non-zero with one ERROR. Code: `backend/workers/voice_pool_separate_worker.py`. + ### Quick version (kjbox make-it jobs) For jobs created by kjbox on behalf of a singer (`request_metadata.client_id` starts with diff --git a/infrastructure/__main__.py b/infrastructure/__main__.py index fab77675..a7fb341b 100644 --- a/infrastructure/__main__.py +++ b/infrastructure/__main__.py @@ -187,6 +187,9 @@ edge_resources = edge_security.configure_edge_security() # These use Cloud Run Jobs instead of Cloud Tasks to avoid instance termination # during long-running processing (the BackgroundTasks issue) audio_separation_job = cloud_run.create_audio_separation_job(bucket, backend_service_account) +# Voice-pool separate-only GPU job (started by karaoke-decide's fetch job). decide's +# Pulumi creates the voice-pool SA and grants it run access on this job. +voice_pool_separate_job = cloud_run.create_voice_pool_separate_job(bucket, backend_service_account) # Latency-critical CPU jobs run in CPU_JOBS_REGION (us-east4): us-central1 Cloud # Run Jobs queue 2-5 min before starting. The legacy us-central1 copies were # removed after the us-east4 jobs were verified in prod (2026-09-30). To roll @@ -828,6 +831,7 @@ pulumi.export("video_worker_queue", queues["video-worker-queue"].name) pulumi.export("video_encoding_job", video_encoding_job.name) pulumi.export("lyrics_transcription_job", lyrics_transcription_job.name) pulumi.export("audio_separation_job", audio_separation_job.name) +pulumi.export("voice_pool_separate_job", voice_pool_separate_job.name) pulumi.export("audio_download_job", audio_download_job.name) pulumi.export("bulk_search_job", bulk_search_job.name) pulumi.export("backend_url", "https://api.nomadkaraoke.com") diff --git a/infrastructure/modules/cloud_run.py b/infrastructure/modules/cloud_run.py index a405a348..6f804294 100644 --- a/infrastructure/modules/cloud_run.py +++ b/infrastructure/modules/cloud_run.py @@ -329,6 +329,62 @@ def create_audio_separation_job( return audio_separation_job +def create_voice_pool_separate_job( + bucket: gcp.storage.Bucket, + service_account: gcp.serviceaccount.Account, +) -> cloudrunv2.Job: + """ + Create the Cloud Run Job that separates vocals for the karaoke-decide voice pool. + + Reads a JSONL manifest of original-audio objects (voice-pool/ prefix), runs the + instrumental_clean ensemble on an L4 GPU and writes vocals stems + per-task + results back to GCS. No Firestore and no gen job state. decide overrides + task_count per run; one Separator is loaded per task and reused per song. + + Args: + bucket: The GCS bucket holding voice-pool/ objects. + service_account: The service account to run the job (audio worker SA). + """ + return cloudrunv2.Job( + "voice-pool-separate-job", + name="voice-pool-separate-job", + location=AUDIO_WORKER_GPU_REGION, + deletion_protection=False, + template=cloudrunv2.JobTemplateArgs( + task_count=1, # overridden per execution by the decide fetch job + parallelism=1, # one L4 at a time so customer audio-separation-job keeps GPU capacity + template=cloudrunv2.JobTemplateTemplateArgs( + containers=[ + cloudrunv2.JobTemplateTemplateContainerArgs( + image=f"{AUDIO_WORKER_GPU_REGION}-docker.pkg.dev/{PROJECT_ID}/karaoke-backend-gpu/karaoke-backend:latest", + args=["python", "-m", "backend.workers.voice_pool_separate_worker"], + resources=cloudrunv2.JobTemplateTemplateContainerResourcesArgs( + limits={ + "cpu": "4", + "memory": "16Gi", + "nvidia.com/gpu": "1", + }, + ), + envs=[ + cloudrunv2.JobTemplateTemplateContainerEnvArgs(name="MODEL_DIR", value="/models"), + cloudrunv2.JobTemplateTemplateContainerEnvArgs(name="ENVIRONMENT", value="production"), + cloudrunv2.JobTemplateTemplateContainerEnvArgs(name="GCS_BUCKET_NAME", value=bucket.name), + cloudrunv2.JobTemplateTemplateContainerEnvArgs(name="GOOGLE_CLOUD_PROJECT", value=PROJECT_ID), + ], + ) + ], + node_selector=cloudrunv2.JobTemplateTemplateNodeSelectorArgs( + accelerator="nvidia-l4", + ), + gpu_zonal_redundancy_disabled=True, # Required for GPU Jobs (capacity limitation) + service_account=service_account.email, + timeout="3600s", # Cloud Run caps GPU task timeouts at 1h; size shards (task count) so each finishes in <1h + max_retries=0, # decide re-runs; finished songs are skipped via existing stems + ), + ), + ) + + def create_audio_download_job( bucket: gcp.storage.Bucket, service_account: gcp.serviceaccount.Account, diff --git a/pyproject.toml b/pyproject.toml index 4de38e1c..d786a268 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.283.0" +version = "0.283.1" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"