feat(worker): voice-pool separate-only GPU job (v0.283.1) (#1217)

## Summary
- Adds `voice-pool-separate-job`, a Cloud Run GPU (L4) job for
karaoke-decide's voice session. decide writes a JSONL manifest
(`voice-pool/manifests/<run>.jsonl`) and starts the job with a per-run
shard; each task separates its share of songs with the
`instrumental_clean` Separator and writes
`voice-pool/stems/<id>/vocals.flac` (skipped if already present).
Results are written back to the manifest area every 10 songs.
- No Firestore or gen job state. Model-load failure or an
all-songs-failed task exits non-zero with a single ERROR; partial
failures exit 0.
- Task timeout is 3600s (Cloud Run's GPU maximum, found by the local
CodeRabbit review), so decide must size the shard count to keep each
task under 1h. Re-runs skip finished songs.

## Changes
- `backend/workers/voice_pool_separate_worker.py` +
`backend/tests/test_voice_pool_separate_worker.py` (9 tests, 91%
coverage of the worker)
- `infrastructure/modules/cloud_run.py`, `infrastructure/__main__.py`:
Pulumi job definition
- `.github/workflows/ci.yml`: CI guard step (tolerates a missing job)
- `docs/ARCHITECTURE.md`: paragraph on the job
- Version 0.283.0 -> 0.283.1

## Deploy notes
**Andrew: run `pulumi up` in gen infrastructure/ BEFORE merging (creates
voice-pool-separate-job; the CI guard tolerates a missing job).**
decide's Pulumi (later PR) will create the voice-pool SA and grant it
`roles/run.jobsExecutorWithOverrides` on this job.

## Testing
- [x] `pytest backend/tests/ --ignore=backend/tests/emulator`: 6691
passed, 40 skipped
- [x] Local CodeRabbit review completed (2 cycles): the critical timeout
finding fixed; the remaining minor (results flush inside `finally`) is a
known deferred item

@coderabbitai ignore

🤖 Generated with [Claude Code](https://claude.com/claude-code)

https://claude.ai/code/session_0188txBF47GpTo7hym1FoifX

---------

Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Andrew Beveridge ☄️andClaude Sonnet 5.5 authored and GitHub committed 2026-10-22 18:42:35 -04:00
1 parent c4c4f9265c
commit 286e604b75
7 files changed
+516 -1

No files matched your search

+14
View File
@@ -2280,6 +2280,20 @@ jobs:
--project ${{ env.PROJECT_ID }} \
--quiet
# Voice-pool separate-only job (GPU, us-east4) — same GPU image. Created by
# Pulumi; tolerate it not existing yet (pre `pulumi up`).
if gcloud run jobs describe voice-pool-separate-job \
--region us-east4 \
--project ${{ env.PROJECT_ID }} >/dev/null 2>&1; then
gcloud run jobs update voice-pool-separate-job \
--image "$GPU_IMAGE" \
--region us-east4 \
--project ${{ env.PROJECT_ID }} \
--quiet
else
echo "::warning::voice-pool-separate-job not found in us-east4 (run pulumi up); skipping image update"
fi
echo "✅ Cloud Run Jobs updated"
- name: Verify deployment and version
@@ -0,0 +1,209 @@
"""Tests for the voice-pool separate-only worker (synthetic data only)."""
import json
import os
from backend.workers import voice_pool_separate_worker as w
BUCKET = "karaoke-gen-storage-nomadkaraoke"
MANIFEST = "gs://karaoke-gen-storage-nomadkaraoke/voice-pool/manifests/run1.jsonl"
class FakeStorage:
def __init__(self, ids, existing=()):
lines = "".join(
json.dumps({"id": i, "audio": f"gs://{BUCKET}/voice-pool/audio/{i}/original.flac"}) + "\n" for i in ids
)
self.blobs = {"voice-pool/manifests/run1.jsonl": lines.encode()}
for i in existing:
self.blobs[f"voice-pool/stems/{i}/vocals.flac"] = b"x"
self.downloads = []
def download_bytes(self, p):
return self.blobs[p]
def file_exists(self, p):
return p in self.blobs
def download_file(self, src, dest):
self.downloads.append(src)
open(dest, "wb").write(b"audio")
return dest
def upload_file(self, local, dest):
self.blobs[dest] = open(local, "rb").read()
return dest
def upload_text(self, dest, text, content_type="text/plain"):
self.blobs[dest] = text.encode()
return dest
class FakeSeparator:
def __init__(self, out_dir, fail_ids=()):
self.out_dir, self.fail_ids, self.calls = out_dir, fail_ids, 0
def separate(self, path):
self.calls += 1
if any(f in str(self.calls) for f in self.fail_ids):
raise RuntimeError("boom")
paths = []
for tag in ("Vocals", "Instrumental"):
p = os.path.join(self.out_dir, f"{self.calls}_({tag})_preset_instrumental_clean.flac")
open(p, "wb").write(tag.encode())
paths.append(p)
return paths
def _results(storage, idx=0):
raw = storage.blobs[f"voice-pool/manifests/run1.results/part-{idx}.jsonl"].decode()
return [json.loads(x) for x in raw.splitlines()]
def test_shard_complete_and_disjoint():
lines = [{"id": str(i)} for i in range(23)]
parts = [w.shard(lines, i, 4) for i in range(4)]
flat = [x["id"] for p in parts for x in p]
assert sorted(flat) == sorted(x["id"] for x in lines)
assert len(set(flat)) == 23
assert [x["id"] for x in parts[1]] == ["1", "5", "9", "13", "17", "21"]
def test_vocals_path_from_outputs():
paths = [
"/o/x_(Instrumental)_preset_instrumental_clean.flac",
"/o/x_(Vocals)_preset_instrumental_clean.flac",
]
assert w.vocals_path_from_outputs(paths) == paths[1]
assert w.vocals_path_from_outputs(paths[:1]) is None
assert w.vocals_path_from_outputs(["/o/x_(No Vocal)_preset_instrumental_clean.flac"]) is None
def test_run_manifest_skips_existing_and_records_failures(tmp_path):
storage = FakeStorage(["a", "b", "c"], existing=["a"])
sep = FakeSeparator(str(tmp_path), fail_ids=("1",)) # first real separation raises
built = []
def factory():
built.append(1)
return sep
results = w.run_manifest(storage, factory, MANIFEST, 0, 1, bucket=BUCKET)
by_id = {r["id"]: r for r in results}
assert by_id["a"]["ok"] and by_id["a"]["seconds"] == 0
assert not by_id["b"]["ok"] and "boom" in by_id["b"]["error"]
assert by_id["c"]["ok"] and by_id["c"]["vocals"] == "voice-pool/stems/c/vocals.flac"
assert storage.blobs["voice-pool/stems/c/vocals.flac"] == b"Vocals"
assert "voice-pool/stems/b/vocals.flac" not in storage.blobs
assert "voice-pool/audio/a/original.flac" not in storage.downloads
assert len(built) == 1 # one separator reused
assert _results(storage) == results
assert os.listdir(tmp_path) == [] # stems cleaned up
def test_results_written_every_10(tmp_path, monkeypatch):
ids = [str(i) for i in range(12)]
storage = FakeStorage(ids)
writes = []
orig = w._write_results
monkeypatch.setattr(w, "_write_results", lambda s, m, i, r: (writes.append(len(r)), orig(s, m, i, r)))
w.run_manifest(storage, lambda: FakeSeparator(str(tmp_path)), MANIFEST, 0, 1, bucket=BUCKET)
assert writes == [10, 12]
def test_sharded_task_only_processes_its_share(tmp_path):
storage = FakeStorage([str(i) for i in range(6)])
w.run_manifest(storage, lambda: FakeSeparator(str(tmp_path)), MANIFEST, 1, 3, bucket=BUCKET)
assert [r["id"] for r in _results(storage, 1)] == ["1", "4"]
def test_separator_load_failure_fails_fast(tmp_path):
import pytest
storage = FakeStorage(["a", "b"])
calls = []
def factory():
calls.append(1)
raise RuntimeError("no gpu")
with pytest.raises(w.SeparatorLoadError):
w.run_manifest(storage, factory, MANIFEST, 0, 1, bucket=BUCKET)
assert len(calls) == 1 # no per-song retry of the model load
def _patch_main(monkeypatch, run):
import backend.services.storage_service as ss
monkeypatch.setattr(ss, "StorageService", lambda: object())
monkeypatch.setattr(w, "run_manifest", run)
def test_main_all_failed_logs_one_error_and_exits_nonzero(monkeypatch, caplog):
fail = {"id": "a", "ok": False, "vocals": None, "seconds": 0, "error": "oom"}
_patch_main(monkeypatch, lambda *a, **k: [fail, dict(fail, id="b")])
assert w.main(["--manifest", MANIFEST]) == 1
assert len([r for r in caplog.records if r.levelname == "ERROR"]) == 1
def test_main_partial_failure_exits_zero_without_error(monkeypatch, caplog):
ok = {"id": "a", "ok": True, "vocals": "v", "seconds": 1, "error": None}
_patch_main(monkeypatch, lambda *a, **k: [ok, dict(ok, id="b", ok=False, error="x")])
assert w.main(["--manifest", MANIFEST]) == 0
assert not [r for r in caplog.records if r.levelname == "ERROR"]
def test_main_load_failure_exits_nonzero_with_one_error(monkeypatch, caplog):
def boom(*a, **k):
raise w.SeparatorLoadError("x")
_patch_main(monkeypatch, boom)
assert w.main(["--manifest", MANIFEST]) == 1
assert len([r for r in caplog.records if r.levelname == "ERROR"]) == 1
def test_periodic_flush_failure_is_tolerated_final_is_strict(tmp_path, monkeypatch):
import pytest
storage = FakeStorage([str(i) for i in range(11)])
orig = w._write_results
n = []
def flaky(s, m, i, r):
n.append(len(r))
if len(n) == 1:
raise OSError("transient gcs")
return orig(s, m, i, r)
monkeypatch.setattr(w, "_write_results", flaky)
results = w.run_manifest(storage, lambda: FakeSeparator(str(tmp_path)), MANIFEST, 0, 1, bucket=BUCKET)
assert len(results) == 11 and n == [10, 11]
monkeypatch.setattr(w, "_write_results", lambda *a: (_ for _ in ()).throw(OSError("down")))
with pytest.raises(OSError):
w.run_manifest(FakeStorage(["a"]), lambda: FakeSeparator(str(tmp_path)), MANIFEST, 0, 1, bucket=BUCKET)
def test_wrong_bucket_song_recorded_and_manifest_fails(tmp_path):
import pytest
storage = FakeStorage(["a", "b"])
storage.blobs["voice-pool/manifests/run1.jsonl"] = (
json.dumps({"id": "a", "audio": "gs://other-bucket/voice-pool/audio/a/original.flac"}) + "\n"
+ json.dumps({"id": "b", "audio": f"gs://{BUCKET}/voice-pool/audio/b/original.flac"}) + "\n"
).encode()
results = w.run_manifest(storage, lambda: FakeSeparator(str(tmp_path)), MANIFEST, 0, 1, bucket=BUCKET)
assert results[0]["ok"] is False and results[0]["error"] == "wrong_bucket"
assert results[1]["ok"] is True
with pytest.raises(w.SystemicTaskError):
w.run_manifest(storage, lambda: FakeSeparator(str(tmp_path)),
"gs://other-bucket/voice-pool/manifests/run1.jsonl", 0, 1, bucket=BUCKET)
def test_systemic_error_messages_are_task_independent(monkeypatch, caplog):
fail = {"id": "a", "ok": False, "vocals": None, "seconds": 0, "error": "oom"}
_patch_main(monkeypatch, lambda *a, **k: [fail])
monkeypatch.setenv("CLOUD_RUN_TASK_INDEX", "3")
w.main(["--manifest", MANIFEST])
errs = [r for r in caplog.records if r.levelname == "ERROR"]
assert len(errs) == 1 and "3" not in errs[0].getMessage().split("(first")[0]
@@ -0,0 +1,222 @@
"""
Voice-pool separate-only GPU worker (Cloud Run Job ``voice-pool-separate-job``).
Reads a JSONL manifest (written by karaoke-decide) of original-audio objects in
the gen bucket, separates the vocals stem with the ``instrumental_clean``
ensemble preset and writes ``voice-pool/stems/<id>/vocals.flac`` plus per-task
results. It touches no Firestore and no gen job state.
Manifest line: {"id": pool_id, "audio": "gs://<bucket>/voice-pool/audio/<id>/original.flac"}
Results: <manifest without .jsonl>.results/part-<task_index>.jsonl, one line per
song: {"id", "ok", "vocals", "seconds", "error"}
Cloud Run shards the manifest across tasks (CLOUD_RUN_TASK_INDEX / CLOUD_RUN_TASK_COUNT).
Usage: python -m backend.workers.voice_pool_separate_worker --manifest gs://.../<run>.jsonl
"""
import argparse
import json
import logging
import os
import shutil
import sys
import tempfile
import time
from typing import Any, Callable, Dict, List, Optional
logger = logging.getLogger(__name__)
INSTRUMENTAL_PRESET = "instrumental_clean"
STEMS_PREFIX = "voice-pool/stems"
RESULTS_FLUSH_EVERY = 10
class SystemicTaskError(RuntimeError):
"""Failure that affects every song: the task fails fast with one ERROR."""
class SeparatorLoadError(SystemicTaskError):
"""Model load failed: systemic, so the task fails fast instead of retrying per song."""
class WrongBucketError(ValueError):
"""A gs:// URI points outside the gen bucket."""
def shard(lines: List[dict], index: int, count: int) -> List[dict]:
"""Lines owned by task ``index`` of ``count`` (round-robin over manifest order)."""
return [line for i, line in enumerate(lines) if i % count == index]
def vocals_path_from_outputs(paths: List[str]) -> Optional[str]:
"""Pick the vocals stem from ensemble outputs.
Filenames look like ``{base}_(Vocals)_preset_instrumental_clean.flac``; match the
stem tag, never the preset name (which itself contains "instrumental").
"""
for p in paths:
if "_(vocals)_" in os.path.basename(p).lower():
return p
return None
def _blob_path(uri: str, bucket: Optional[str] = None) -> str:
"""``gs://bucket/path`` -> ``path`` (StorageService is bound to the gen bucket).
Raises WrongBucketError when ``bucket`` is given and the URI names another bucket.
"""
if uri.startswith("gs://"):
uri_bucket, _, path = uri[5:].partition("/")
if bucket is not None and uri_bucket != bucket:
raise WrongBucketError("wrong_bucket")
return path
return uri
def _results_path(manifest_uri: str, index: int) -> str:
base = _blob_path(manifest_uri)
if base.endswith(".jsonl"):
base = base[: -len(".jsonl")]
return f"{base}.results/part-{index}.jsonl"
def _write_results(storage, manifest_uri: str, index: int, results: List[dict]) -> None:
body = "".join(json.dumps(r) + "\n" for r in results)
storage.upload_text(_results_path(manifest_uri, index), body, content_type="application/x-ndjson")
def _read_manifest(storage, manifest_uri: str, bucket: Optional[str] = None) -> List[dict]:
raw = storage.download_bytes(_blob_path(manifest_uri, bucket)).decode("utf-8")
return [json.loads(line) for line in raw.splitlines() if line.strip()]
def run_manifest(
storage,
separator_factory: Callable[[], Any],
manifest_uri: str,
index: int,
count: int,
bucket: Optional[str] = None,
) -> List[dict]:
"""Separate this task's share of the manifest; never aborts on a per-song failure."""
if bucket is None:
from backend.config import get_settings
bucket = get_settings().gcs_bucket_name
try:
manifest = _read_manifest(storage, manifest_uri, bucket)
except WrongBucketError:
raise SystemicTaskError(f"manifest bucket is not {bucket} (wrong_bucket)")
lines = shard(manifest, index, count)
logger.info(f"Task {index}/{count}: {len(lines)} songs")
results: List[dict] = []
separator = None # built lazily so an all-skipped shard never loads models
for n, line in enumerate(lines, 1):
pool_id = line.get("id")
vocals_blob = f"{STEMS_PREFIX}/{pool_id}/vocals.flac"
started = time.time()
tmp = tempfile.mkdtemp(prefix="voicepool_")
try:
if storage.file_exists(vocals_blob):
results.append({"id": pool_id, "ok": True, "vocals": vocals_blob, "seconds": 0, "error": None})
continue
if separator is None:
try:
separator = separator_factory()
except Exception as e:
raise SeparatorLoadError(f"separator failed to load: {e}") from e
local = os.path.join(tmp, "original.flac")
storage.download_file(_blob_path(line["audio"], bucket), local)
outputs = separator.separate(local)
# Separator may return bare filenames relative to its output_dir
out_dir = getattr(separator, "output_dir", None)
outputs = [o if os.path.isabs(o) or not out_dir else os.path.join(out_dir, o) for o in outputs]
vocals = vocals_path_from_outputs(outputs)
if not vocals or not os.path.exists(vocals):
raise RuntimeError(f"no vocals stem in separator outputs: {[os.path.basename(o) for o in outputs]}")
storage.upload_file(vocals, vocals_blob)
for o in outputs: # all stems live in the shared output dir; free it per song
try:
os.remove(o)
except OSError:
pass
results.append({
"id": pool_id, "ok": True, "vocals": vocals_blob,
"seconds": round(time.time() - started, 1), "error": None,
})
except SystemicTaskError:
_write_results(storage, manifest_uri, index, results)
raise
except Exception as e: # per-song failure must not abort the task
logger.warning(f"voice-pool separate failed for {pool_id}: {e}")
results.append({
"id": pool_id, "ok": False, "vocals": None,
"seconds": round(time.time() - started, 1), "error": str(e)[:500],
})
finally:
shutil.rmtree(tmp, ignore_errors=True)
if n % RESULTS_FLUSH_EVERY == 0:
try: # periodic flush is best-effort; the final write stays strict
_write_results(storage, manifest_uri, index, results)
except Exception as e:
logger.warning(f"periodic results flush failed (will retry): {e}")
_write_results(storage, manifest_uri, index, results)
return results
def make_separator_factory(output_dir: str) -> Callable[[], Any]:
def factory():
from audio_separator.separator import Separator
from karaoke_gen.audio_processor import _separator_kwargs
separator = Separator(
model_file_dir=os.environ["MODEL_DIR"],
output_format="FLAC",
output_dir=output_dir,
ensemble_preset=INSTRUMENTAL_PRESET,
**_separator_kwargs(),
)
separator.load_model()
return separator
return factory
def main(argv: Optional[List[str]] = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--manifest", required=True, help="gs://bucket/voice-pool/manifests/<run>.jsonl")
args = parser.parse_args(argv)
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
index = int(os.environ.get("CLOUD_RUN_TASK_INDEX", "0"))
count = int(os.environ.get("CLOUD_RUN_TASK_COUNT", "1"))
from backend.services.storage_service import StorageService
out_dir = tempfile.mkdtemp(prefix="voicepool_out_")
try:
results = run_manifest(StorageService(), make_separator_factory(out_dir), args.manifest, index, count)
except SystemicTaskError as e:
# task-independent text so every failing task groups as one alert pattern
logger.info(f"Task {index}/{count} aborting")
logger.error(f"voice-pool separate task aborted: {e}", extra={"task_index": index})
return 1
finally:
shutil.rmtree(out_dir, ignore_errors=True)
failed = sum(1 for r in results if not r["ok"])
if results and failed == len(results):
logger.info(f"Task {index}/{count}: all {failed} songs failed")
logger.error(
f"voice-pool separate task failed: every song failed (first error: {results[0]['error']})",
extra={"task_index": index, "failed": failed},
)
return 1
logger.info(f"Task {index}/{count} done: {len(results) - failed} ok, {failed} failed")
return 0
if __name__ == "__main__":
sys.exit(main())
+10
View File
@@ -749,6 +749,16 @@ is still defined in Pulumi behind the `audioSeparatorServiceEnabled` flag — se
[GCP-COST-OPTIMIZATION.md](GCP-COST-OPTIMIZATION.md#audio-separator-service-shut-down-on-demand-redeploy)
to bring it back on demand.
### Voice-pool separate-only job
`voice-pool-separate-job` (Cloud Run Job, L4 GPU, us-east4, same GPU image) serves karaoke-decide's
voice session. decide writes a JSONL manifest (`voice-pool/manifests/<run>.jsonl`, lines
`{"id", "audio": "gs://…/voice-pool/audio/<id>/original.flac"}`) and starts the job with a per-run
`task_count`. Each task (`CLOUD_RUN_TASK_INDEX`/`COUNT`, round-robin over manifest order) loads one
`instrumental_clean` Separator, writes `voice-pool/stems/<id>/vocals.flac` (skipped if present) and
`<manifest>.results/part-<task>.jsonl` (`{id, ok, vocals, seconds, error}`, flushed every 10 songs).
No Firestore or gen job state. gen's Pulumi only creates the job; decide's Pulumi creates the `voice-pool` SA and grants it `roles/run.jobsExecutorWithOverrides` on it. The job has `parallelism=1` (decide cannot override it), so tasks run one after another and a pool run never uses more than one L4, leaving GPU capacity to customers' `audio-separation-job`. Task timeout is 3600s (Cloud Run's GPU maximum), so each task must finish in under 1h: decide chunks manifests accordingly and treats a timed-out execution as "re-run" (re-runs skip finished songs). URIs outside the gen bucket fail as `wrong_bucket`. A model-load failure or an all-songs-failed task exits non-zero with one ERROR. Code: `backend/workers/voice_pool_separate_worker.py`.
### Quick version (kjbox make-it jobs)
For jobs created by kjbox on behalf of a singer (`request_metadata.client_id` starts with
+4
View File
@@ -187,6 +187,9 @@ edge_resources = edge_security.configure_edge_security()
# These use Cloud Run Jobs instead of Cloud Tasks to avoid instance termination
# during long-running processing (the BackgroundTasks issue)
audio_separation_job = cloud_run.create_audio_separation_job(bucket, backend_service_account)
# Voice-pool separate-only GPU job (started by karaoke-decide's fetch job). decide's
# Pulumi creates the voice-pool SA and grants it run access on this job.
voice_pool_separate_job = cloud_run.create_voice_pool_separate_job(bucket, backend_service_account)
# Latency-critical CPU jobs run in CPU_JOBS_REGION (us-east4): us-central1 Cloud
# Run Jobs queue 2-5 min before starting. The legacy us-central1 copies were
# removed after the us-east4 jobs were verified in prod (2026-09-30). To roll
@@ -828,6 +831,7 @@ pulumi.export("video_worker_queue", queues["video-worker-queue"].name)
pulumi.export("video_encoding_job", video_encoding_job.name)
pulumi.export("lyrics_transcription_job", lyrics_transcription_job.name)
pulumi.export("audio_separation_job", audio_separation_job.name)
pulumi.export("voice_pool_separate_job", voice_pool_separate_job.name)
pulumi.export("audio_download_job", audio_download_job.name)
pulumi.export("bulk_search_job", bulk_search_job.name)
pulumi.export("backend_url", "https://api.nomadkaraoke.com")
+56
View File
@@ -329,6 +329,62 @@ def create_audio_separation_job(
return audio_separation_job
def create_voice_pool_separate_job(
bucket: gcp.storage.Bucket,
service_account: gcp.serviceaccount.Account,
) -> cloudrunv2.Job:
"""
Create the Cloud Run Job that separates vocals for the karaoke-decide voice pool.
Reads a JSONL manifest of original-audio objects (voice-pool/ prefix), runs the
instrumental_clean ensemble on an L4 GPU and writes vocals stems + per-task
results back to GCS. No Firestore and no gen job state. decide overrides
task_count per run; one Separator is loaded per task and reused per song.
Args:
bucket: The GCS bucket holding voice-pool/ objects.
service_account: The service account to run the job (audio worker SA).
"""
return cloudrunv2.Job(
"voice-pool-separate-job",
name="voice-pool-separate-job",
location=AUDIO_WORKER_GPU_REGION,
deletion_protection=False,
template=cloudrunv2.JobTemplateArgs(
task_count=1, # overridden per execution by the decide fetch job
parallelism=1, # one L4 at a time so customer audio-separation-job keeps GPU capacity
template=cloudrunv2.JobTemplateTemplateArgs(
containers=[
cloudrunv2.JobTemplateTemplateContainerArgs(
image=f"{AUDIO_WORKER_GPU_REGION}-docker.pkg.dev/{PROJECT_ID}/karaoke-backend-gpu/karaoke-backend:latest",
args=["python", "-m", "backend.workers.voice_pool_separate_worker"],
resources=cloudrunv2.JobTemplateTemplateContainerResourcesArgs(
limits={
"cpu": "4",
"memory": "16Gi",
"nvidia.com/gpu": "1",
},
),
envs=[
cloudrunv2.JobTemplateTemplateContainerEnvArgs(name="MODEL_DIR", value="/models"),
cloudrunv2.JobTemplateTemplateContainerEnvArgs(name="ENVIRONMENT", value="production"),
cloudrunv2.JobTemplateTemplateContainerEnvArgs(name="GCS_BUCKET_NAME", value=bucket.name),
cloudrunv2.JobTemplateTemplateContainerEnvArgs(name="GOOGLE_CLOUD_PROJECT", value=PROJECT_ID),
],
)
],
node_selector=cloudrunv2.JobTemplateTemplateNodeSelectorArgs(
accelerator="nvidia-l4",
),
gpu_zonal_redundancy_disabled=True, # Required for GPU Jobs (capacity limitation)
service_account=service_account.email,
timeout="3600s", # Cloud Run caps GPU task timeouts at 1h; size shards (task count) so each finishes in <1h
max_retries=0, # decide re-runs; finished songs are skipped via existing stems
),
),
)
def create_audio_download_job(
bucket: gcp.storage.Bucket,
service_account: gcp.serviceaccount.Account,
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.283.0"
version = "0.283.1"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"