From 66dcfd7d192596bf64cb67757af5f5ccdde14abd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrew=20Beveridge=20=E2=98=84=EF=B8=8F?= Date: Mon, 15 Jun 2026 07:44:17 -0400 Subject: [PATCH] fix(tenant): stage instrumental in the orchestrator encode path (#829) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @coderabbitai ignore Follow-up to #828. That PR staged the instrumental in `_encode_via_gce` (legacy `video_worker.py`), but production jobs run the **orchestrator** pipeline (`video_worker_orchestrator.py`), whose `_run_encoding` builds its own `EncodingInput` and never made the user-supplied instrumental available under `jobs/{job_id}/`. Tenant jobs still failed at the GCE encoder with "No instrumental audio found" (confirmed — the "Staged existing instrumental" log never appeared). **Fix:** in the orchestrator's `_run_encoding`, when using the GCE backend, upload the local instrumental (existing/custom — identified by the `(Instrumental User)` / `(Instrumental Custom)` filename) into `jobs/{job_id}/` so the encoder's recursive `find_file` locates it. Consumer `clean`/`backing` jobs are untouched (their stems already live under `jobs/{job_id}/`). Also: the daily E2E uses a tenant-specific title so the two matrix tenants don't share a `base_name` (the GCE encoder caches by base_name). Validated by re-running the full prod E2E to completion after deploy. Co-authored-by: Claude Opus 4.8 (1M context) --- backend/workers/video_worker_orchestrator.py | 16 ++++++++++++++++ pyproject.toml | 2 +- scripts/e2e/tenant_e2e.py | 6 +++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/backend/workers/video_worker_orchestrator.py b/backend/workers/video_worker_orchestrator.py index 0ab6ab40..c43acbbd 100644 --- a/backend/workers/video_worker_orchestrator.py +++ b/backend/workers/video_worker_orchestrator.py @@ -452,6 +452,22 @@ class VideoWorkerOrchestrator: if self.config.countdown_padding_seconds: self.job_log.info(f"Countdown padding: {self.config.countdown_padding_seconds}s - instrumental will be padded") + # The GCE encoder downloads its inputs from input_gcs_path (jobs/{job_id}/) and locates + # the instrumental via find_file("*Instrumental User*", "*existing_instrumental*", ...). + # For user-supplied instrumentals (existing_instrumental / custom) the file only exists + # locally (and under uploads/), never under jobs/{job_id}/ — so the encoder fails with + # "No instrumental audio found". Upload it into the input prefix so the encoder finds it. + if encoding_backend.name == "gce" and self.storage and self.config.instrumental_audio_path: + instr_basename = os.path.basename(self.config.instrumental_audio_path) + needs_staging = ("Instrumental User" in instr_basename) or ("Instrumental Custom" in instr_basename) + if needs_staging and os.path.isfile(self.config.instrumental_audio_path): + try: + dest = f"jobs/{self.config.job_id}/{instr_basename}" + self.storage.upload_file(self.config.instrumental_audio_path, dest) + self.job_log.info(f"Staged instrumental for GCE encoder: {dest}") + except Exception as e: + self.job_log.warning(f"Failed to stage instrumental for GCE encoder: {e}") + # Run encoding with job_span("encoding", self.config.job_id) as span: add_span_event("encoding_started", {"backend": encoding_backend.name}) diff --git a/pyproject.toml b/pyproject.toml index 1c244157..f0e622d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.183.4" +version = "0.183.5" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" diff --git a/scripts/e2e/tenant_e2e.py b/scripts/e2e/tenant_e2e.py index efe106ea..25f0908c 100644 --- a/scripts/e2e/tenant_e2e.py +++ b/scripts/e2e/tenant_e2e.py @@ -33,8 +33,11 @@ BUCKET = os.environ.get("KARAOKE_GCS_BUCKET", "karaoke-gen-storage-nomadkaraoke" TEST_MIXED = f"gs://{BUCKET}/e2e-tests/shared/e2e-mixed.mp3" TEST_INSTRUMENTAL = f"gs://{BUCKET}/e2e-tests/shared/e2e-instrumental.mp3" # Clearly-labelled so the daily output is identifiable in the tenant's Dropbox folder. +# Title is tenant-specific: the GCE encoder caches by base_name (artist - title), so two +# tenants running the matrix concurrently must NOT share a base_name or one would receive +# the other's (differently-themed) cached encode. ARTIST = "Nomad Karaoke" -TITLE = "E2E Health Check" +TITLE = None # set per-tenant below EXPECTED = { "vocalstar": {"theme": "vocalstar"}, @@ -78,6 +81,7 @@ TENANT = sys.argv[1] if len(sys.argv) > 1 else os.environ.get("E2E_TENANT", "voc if TENANT not in EXPECTED: log(f"Unknown tenant: {TENANT}") sys.exit(2) +TITLE = f"E2E Health Check {TENANT}" TOKEN = admin_token()