fix(tenant): stage instrumental in the orchestrator encode path (#829)

@coderabbitai ignore

Follow-up to #828. That PR staged the instrumental in `_encode_via_gce`
(legacy `video_worker.py`), but production jobs run the **orchestrator**
pipeline (`video_worker_orchestrator.py`), whose `_run_encoding` builds
its own `EncodingInput` and never made the user-supplied instrumental
available under `jobs/{job_id}/`. Tenant jobs still failed at the GCE
encoder with "No instrumental audio found" (confirmed — the "Staged
existing instrumental" log never appeared).

**Fix:** in the orchestrator's `_run_encoding`, when using the GCE
backend, upload the local instrumental (existing/custom — identified by
the `(Instrumental User)` / `(Instrumental Custom)` filename) into
`jobs/{job_id}/` so the encoder's recursive `find_file` locates it.
Consumer `clean`/`backing` jobs are untouched (their stems already live
under `jobs/{job_id}/`).

Also: the daily E2E uses a tenant-specific title so the two matrix
tenants don't share a `base_name` (the GCE encoder caches by base_name).

Validated by re-running the full prod E2E to completion after deploy.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Andrew Beveridge ☄️andClaude Opus 4.8 authored and GitHub committed 2026-06-15 07:44:17 -04:00
1 parent 40c7960854
commit 66dcfd7d19
3 files changed
+22 -2

No files matched your search

@@ -452,6 +452,22 @@ class VideoWorkerOrchestrator:
if self.config.countdown_padding_seconds:
self.job_log.info(f"Countdown padding: {self.config.countdown_padding_seconds}s - instrumental will be padded")
# The GCE encoder downloads its inputs from input_gcs_path (jobs/{job_id}/) and locates
# the instrumental via find_file("*Instrumental User*", "*existing_instrumental*", ...).
# For user-supplied instrumentals (existing_instrumental / custom) the file only exists
# locally (and under uploads/), never under jobs/{job_id}/ — so the encoder fails with
# "No instrumental audio found". Upload it into the input prefix so the encoder finds it.
if encoding_backend.name == "gce" and self.storage and self.config.instrumental_audio_path:
instr_basename = os.path.basename(self.config.instrumental_audio_path)
needs_staging = ("Instrumental User" in instr_basename) or ("Instrumental Custom" in instr_basename)
if needs_staging and os.path.isfile(self.config.instrumental_audio_path):
try:
dest = f"jobs/{self.config.job_id}/{instr_basename}"
self.storage.upload_file(self.config.instrumental_audio_path, dest)
self.job_log.info(f"Staged instrumental for GCE encoder: {dest}")
except Exception as e:
self.job_log.warning(f"Failed to stage instrumental for GCE encoder: {e}")
# Run encoding
with job_span("encoding", self.config.job_id) as span:
add_span_event("encoding_started", {"backend": encoding_backend.name})
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.183.4"
version = "0.183.5"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"
+5 -1
View File
@@ -33,8 +33,11 @@ BUCKET = os.environ.get("KARAOKE_GCS_BUCKET", "karaoke-gen-storage-nomadkaraoke"
TEST_MIXED = f"gs://{BUCKET}/e2e-tests/shared/e2e-mixed.mp3"
TEST_INSTRUMENTAL = f"gs://{BUCKET}/e2e-tests/shared/e2e-instrumental.mp3"
# Clearly-labelled so the daily output is identifiable in the tenant's Dropbox folder.
# Title is tenant-specific: the GCE encoder caches by base_name (artist - title), so two
# tenants running the matrix concurrently must NOT share a base_name or one would receive
# the other's (differently-themed) cached encode.
ARTIST = "Nomad Karaoke"
TITLE = "E2E Health Check"
TITLE = None # set per-tenant below
EXPECTED = {
"vocalstar": {"theme": "vocalstar"},
@@ -78,6 +81,7 @@ TENANT = sys.argv[1] if len(sys.argv) > 1 else os.environ.get("E2E_TENANT", "voc
if TENANT not in EXPECTED:
log(f"Unknown tenant: {TENANT}")
sys.exit(2)
TITLE = f"E2E Health Check {TENANT}"
TOKEN = admin_token()