mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
feat(tenant): track code prefixes every delivered file + CDG title shrinks unbreakable words (v0.273.1) (#1158)
## Summary Follow-up to #1153 and #1157, found by running a real Vocal Star job through the portal in prod (job `8dc6945f`, E2E with track code `E2E1531717`, which passed). 1. **Track code on every file.** #1153 put the tenant's code on the Dropbox **folder** (`VS6588 - Artist - Title/`), but the files inside were still `Artist - Title (…)`. Matt's April 2025 ask was the VS code at the start of **each file**, and Andrew had confirmed "all files prefixed with the track code". - Tenant jobs with their own code now get `VS6588 - Artist - Title (…)` on every Dropbox file. - The entries inside the CDG/TXT zips are renamed too, so each CDG+MP3 pair still matches. - Nomad's own `keep_brand_code` flows (finalise-only, re-render) are unchanged, because this requires `tenant_id`. 2. **CDG title card:** a single word wider than the screen can't wrap, so the card now shrinks the font instead of clipping it. Seen on the first real prod render, where the E2E title has a 24-character timestamp token. ## Changes - `dropbox_service.upload_folder(filename_prefix=…)`. - `prefix_zip_entries()` streams entries and keeps their timestamps/compression, so a repack is deterministic. - `skip_unchanged` also works for repacked zips. - `tenant_track_code_prefix(job)` is set into `OrchestratorConfig.output_filename_prefix`. Tenant-only, and the code is sanitized again before use. - cdgmaker composer: width check in the title-card shrink loop. - Version 0.273.0 → 0.273.1. ## Testing - Prefixed upload, including zip entries and subfolders. - No-prefix path unchanged. - Deterministic, streamed repack. - Tenant-only prefix, with sanitization. - Real-render test for the unbreakable word. - Package unit tests: 2054 passed. Backend: 6066 passed (emulator tests excluded locally). - CodeRabbit: 3 cycles, all findings fixed. @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01Gp5fBmsmWJxgkEHb4JfhXR --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
bfc8f7fb0f
commit
621077474e
7 files changed
+173
-4
No files matched your search
@@ -13,6 +13,8 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import zipfile
|
||||
from typing import Optional, Sequence, Tuple
|
||||
|
||||
from google.cloud import secretmanager
|
||||
@@ -20,6 +22,27 @@ from google.cloud import secretmanager
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def prefix_zip_entries(src_zip: str, dst_zip: str, prefix: str) -> None:
|
||||
"""Copy a zip, prepending ``prefix`` to every entry's file name (not its folders).
|
||||
|
||||
Entries are streamed (never fully loaded into memory) and keep their original
|
||||
timestamps/compression, so re-packing the same zip is deterministic.
|
||||
"""
|
||||
import shutil
|
||||
|
||||
with zipfile.ZipFile(src_zip) as zin, zipfile.ZipFile(dst_zip, "w") as zout:
|
||||
for info in zin.infolist():
|
||||
folder, _, name = info.filename.rpartition("/")
|
||||
if info.is_dir() or not name:
|
||||
zout.writestr(info, b"")
|
||||
continue
|
||||
new_info = zipfile.ZipInfo(f"{folder}/{prefix}{name}" if folder else f"{prefix}{name}", info.date_time)
|
||||
new_info.compress_type = info.compress_type
|
||||
new_info.external_attr = info.external_attr
|
||||
with zin.open(info) as src, zout.open(new_info, "w") as dst:
|
||||
shutil.copyfileobj(src, dst, 1024 * 1024)
|
||||
|
||||
|
||||
_DROPBOX_HASH_BLOCK = 4 * 1024 * 1024
|
||||
|
||||
|
||||
@@ -341,6 +364,7 @@ class DropboxService:
|
||||
remote_path: str,
|
||||
exclude_suffixes: Sequence[str] = (),
|
||||
skip_unchanged: bool = False,
|
||||
filename_prefix: str = "",
|
||||
) -> None:
|
||||
"""
|
||||
Recursively upload all files and subdirectories to Dropbox folder.
|
||||
@@ -353,6 +377,9 @@ class DropboxService:
|
||||
skip_unchanged: Don't re-upload files already in the folder with the
|
||||
same Dropbox content hash (one recursive listing up front). Used
|
||||
when a second pass adds files to a folder that was just uploaded.
|
||||
filename_prefix: Prepended to every uploaded file's name (e.g. a
|
||||
tenant's "VS6588 - " catalogue code). Entries inside .zip packages
|
||||
(CDG/TXT) are renamed too, so CDG+MP3 pairs keep matching names.
|
||||
"""
|
||||
# Ensure remote path starts with /
|
||||
if not remote_path.startswith("/"):
|
||||
@@ -379,7 +406,18 @@ class DropboxService:
|
||||
skipped_count += 1
|
||||
continue
|
||||
local_file = os.path.join(root, filename)
|
||||
remote_file = f"{current_remote}/{filename}"
|
||||
remote_file = f"{current_remote}/{filename_prefix}{filename}"
|
||||
if filename_prefix and filename.lower().endswith(".zip"):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
prefixed_zip = os.path.join(tmp, filename)
|
||||
prefix_zip_entries(local_file, prefixed_zip, filename_prefix)
|
||||
remote_hash = remote_hashes.get(remote_file.lower())
|
||||
if remote_hash and remote_hash == dropbox_content_hash(prefixed_zip):
|
||||
unchanged_count += 1
|
||||
continue
|
||||
self.upload_file(prefixed_zip, remote_file)
|
||||
uploaded_count += 1
|
||||
continue
|
||||
remote_hash = remote_hashes.get(remote_file.lower())
|
||||
if remote_hash and remote_hash == dropbox_content_hash(local_file):
|
||||
unchanged_count += 1
|
||||
|
||||
@@ -777,3 +777,73 @@ class TestUploadFolderSkipUnchanged:
|
||||
service.upload_folder(str(tmp_path), "/f")
|
||||
service._client.files_list_folder.assert_not_called()
|
||||
up.assert_called_once()
|
||||
|
||||
|
||||
class TestTenantFilenamePrefix:
|
||||
"""Tenant track codes prefix every delivered file (and the entries inside zips)."""
|
||||
|
||||
@patch("backend.services.dropbox_service.secretmanager.SecretManagerServiceClient")
|
||||
def test_upload_folder_prefixes_files_and_zip_entries(self, mock_sm_client_class, tmp_path):
|
||||
import zipfile
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.payload.data = json.dumps({"access_token": "token"}).encode()
|
||||
mock_sm_client_class.return_value.access_secret_version.return_value = mock_response
|
||||
|
||||
(tmp_path / "Gerry - Ferry (Final Karaoke Lossy 4k).mp4").write_text("v")
|
||||
(tmp_path / "lyrics").mkdir()
|
||||
(tmp_path / "lyrics" / "Gerry - Ferry (Karaoke).lrc").write_text("l")
|
||||
with zipfile.ZipFile(tmp_path / "Gerry - Ferry (Final Karaoke CDG).zip", "w") as z:
|
||||
z.writestr("Gerry - Ferry (Karaoke).cdg", b"cdg")
|
||||
z.writestr("Gerry - Ferry (Karaoke).mp3", b"mp3")
|
||||
|
||||
service = DropboxService()
|
||||
uploaded = {}
|
||||
|
||||
def capture(local, remote):
|
||||
uploaded[remote] = local
|
||||
if remote.endswith(".zip"):
|
||||
with zipfile.ZipFile(local) as z:
|
||||
uploaded[remote + "#entries"] = sorted(z.namelist())
|
||||
|
||||
with patch.object(service, "upload_file", side_effect=capture):
|
||||
service.upload_folder(str(tmp_path), "/T/VS6588 - Gerry - Ferry", filename_prefix="VS6588 - ")
|
||||
|
||||
assert "/T/VS6588 - Gerry - Ferry/VS6588 - Gerry - Ferry (Final Karaoke Lossy 4k).mp4" in uploaded
|
||||
assert "/T/VS6588 - Gerry - Ferry/lyrics/VS6588 - Gerry - Ferry (Karaoke).lrc" in uploaded
|
||||
zip_remote = "/T/VS6588 - Gerry - Ferry/VS6588 - Gerry - Ferry (Final Karaoke CDG).zip"
|
||||
assert uploaded[zip_remote + "#entries"] == [
|
||||
"VS6588 - Gerry - Ferry (Karaoke).cdg",
|
||||
"VS6588 - Gerry - Ferry (Karaoke).mp3",
|
||||
]
|
||||
|
||||
@patch("backend.services.dropbox_service.secretmanager.SecretManagerServiceClient")
|
||||
def test_no_prefix_keeps_names(self, mock_sm_client_class, tmp_path):
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.payload.data = json.dumps({"access_token": "token"}).encode()
|
||||
mock_sm_client_class.return_value.access_secret_version.return_value = mock_response
|
||||
(tmp_path / "A - T (Karaoke).zip").write_bytes(b"not even a zip")
|
||||
|
||||
service = DropboxService()
|
||||
with patch.object(service, "upload_file") as mock_upload:
|
||||
service.upload_folder(str(tmp_path), "/T/NOMAD-1 - A - T")
|
||||
mock_upload.assert_called_once_with(str(tmp_path / "A - T (Karaoke).zip"), "/T/NOMAD-1 - A - T/A - T (Karaoke).zip")
|
||||
|
||||
def test_prefix_zip_entries_is_deterministic_and_streams(self, tmp_path):
|
||||
import zipfile
|
||||
from backend.services.dropbox_service import prefix_zip_entries, dropbox_content_hash
|
||||
|
||||
src = tmp_path / "src.zip"
|
||||
with zipfile.ZipFile(src, "w", zipfile.ZIP_DEFLATED) as z:
|
||||
z.writestr(zipfile.ZipInfo("A - T (Karaoke).cdg", (2026, 1, 1, 0, 0, 0)), b"x" * 100_000)
|
||||
z.writestr(zipfile.ZipInfo("sub/A - T.txt", (2026, 1, 1, 0, 0, 0)), b"lyrics")
|
||||
prefix_zip_entries(str(src), str(tmp_path / "a.zip"), "VS1 - ")
|
||||
prefix_zip_entries(str(src), str(tmp_path / "b.zip"), "VS1 - ")
|
||||
assert dropbox_content_hash(str(tmp_path / "a.zip")) == dropbox_content_hash(str(tmp_path / "b.zip"))
|
||||
with zipfile.ZipFile(tmp_path / "a.zip") as z:
|
||||
assert sorted(z.namelist()) == ["VS1 - A - T (Karaoke).cdg", "sub/VS1 - A - T.txt"]
|
||||
assert z.read("VS1 - A - T (Karaoke).cdg") == b"x" * 100_000
|
||||
|
||||
@@ -2479,3 +2479,26 @@ class TestEncodePublishSeam:
|
||||
# which with still-malformed output raises the stale signal again.
|
||||
assert mock_service.encode_videos.call_count == 2
|
||||
assert orchestrator.result.final_video is None
|
||||
|
||||
|
||||
class TestTenantTrackCodePrefix:
|
||||
def test_tenant_job_with_code_gets_prefix(self):
|
||||
from types import SimpleNamespace
|
||||
from backend.workers.video_worker_orchestrator import tenant_track_code_prefix
|
||||
|
||||
assert tenant_track_code_prefix(SimpleNamespace(keep_brand_code="VS6588", tenant_id="vocalstar")) == "VS6588 - "
|
||||
|
||||
def test_nomad_keep_brand_code_does_not_rename_files(self):
|
||||
from types import SimpleNamespace
|
||||
from backend.workers.video_worker_orchestrator import tenant_track_code_prefix
|
||||
|
||||
assert tenant_track_code_prefix(SimpleNamespace(keep_brand_code="NOMAD-1234", tenant_id="")) == ""
|
||||
assert tenant_track_code_prefix(SimpleNamespace(keep_brand_code=None, tenant_id="vocalstar")) == ""
|
||||
|
||||
def test_unsafe_code_is_sanitized(self):
|
||||
from types import SimpleNamespace
|
||||
from backend.workers.video_worker_orchestrator import tenant_track_code_prefix
|
||||
|
||||
prefix = tenant_track_code_prefix(SimpleNamespace(keep_brand_code="../VS/1", tenant_id="vocalstar"))
|
||||
assert "/" not in prefix and not prefix.startswith(".")
|
||||
|
||||
@@ -89,6 +89,9 @@ class OrchestratorConfig:
|
||||
|
||||
# Keep existing brand code (for re-processing)
|
||||
keep_brand_code: Optional[str] = None
|
||||
# Tenant track code prefix for delivered file names (e.g. "VS6588 - "), so a
|
||||
# tenant's own catalogue code starts every file, not just the folder.
|
||||
output_filename_prefix: str = ""
|
||||
|
||||
# Whether this run may be announced (False for an admin re-render the admin
|
||||
# didn't opt into announcing). Gates the Discord "new video" post and the
|
||||
@@ -985,7 +988,12 @@ class VideoWorkerOrchestrator:
|
||||
skip = dropbox_skip_suffixes_for(self.config.dropbox_path)
|
||||
if skip:
|
||||
self.job_log.info(f"Excluding from Dropbox upload (kept in GCS): {list(skip)}")
|
||||
dropbox.upload_folder(self.config.output_dir, remote_folder, exclude_suffixes=skip)
|
||||
dropbox.upload_folder(
|
||||
self.config.output_dir,
|
||||
remote_folder,
|
||||
exclude_suffixes=skip,
|
||||
filename_prefix=self.config.output_filename_prefix,
|
||||
)
|
||||
|
||||
# Create sharing link
|
||||
try:
|
||||
@@ -1193,6 +1201,20 @@ def _screen_ext(job, screen_type: str) -> str:
|
||||
return '.png'
|
||||
|
||||
|
||||
def tenant_track_code_prefix(job) -> str:
|
||||
""""<CODE> - " for a tenant job that carries its own catalogue code, else "".
|
||||
|
||||
Only tenant jobs: keep_brand_code is also used by Nomad's finalise-only /
|
||||
re-render flows, whose files must keep the plain "Artist - Title" names.
|
||||
"""
|
||||
code = getattr(job, "keep_brand_code", None)
|
||||
if code and getattr(job, "tenant_id", None):
|
||||
# Validated at job creation; sanitize again since it becomes a path segment.
|
||||
safe_code = sanitize_filename(str(code)).replace("/", "").replace("\\", "").strip(" .")
|
||||
return f"{safe_code} - " if safe_code else ""
|
||||
return ""
|
||||
|
||||
|
||||
def create_orchestrator_config_from_job(
|
||||
job,
|
||||
temp_dir: str,
|
||||
@@ -1314,6 +1336,7 @@ def create_orchestrator_config_from_job(
|
||||
|
||||
# Keep existing brand code
|
||||
keep_brand_code=getattr(job, 'keep_brand_code', None) or rerender_brand_code(job),
|
||||
output_filename_prefix=tenant_track_code_prefix(job),
|
||||
notify_customer=not suppress_customer_notifications(job),
|
||||
|
||||
# Instrumental selection (for GCE encoding)
|
||||
|
||||
@@ -1572,7 +1572,10 @@ class KaraokeComposer:
|
||||
bigfont = ImageFont.truetype(self._title_font_path(), bigfont_size)
|
||||
smallfont = ImageFont.truetype(self._title_font_path(), artist_font_size)
|
||||
|
||||
too_wide = False
|
||||
|
||||
def draw_block(text: str, font, color_index: int, y: int) -> int:
|
||||
nonlocal too_wide
|
||||
for image in render_lines(
|
||||
get_wrapped_text(
|
||||
text,
|
||||
@@ -1581,6 +1584,8 @@ class KaraokeComposer:
|
||||
).split("\n"),
|
||||
font=font,
|
||||
):
|
||||
# A single word wider than the screen can't wrap — shrink instead.
|
||||
too_wide = too_wide or image.width > text_image.width
|
||||
text_image.paste(
|
||||
image.point(lambda v: v and color_index, "P"),
|
||||
((text_image.width - image.width) // 2, y),
|
||||
@@ -1619,7 +1624,7 @@ class KaraokeComposer:
|
||||
# We'll crop to just the content area, without padding
|
||||
text_image = text_image.crop(bbox)
|
||||
|
||||
if text_image.height <= MAX_HEIGHT:
|
||||
if text_image.height <= MAX_HEIGHT and not too_wide:
|
||||
self.logger.debug("height just right")
|
||||
break
|
||||
# If text box is not small enough, reduce the title font size first;
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.273.0"
|
||||
version = "0.273.1"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
@@ -153,3 +153,13 @@ class TestOutro:
|
||||
frame = decode_frame(cdg, data_len / CDG_FPS)
|
||||
rows = rows_with(frame, YELLOW)
|
||||
assert rows and min(rows) >= 60
|
||||
|
||||
|
||||
class TestUnbreakableWords:
|
||||
def test_long_unbreakable_title_shrinks_to_fit_the_width(self, tmp_path):
|
||||
cdg = render(tmp_path, "E2E vocalstar 2026-10-15T05-18-51-717Z", "Nomad E2E", title_font_size=28)
|
||||
frame = decode_frame(cdg, 4.8)
|
||||
cols = sorted({x for y in range(H) for x in range(W) if frame.getpixel((x, y)) == YELLOW})
|
||||
# Text stays inside the visible area instead of being clipped at the edges.
|
||||
assert cols and cols[0] > 6 and cols[-1] < W - 6
|
||||
|
||||
Reference in new issue
Block a user