diff --git a/backend/services/translation_preview.py b/backend/services/translation_preview.py index 575f479c..666cd873 100644 --- a/backend/services/translation_preview.py +++ b/backend/services/translation_preview.py @@ -4,8 +4,16 @@ Renders one karaoke frame of the default theme (same renderer as production, via ``theme_preview_service``) with sample lyrics and their translation in the chosen language. Translations come from ``translation_preview_samples`` (generated once, committed), so the preview never calls the LLM. + +Each frame is a 4K ffmpeg render (~20s, ~1 GB peak). Two concurrent renders for +different languages got the API instance OOM-killed (10-09, 10-16), 503ing every +in-flight request. So renders are serialised process-wide, and each rendered frame +is kept in GCS (keyed by a digest of the theme style + sample) so a language is +rendered once per theme change, not once per cold instance. """ import base64 +import hashlib +import json import logging import tempfile import threading @@ -24,6 +32,8 @@ _CACHE: "OrderedDict[tuple, str]" = OrderedDict() _CACHE_MAX = 40 _lock = threading.Lock() _key_locks: "dict[tuple, threading.Lock]" = {} # bounded: themes x 33 languages +_render_slot = threading.BoundedSemaphore(1) # one 4K render at a time per instance +GCS_PREFIX = "themes/_translation_previews" def render_translation_preview(language: str, theme_id: Optional[str] = None) -> str: @@ -48,14 +58,50 @@ def render_translation_preview(language: str, theme_id: Optional[str] = None) -> return _render_and_cache(key, theme_service, theme_id, code) +def _gcs_path(theme_id: str, code: str, style_params: dict) -> str: + digest = hashlib.sha256( + json.dumps([style_params, SAMPLES[code]], sort_keys=True, default=str).encode() + ).hexdigest()[:16] + return f"{GCS_PREFIX}/{theme_id}/{code}-{digest}.jpg" + + +def _gcs_get(path: str) -> Optional[bytes]: + try: + from google.cloud.exceptions import NotFound + + from backend.services.storage_service import StorageService + + try: + return StorageService().bucket.blob(path).download_as_bytes() + except NotFound: + return None + except Exception as e: # cache read is best-effort; fall back to rendering + logger.warning(f"Translation preview cache read failed for {path}: {e}") + return None + + +def _gcs_put(path: str, jpeg: bytes) -> None: + try: + from backend.services.storage_service import StorageService + + StorageService().bucket.blob(path).upload_from_string(jpeg, content_type="image/jpeg") + except Exception as e: # best-effort; the next cold instance just renders again + logger.warning(f"Translation preview cache write failed for {path}: {e}") + + def _render_and_cache(key: tuple, theme_service, theme_id: str, code: str) -> str: style_params = theme_service.get_theme_style_params(theme_id) if style_params is None: raise ValueError(f"Theme not found: {theme_id}") - styles = resolve_assets(style_params, theme_id) - sample = SAMPLES[code] - with tempfile.TemporaryDirectory(prefix="translation-preview-") as workdir: - jpeg = render_karaoke_frame(styles, sample["lyrics"], workdir, translations=sample["translations"]) + path = _gcs_path(theme_id, code, style_params) + jpeg = _gcs_get(path) + if jpeg is None: + styles = resolve_assets(style_params, theme_id) + sample = SAMPLES[code] + with _render_slot: + with tempfile.TemporaryDirectory(prefix="translation-preview-") as workdir: + jpeg = render_karaoke_frame(styles, sample["lyrics"], workdir, translations=sample["translations"]) + _gcs_put(path, jpeg) data_url = "data:image/jpeg;base64," + base64.b64encode(jpeg).decode() with _lock: _CACHE[key] = data_url diff --git a/backend/tests/test_lyrics_translation.py b/backend/tests/test_lyrics_translation.py index 6338e7a0..aff471a6 100644 --- a/backend/tests/test_lyrics_translation.py +++ b/backend/tests/test_lyrics_translation.py @@ -283,17 +283,20 @@ def test_preview_renders_sample_with_translations(): theme_service.get_theme_style_params.return_value = {"karaoke": {}} with patch.object(tp, "get_theme_service", return_value=theme_service), \ patch.object(tp, "resolve_assets", side_effect=lambda s, t: s), \ + patch.object(tp, "_gcs_get", return_value=None), \ + patch.object(tp, "_gcs_put") as put, \ patch.object(tp, "render_karaoke_frame", return_value=b"jpeg") as render: url = tp.render_translation_preview("fr") tp.render_translation_preview("fr") # cached assert url.startswith("data:image/jpeg;base64,") render.assert_called_once() assert render.call_args.kwargs["translations"] == tp.SAMPLES["fr"]["translations"] + put.assert_called_once() + assert put.call_args.args[0].startswith("themes/_translation_previews/nomad/fr-") def test_concurrent_cold_previews_render_once(): import threading - import time as _time from backend.services import translation_preview as tp @@ -303,11 +306,13 @@ def test_concurrent_cold_previews_render_once(): theme_service.get_theme_style_params.return_value = {"karaoke": {}} def slow_render(*a, **k): - _time.sleep(0.2) + threading.Event().wait(0.2) # not time.sleep: the module's no_sleep fixture stubs it return b"jpeg" with patch.object(tp, "get_theme_service", return_value=theme_service), \ patch.object(tp, "resolve_assets", side_effect=lambda s, t: s), \ + patch.object(tp, "_gcs_get", return_value=None), \ + patch.object(tp, "_gcs_put"), \ patch.object(tp, "render_karaoke_frame", side_effect=slow_render) as render: threads = [threading.Thread(target=tp.render_translation_preview, args=("de",)) for _ in range(5)] for t in threads: @@ -315,3 +320,65 @@ def test_concurrent_cold_previews_render_once(): for t in threads: t.join() assert render.call_count == 1 + + +def _preview_theme_service(style_params=None): + theme_service = MagicMock() + theme_service.get_default_theme_id.return_value = "nomad" + theme_service.get_theme_style_params.return_value = style_params or {"karaoke": {}} + return theme_service + + +def test_preview_served_from_gcs_without_rendering(): + from backend.services import translation_preview as tp + + tp._CACHE.clear() + with patch.object(tp, "get_theme_service", return_value=_preview_theme_service()), \ + patch.object(tp, "_gcs_get", return_value=b"cached") as get, \ + patch.object(tp, "render_karaoke_frame") as render: + url = tp.render_translation_preview("it") + render.assert_not_called() + assert url == "data:image/jpeg;base64,Y2FjaGVk" + assert get.call_args.args[0].startswith("themes/_translation_previews/nomad/it-") + + +def test_theme_change_gets_a_new_gcs_key(): + from backend.services import translation_preview as tp + + a = tp._gcs_path("nomad", "it", {"karaoke": {"font": "A"}}) + b = tp._gcs_path("nomad", "it", {"karaoke": {"font": "B"}}) + assert a != b and a == tp._gcs_path("nomad", "it", {"karaoke": {"font": "A"}}) + + +def test_different_languages_never_render_concurrently(): + """Two concurrent 4K renders OOM-killed the API instance (10-09, 10-16).""" + import threading + + from backend.services import translation_preview as tp + + tp._CACHE.clear() + active, peak = [0], [0] + guard = threading.Lock() + + def render(*a, **k): + with guard: + active[0] += 1 + peak[0] = max(peak[0], active[0]) + threading.Event().wait(0.05) # not time.sleep: the module's no_sleep fixture stubs it + with guard: + active[0] -= 1 + return b"jpeg" + + with patch.object(tp, "get_theme_service", return_value=_preview_theme_service()), \ + patch.object(tp, "resolve_assets", side_effect=lambda s, t: s), \ + patch.object(tp, "_gcs_get", return_value=None), \ + patch.object(tp, "_gcs_put"), \ + patch.object(tp, "render_karaoke_frame", side_effect=render) as mock_render: + threads = [threading.Thread(target=tp.render_translation_preview, args=(code,)) + for code in ("de", "fr", "es", "it")] + for t in threads: + t.start() + for t in threads: + t.join() + assert mock_render.call_count == 4 + assert peak[0] == 1 diff --git a/pyproject.toml b/pyproject.toml index f4b3b0c4..5e9c354a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.278.16" +version = "0.278.17" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"