mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix: serialise translation-preview renders and keep them in GCS (OOM SIGKILLs) (v0.278.17) (#1197)
@coderabbitai ignore ## Why BACKLOG "translation-preview renders get the API instance SIGKILLed (OOM)" (#alerts msgs 1557981404242116608, 1560563690053439628). On 10-09 and 10-16, two concurrent `GET /api/themes/translation-preview` renders for **different** languages, each a 4K ffmpeg frame taking about 20s, were followed by `Container terminated on signal 9`. Every in-flight request got a 503, including a real user's `create-from-search` on 10-09. The existing per-key lock only prevented duplicate renders of the *same* language. ## What - `_render_slot = BoundedSemaphore(1)`: one render at a time per instance. - **Persistent cache:** rendered frames go to GCS at `themes/_translation_previews/<theme>/<lang>-<sha256(style_params + sample)[:16]>.jpg`. Each language renders once per theme change rather than once per cold instance, so very few requests ever wait on the slot. A theme edit changes the digest, so stale frames are never served. Cache reads and writes are best-effort: failures log a WARNING and fall back to rendering. They use the bucket directly, so a cache miss isn't logged as a "not found". - **Fixed an existing test blind spot:** this module's autouse `no_sleep` fixture stubs the global `time.sleep`. As a result `test_concurrent_cold_previews_render_once` passed without any threads overlapping, and so did my first version of the new concurrency test. Both now wait on `threading.Event().wait()`. ## Testing - New: a GCS hit serves without rendering; a theme change gets a new key; four languages never render concurrently. Mutation-checked: with `BoundedSemaphore(8)` it fails with `assert 4 == 1`. - `test_lyrics_translation.py`: 33 passed. Local CodeRabbit: 0 findings. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01Jw21syo4HizJJjYw2mg2Vp Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
acb02e138a
commit
37b4f5126a
3 files changed
+120
-7
No files matched your search
@@ -4,8 +4,16 @@ Renders one karaoke frame of the default theme (same renderer as production, via
|
||||
``theme_preview_service``) with sample lyrics and their translation in the chosen
|
||||
language. Translations come from ``translation_preview_samples`` (generated once,
|
||||
committed), so the preview never calls the LLM.
|
||||
|
||||
Each frame is a 4K ffmpeg render (~20s, ~1 GB peak). Two concurrent renders for
|
||||
different languages got the API instance OOM-killed (10-09, 10-16), 503ing every
|
||||
in-flight request. So renders are serialised process-wide, and each rendered frame
|
||||
is kept in GCS (keyed by a digest of the theme style + sample) so a language is
|
||||
rendered once per theme change, not once per cold instance.
|
||||
"""
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import tempfile
|
||||
import threading
|
||||
@@ -24,6 +32,8 @@ _CACHE: "OrderedDict[tuple, str]" = OrderedDict()
|
||||
_CACHE_MAX = 40
|
||||
_lock = threading.Lock()
|
||||
_key_locks: "dict[tuple, threading.Lock]" = {} # bounded: themes x 33 languages
|
||||
_render_slot = threading.BoundedSemaphore(1) # one 4K render at a time per instance
|
||||
GCS_PREFIX = "themes/_translation_previews"
|
||||
|
||||
|
||||
def render_translation_preview(language: str, theme_id: Optional[str] = None) -> str:
|
||||
@@ -48,14 +58,50 @@ def render_translation_preview(language: str, theme_id: Optional[str] = None) ->
|
||||
return _render_and_cache(key, theme_service, theme_id, code)
|
||||
|
||||
|
||||
def _gcs_path(theme_id: str, code: str, style_params: dict) -> str:
|
||||
digest = hashlib.sha256(
|
||||
json.dumps([style_params, SAMPLES[code]], sort_keys=True, default=str).encode()
|
||||
).hexdigest()[:16]
|
||||
return f"{GCS_PREFIX}/{theme_id}/{code}-{digest}.jpg"
|
||||
|
||||
|
||||
def _gcs_get(path: str) -> Optional[bytes]:
|
||||
try:
|
||||
from google.cloud.exceptions import NotFound
|
||||
|
||||
from backend.services.storage_service import StorageService
|
||||
|
||||
try:
|
||||
return StorageService().bucket.blob(path).download_as_bytes()
|
||||
except NotFound:
|
||||
return None
|
||||
except Exception as e: # cache read is best-effort; fall back to rendering
|
||||
logger.warning(f"Translation preview cache read failed for {path}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def _gcs_put(path: str, jpeg: bytes) -> None:
|
||||
try:
|
||||
from backend.services.storage_service import StorageService
|
||||
|
||||
StorageService().bucket.blob(path).upload_from_string(jpeg, content_type="image/jpeg")
|
||||
except Exception as e: # best-effort; the next cold instance just renders again
|
||||
logger.warning(f"Translation preview cache write failed for {path}: {e}")
|
||||
|
||||
|
||||
def _render_and_cache(key: tuple, theme_service, theme_id: str, code: str) -> str:
|
||||
style_params = theme_service.get_theme_style_params(theme_id)
|
||||
if style_params is None:
|
||||
raise ValueError(f"Theme not found: {theme_id}")
|
||||
styles = resolve_assets(style_params, theme_id)
|
||||
sample = SAMPLES[code]
|
||||
with tempfile.TemporaryDirectory(prefix="translation-preview-") as workdir:
|
||||
jpeg = render_karaoke_frame(styles, sample["lyrics"], workdir, translations=sample["translations"])
|
||||
path = _gcs_path(theme_id, code, style_params)
|
||||
jpeg = _gcs_get(path)
|
||||
if jpeg is None:
|
||||
styles = resolve_assets(style_params, theme_id)
|
||||
sample = SAMPLES[code]
|
||||
with _render_slot:
|
||||
with tempfile.TemporaryDirectory(prefix="translation-preview-") as workdir:
|
||||
jpeg = render_karaoke_frame(styles, sample["lyrics"], workdir, translations=sample["translations"])
|
||||
_gcs_put(path, jpeg)
|
||||
data_url = "data:image/jpeg;base64," + base64.b64encode(jpeg).decode()
|
||||
with _lock:
|
||||
_CACHE[key] = data_url
|
||||
|
||||
@@ -283,17 +283,20 @@ def test_preview_renders_sample_with_translations():
|
||||
theme_service.get_theme_style_params.return_value = {"karaoke": {}}
|
||||
with patch.object(tp, "get_theme_service", return_value=theme_service), \
|
||||
patch.object(tp, "resolve_assets", side_effect=lambda s, t: s), \
|
||||
patch.object(tp, "_gcs_get", return_value=None), \
|
||||
patch.object(tp, "_gcs_put") as put, \
|
||||
patch.object(tp, "render_karaoke_frame", return_value=b"jpeg") as render:
|
||||
url = tp.render_translation_preview("fr")
|
||||
tp.render_translation_preview("fr") # cached
|
||||
assert url.startswith("data:image/jpeg;base64,")
|
||||
render.assert_called_once()
|
||||
assert render.call_args.kwargs["translations"] == tp.SAMPLES["fr"]["translations"]
|
||||
put.assert_called_once()
|
||||
assert put.call_args.args[0].startswith("themes/_translation_previews/nomad/fr-")
|
||||
|
||||
|
||||
def test_concurrent_cold_previews_render_once():
|
||||
import threading
|
||||
import time as _time
|
||||
|
||||
from backend.services import translation_preview as tp
|
||||
|
||||
@@ -303,11 +306,13 @@ def test_concurrent_cold_previews_render_once():
|
||||
theme_service.get_theme_style_params.return_value = {"karaoke": {}}
|
||||
|
||||
def slow_render(*a, **k):
|
||||
_time.sleep(0.2)
|
||||
threading.Event().wait(0.2) # not time.sleep: the module's no_sleep fixture stubs it
|
||||
return b"jpeg"
|
||||
|
||||
with patch.object(tp, "get_theme_service", return_value=theme_service), \
|
||||
patch.object(tp, "resolve_assets", side_effect=lambda s, t: s), \
|
||||
patch.object(tp, "_gcs_get", return_value=None), \
|
||||
patch.object(tp, "_gcs_put"), \
|
||||
patch.object(tp, "render_karaoke_frame", side_effect=slow_render) as render:
|
||||
threads = [threading.Thread(target=tp.render_translation_preview, args=("de",)) for _ in range(5)]
|
||||
for t in threads:
|
||||
@@ -315,3 +320,65 @@ def test_concurrent_cold_previews_render_once():
|
||||
for t in threads:
|
||||
t.join()
|
||||
assert render.call_count == 1
|
||||
|
||||
|
||||
def _preview_theme_service(style_params=None):
|
||||
theme_service = MagicMock()
|
||||
theme_service.get_default_theme_id.return_value = "nomad"
|
||||
theme_service.get_theme_style_params.return_value = style_params or {"karaoke": {}}
|
||||
return theme_service
|
||||
|
||||
|
||||
def test_preview_served_from_gcs_without_rendering():
|
||||
from backend.services import translation_preview as tp
|
||||
|
||||
tp._CACHE.clear()
|
||||
with patch.object(tp, "get_theme_service", return_value=_preview_theme_service()), \
|
||||
patch.object(tp, "_gcs_get", return_value=b"cached") as get, \
|
||||
patch.object(tp, "render_karaoke_frame") as render:
|
||||
url = tp.render_translation_preview("it")
|
||||
render.assert_not_called()
|
||||
assert url == "data:image/jpeg;base64,Y2FjaGVk"
|
||||
assert get.call_args.args[0].startswith("themes/_translation_previews/nomad/it-")
|
||||
|
||||
|
||||
def test_theme_change_gets_a_new_gcs_key():
|
||||
from backend.services import translation_preview as tp
|
||||
|
||||
a = tp._gcs_path("nomad", "it", {"karaoke": {"font": "A"}})
|
||||
b = tp._gcs_path("nomad", "it", {"karaoke": {"font": "B"}})
|
||||
assert a != b and a == tp._gcs_path("nomad", "it", {"karaoke": {"font": "A"}})
|
||||
|
||||
|
||||
def test_different_languages_never_render_concurrently():
|
||||
"""Two concurrent 4K renders OOM-killed the API instance (10-09, 10-16)."""
|
||||
import threading
|
||||
|
||||
from backend.services import translation_preview as tp
|
||||
|
||||
tp._CACHE.clear()
|
||||
active, peak = [0], [0]
|
||||
guard = threading.Lock()
|
||||
|
||||
def render(*a, **k):
|
||||
with guard:
|
||||
active[0] += 1
|
||||
peak[0] = max(peak[0], active[0])
|
||||
threading.Event().wait(0.05) # not time.sleep: the module's no_sleep fixture stubs it
|
||||
with guard:
|
||||
active[0] -= 1
|
||||
return b"jpeg"
|
||||
|
||||
with patch.object(tp, "get_theme_service", return_value=_preview_theme_service()), \
|
||||
patch.object(tp, "resolve_assets", side_effect=lambda s, t: s), \
|
||||
patch.object(tp, "_gcs_get", return_value=None), \
|
||||
patch.object(tp, "_gcs_put"), \
|
||||
patch.object(tp, "render_karaoke_frame", side_effect=render) as mock_render:
|
||||
threads = [threading.Thread(target=tp.render_translation_preview, args=(code,))
|
||||
for code in ("de", "fr", "es", "it")]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
assert mock_render.call_count == 4
|
||||
assert peak[0] == 1
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.278.16"
|
||||
version = "0.278.17"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in new issue
Block a user