fix: serialise translation-preview renders and keep them in GCS (OOM SIGKILLs) (v0.278.17) (#1197)

@coderabbitai ignore

## Why
BACKLOG "translation-preview renders get the API instance SIGKILLed
(OOM)" (#alerts msgs 1557981404242116608, 1560563690053439628). On 10-09
and 10-16, two concurrent `GET /api/themes/translation-preview` renders
for **different** languages, each a 4K ffmpeg frame taking about 20s,
were followed by `Container terminated on signal 9`. Every in-flight
request got a 503, including a real user's `create-from-search` on
10-09. The existing per-key lock only prevented duplicate renders of the
*same* language.

## What
- `_render_slot = BoundedSemaphore(1)`: one render at a time per
instance.
- **Persistent cache:** rendered frames go to GCS at
`themes/_translation_previews/<theme>/<lang>-<sha256(style_params +
sample)[:16]>.jpg`. Each language renders once per theme change rather
than once per cold instance, so very few requests ever wait on the slot.
A theme edit changes the digest, so stale frames are never served. Cache
reads and writes are best-effort: failures log a WARNING and fall back
to rendering. They use the bucket directly, so a cache miss isn't logged
as a "not found".
- **Fixed an existing test blind spot:** this module's autouse
`no_sleep` fixture stubs the global `time.sleep`. As a result
`test_concurrent_cold_previews_render_once` passed without any threads
overlapping, and so did my first version of the new concurrency test.
Both now wait on `threading.Event().wait()`.

## Testing
- New: a GCS hit serves without rendering; a theme change gets a new
key; four languages never render concurrently. Mutation-checked: with
`BoundedSemaphore(8)` it fails with `assert 4 == 1`.
- `test_lyrics_translation.py`: 33 passed. Local CodeRabbit: 0 findings.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

https://claude.ai/code/session_01Jw21syo4HizJJjYw2mg2Vp

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andrew Beveridge ☄️andClaude Opus 5.5 authored and GitHub committed 2026-10-19 15:38:28 -04:00
1 parent acb02e138a
commit 37b4f5126a
3 files changed
+120 -7

No files matched your search

+50 -4
View File
@@ -4,8 +4,16 @@ Renders one karaoke frame of the default theme (same renderer as production, via
``theme_preview_service``) with sample lyrics and their translation in the chosen
language. Translations come from ``translation_preview_samples`` (generated once,
committed), so the preview never calls the LLM.
Each frame is a 4K ffmpeg render (~20s, ~1 GB peak). Two concurrent renders for
different languages got the API instance OOM-killed (10-09, 10-16), 503ing every
in-flight request. So renders are serialised process-wide, and each rendered frame
is kept in GCS (keyed by a digest of the theme style + sample) so a language is
rendered once per theme change, not once per cold instance.
"""
import base64
import hashlib
import json
import logging
import tempfile
import threading
@@ -24,6 +32,8 @@ _CACHE: "OrderedDict[tuple, str]" = OrderedDict()
_CACHE_MAX = 40
_lock = threading.Lock()
_key_locks: "dict[tuple, threading.Lock]" = {} # bounded: themes x 33 languages
_render_slot = threading.BoundedSemaphore(1) # one 4K render at a time per instance
GCS_PREFIX = "themes/_translation_previews"
def render_translation_preview(language: str, theme_id: Optional[str] = None) -> str:
@@ -48,14 +58,50 @@ def render_translation_preview(language: str, theme_id: Optional[str] = None) ->
return _render_and_cache(key, theme_service, theme_id, code)
def _gcs_path(theme_id: str, code: str, style_params: dict) -> str:
digest = hashlib.sha256(
json.dumps([style_params, SAMPLES[code]], sort_keys=True, default=str).encode()
).hexdigest()[:16]
return f"{GCS_PREFIX}/{theme_id}/{code}-{digest}.jpg"
def _gcs_get(path: str) -> Optional[bytes]:
try:
from google.cloud.exceptions import NotFound
from backend.services.storage_service import StorageService
try:
return StorageService().bucket.blob(path).download_as_bytes()
except NotFound:
return None
except Exception as e: # cache read is best-effort; fall back to rendering
logger.warning(f"Translation preview cache read failed for {path}: {e}")
return None
def _gcs_put(path: str, jpeg: bytes) -> None:
try:
from backend.services.storage_service import StorageService
StorageService().bucket.blob(path).upload_from_string(jpeg, content_type="image/jpeg")
except Exception as e: # best-effort; the next cold instance just renders again
logger.warning(f"Translation preview cache write failed for {path}: {e}")
def _render_and_cache(key: tuple, theme_service, theme_id: str, code: str) -> str:
style_params = theme_service.get_theme_style_params(theme_id)
if style_params is None:
raise ValueError(f"Theme not found: {theme_id}")
styles = resolve_assets(style_params, theme_id)
sample = SAMPLES[code]
with tempfile.TemporaryDirectory(prefix="translation-preview-") as workdir:
jpeg = render_karaoke_frame(styles, sample["lyrics"], workdir, translations=sample["translations"])
path = _gcs_path(theme_id, code, style_params)
jpeg = _gcs_get(path)
if jpeg is None:
styles = resolve_assets(style_params, theme_id)
sample = SAMPLES[code]
with _render_slot:
with tempfile.TemporaryDirectory(prefix="translation-preview-") as workdir:
jpeg = render_karaoke_frame(styles, sample["lyrics"], workdir, translations=sample["translations"])
_gcs_put(path, jpeg)
data_url = "data:image/jpeg;base64," + base64.b64encode(jpeg).decode()
with _lock:
_CACHE[key] = data_url
+69 -2
View File
@@ -283,17 +283,20 @@ def test_preview_renders_sample_with_translations():
theme_service.get_theme_style_params.return_value = {"karaoke": {}}
with patch.object(tp, "get_theme_service", return_value=theme_service), \
patch.object(tp, "resolve_assets", side_effect=lambda s, t: s), \
patch.object(tp, "_gcs_get", return_value=None), \
patch.object(tp, "_gcs_put") as put, \
patch.object(tp, "render_karaoke_frame", return_value=b"jpeg") as render:
url = tp.render_translation_preview("fr")
tp.render_translation_preview("fr") # cached
assert url.startswith("data:image/jpeg;base64,")
render.assert_called_once()
assert render.call_args.kwargs["translations"] == tp.SAMPLES["fr"]["translations"]
put.assert_called_once()
assert put.call_args.args[0].startswith("themes/_translation_previews/nomad/fr-")
def test_concurrent_cold_previews_render_once():
import threading
import time as _time
from backend.services import translation_preview as tp
@@ -303,11 +306,13 @@ def test_concurrent_cold_previews_render_once():
theme_service.get_theme_style_params.return_value = {"karaoke": {}}
def slow_render(*a, **k):
_time.sleep(0.2)
threading.Event().wait(0.2) # not time.sleep: the module's no_sleep fixture stubs it
return b"jpeg"
with patch.object(tp, "get_theme_service", return_value=theme_service), \
patch.object(tp, "resolve_assets", side_effect=lambda s, t: s), \
patch.object(tp, "_gcs_get", return_value=None), \
patch.object(tp, "_gcs_put"), \
patch.object(tp, "render_karaoke_frame", side_effect=slow_render) as render:
threads = [threading.Thread(target=tp.render_translation_preview, args=("de",)) for _ in range(5)]
for t in threads:
@@ -315,3 +320,65 @@ def test_concurrent_cold_previews_render_once():
for t in threads:
t.join()
assert render.call_count == 1
def _preview_theme_service(style_params=None):
theme_service = MagicMock()
theme_service.get_default_theme_id.return_value = "nomad"
theme_service.get_theme_style_params.return_value = style_params or {"karaoke": {}}
return theme_service
def test_preview_served_from_gcs_without_rendering():
from backend.services import translation_preview as tp
tp._CACHE.clear()
with patch.object(tp, "get_theme_service", return_value=_preview_theme_service()), \
patch.object(tp, "_gcs_get", return_value=b"cached") as get, \
patch.object(tp, "render_karaoke_frame") as render:
url = tp.render_translation_preview("it")
render.assert_not_called()
assert url == "data:image/jpeg;base64,Y2FjaGVk"
assert get.call_args.args[0].startswith("themes/_translation_previews/nomad/it-")
def test_theme_change_gets_a_new_gcs_key():
from backend.services import translation_preview as tp
a = tp._gcs_path("nomad", "it", {"karaoke": {"font": "A"}})
b = tp._gcs_path("nomad", "it", {"karaoke": {"font": "B"}})
assert a != b and a == tp._gcs_path("nomad", "it", {"karaoke": {"font": "A"}})
def test_different_languages_never_render_concurrently():
"""Two concurrent 4K renders OOM-killed the API instance (10-09, 10-16)."""
import threading
from backend.services import translation_preview as tp
tp._CACHE.clear()
active, peak = [0], [0]
guard = threading.Lock()
def render(*a, **k):
with guard:
active[0] += 1
peak[0] = max(peak[0], active[0])
threading.Event().wait(0.05) # not time.sleep: the module's no_sleep fixture stubs it
with guard:
active[0] -= 1
return b"jpeg"
with patch.object(tp, "get_theme_service", return_value=_preview_theme_service()), \
patch.object(tp, "resolve_assets", side_effect=lambda s, t: s), \
patch.object(tp, "_gcs_get", return_value=None), \
patch.object(tp, "_gcs_put"), \
patch.object(tp, "render_karaoke_frame", side_effect=render) as mock_render:
threads = [threading.Thread(target=tp.render_translation_preview, args=(code,))
for code in ("de", "fr", "es", "it")]
for t in threads:
t.start()
for t in threads:
t.join()
assert mock_render.call_count == 4
assert peak[0] == 1
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.278.16"
version = "0.278.17"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"