mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix(api): sign final MP4s in GET /api/jobs/{id} download_urls (v0.278.1) (#1179)
## Summary
`GET /api/jobs/{id}` has always returned an empty `download_urls` for
real jobs. `JobManager.get_output_urls` signs only `job.output_files`,
and current workers never populate that field. Final videos live in
`file_urls.finals`.
This change also signs each `file_urls.finals` entry whose key ends in
`_mp4` (for example `lossy_720p_mp4`):
- URLs expire after 120 minutes.
- When a key exists in both, the `output_files` entry wins.
- Entries that aren't strings are skipped.
- After a real signing failure, it stops signing further entries, so a
stalled signer can't add a timeout per MP4.
**Why:** the mobile app (nomad-mobile Phase 1) streams the 720p MP4 in
`expo-video` and shows download progress. Both need a URL that supports
HTTP Range requests. The authenticated `/download/` proxy doesn't
support them.
Signing still goes through the bounded executor in `storage_service.py`
with its 12s timeout. The route is a sync `def`, so the event loop is
never blocked.
Version 0.278.1.
## Testing
- 6 new `TestGetOutputUrls` cases in
`backend/tests/test_job_manager.py`. That file passes: 58 tests.
- Local CodeRabbit raised one finding (the type guard), which is fixed.
@coderabbitai ignore
🤖 Generated with [Claude Code](https://claude.com/claude-code)
https://claude.ai/code/session_011yLH4kWXMqjeKfZUG3ZiWZ
---------
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
695b8ddf84
commit
1616bc997b
4 files changed
+97
-9
No files matched your search
@@ -376,20 +376,43 @@ class JobManager:
|
||||
logger.error(f"Job {job_id} marked as error: {error_message}")
|
||||
|
||||
def get_output_urls(self, job_id: str) -> Dict[str, str]:
|
||||
"""Generate signed URLs for job output files."""
|
||||
"""Generate signed URLs (120 min) for a job's downloadable outputs.
|
||||
|
||||
Signs legacy ``job.output_files`` plus the final MP4 videos in
|
||||
``file_urls['finals']`` (keys ending ``_mp4``, e.g. ``lossy_720p_mp4``) -
|
||||
current workers only populate the latter. ``output_files`` wins on key
|
||||
collision. Signed URLs support HTTP Range, so clients can stream them.
|
||||
Blobs are not existence-checked (a purged final yields a URL that 404s).
|
||||
"""
|
||||
job = self.get_job(job_id)
|
||||
if not job or not job.output_files:
|
||||
if not job:
|
||||
return {}
|
||||
|
||||
urls = {}
|
||||
for file_type, gcs_path in job.output_files.items():
|
||||
|
||||
urls: Dict[str, str] = {}
|
||||
for file_type, gcs_path in (job.output_files or {}).items():
|
||||
try:
|
||||
urls[file_type] = self.storage.generate_signed_url(gcs_path, expiration_minutes=120)
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating URL for {file_type}: {e}")
|
||||
|
||||
|
||||
finals = (job.file_urls or {}).get("finals")
|
||||
if isinstance(finals, dict):
|
||||
for key, gcs_path in finals.items():
|
||||
if not isinstance(key, str) or not isinstance(gcs_path, str):
|
||||
continue
|
||||
if not key.endswith("_mp4") or key in urls or not gcs_path:
|
||||
continue
|
||||
try:
|
||||
urls[key] = self.storage.generate_signed_url(gcs_path, expiration_minutes=120)
|
||||
except Exception as e:
|
||||
# Signing is bounded per call (SIGNED_URL_TIMEOUT_S); if it
|
||||
# failed, IAM is likely stalled - don't stack more timeouts
|
||||
# onto this request.
|
||||
logger.error(f"Error generating URL for final {key}: {e}")
|
||||
break
|
||||
|
||||
return urls
|
||||
|
||||
|
||||
def delete_job(self, job_id: str, delete_files: bool = True) -> None:
|
||||
"""Delete a job, its files, and its logs subcollection."""
|
||||
job = self.get_job(job_id)
|
||||
|
||||
@@ -1117,3 +1117,67 @@ class TestAtomicNestedUpdates:
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
|
||||
|
||||
class TestGetOutputUrls:
|
||||
"""get_output_urls signs output_files plus the final MP4s in file_urls['finals']."""
|
||||
|
||||
@pytest.fixture
|
||||
def signer(self, job_manager):
|
||||
job_manager.storage = Mock()
|
||||
job_manager.storage.generate_signed_url.side_effect = lambda path, expiration_minutes=60: f"signed://{path}?exp={expiration_minutes}"
|
||||
return job_manager.storage
|
||||
|
||||
def _job(self, job_manager, **kwargs):
|
||||
job = Mock(output_files=kwargs.get("output_files", {}), file_urls=kwargs.get("file_urls", {}))
|
||||
job_manager.get_job = Mock(return_value=job)
|
||||
|
||||
def test_signs_only_mp4_finals(self, job_manager, signer):
|
||||
self._job(job_manager, file_urls={"finals": {
|
||||
"lossy_720p_mp4": "jobs/j/finals/720.mp4",
|
||||
"lossy_4k_mp4": "jobs/j/finals/4k.mp4",
|
||||
"lossless_4k_mkv": "jobs/j/finals/4k.mkv",
|
||||
"cdg_zip": "jobs/j/finals/x.zip",
|
||||
}})
|
||||
urls = job_manager.get_output_urls("j")
|
||||
assert set(urls) == {"lossy_720p_mp4", "lossy_4k_mp4"}
|
||||
assert urls["lossy_720p_mp4"] == "signed://jobs/j/finals/720.mp4?exp=120"
|
||||
|
||||
def test_output_files_win_on_collision(self, job_manager, signer):
|
||||
self._job(
|
||||
job_manager,
|
||||
output_files={"lossy_720p_mp4": "legacy/720.mp4", "other": "legacy/o.bin"},
|
||||
file_urls={"finals": {"lossy_720p_mp4": "jobs/j/finals/720.mp4"}},
|
||||
)
|
||||
urls = job_manager.get_output_urls("j")
|
||||
assert urls["lossy_720p_mp4"] == "signed://legacy/720.mp4?exp=120"
|
||||
assert urls["other"] == "signed://legacy/o.bin?exp=120"
|
||||
assert signer.generate_signed_url.call_count == 2
|
||||
|
||||
def test_empty_when_nothing_to_sign(self, job_manager, signer):
|
||||
self._job(job_manager)
|
||||
assert job_manager.get_output_urls("j") == {}
|
||||
self._job(job_manager, file_urls={"finals": None, "videos": {"a_mp4": "x"}})
|
||||
assert job_manager.get_output_urls("j") == {}
|
||||
signer.generate_signed_url.assert_not_called()
|
||||
|
||||
def test_missing_job(self, job_manager, signer):
|
||||
job_manager.get_job = Mock(return_value=None)
|
||||
assert job_manager.get_output_urls("j") == {}
|
||||
|
||||
def test_signing_failure_skips_entry_without_raising(self, job_manager, signer):
|
||||
signer.generate_signed_url.side_effect = TimeoutError("iam stalled")
|
||||
self._job(job_manager, file_urls={"finals": {"lossy_720p_mp4": "p", "lossy_4k_mp4": "q"}})
|
||||
assert job_manager.get_output_urls("j") == {}
|
||||
# Fail fast: a stalled signer is not retried for every remaining final
|
||||
assert signer.generate_signed_url.call_count == 1
|
||||
|
||||
def test_malformed_final_entry_skipped_not_fatal(self, job_manager, signer):
|
||||
self._job(job_manager, file_urls={"finals": {
|
||||
"bad_none_mp4": None,
|
||||
"bad_dict_mp4": {"x": 1},
|
||||
"lossy_720p_mp4": "jobs/j/finals/720.mp4",
|
||||
}})
|
||||
urls = job_manager.get_output_urls("j")
|
||||
assert urls == {"lossy_720p_mp4": "signed://jobs/j/finals/720.mp4?exp=120"}
|
||||
assert signer.generate_signed_url.call_count == 1
|
||||
+2
-1
@@ -259,7 +259,8 @@ GET /api/jobs/{job_id}
|
||||
|
||||
Response includes:
|
||||
- `status` - Current job state
|
||||
- `file_urls` - Signed URLs for outputs (when ready)
|
||||
- `file_urls` - GCS paths of outputs (when ready)
|
||||
- `download_urls` - (complete jobs only) short-lived (120 min) signed GCS URLs, keyed by output name. Includes the final MP4s from `file_urls.finals` keyed by finals key (e.g. `lossy_720p_mp4`, `lossy_4k_mp4`, `with_vocals_mp4`); non-MP4 finals (mkv, zips) are not included. Signed URLs support HTTP Range (streaming/progress) unlike the `/download/...` proxy. Purged outputs still get a URL that returns 404
|
||||
- `state_data` - Worker progress details
|
||||
- `timeline` - Processing history
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.278.0"
|
||||
version = "0.278.1"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in new issue
Block a user