mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix(review): preview shows the countdown intro like the final render (v0.296.0) (#1264)
## Summary
Songs whose vocals start within 3s get a "3... 2... 1..." intro in the
final render (`CountdownProcessor`: lyrics +3s, 3s leading silence), but
the **review preview never applied it**. So the preview opened straight
on vocals, and a customer (Randy Vild, "Caught Up Alive" / "Faithful and
True") asked us to add a countdown that was already coming.
## Changes
- `CountdownProcessor.add_countdown_to_lyrics_if_needed` (lyrics-only
half of `process`) + `padding_for_segments_data` (same rule on the
serialized segments the client sends).
- `CorrectionOperations.generate_preview_video` applies the countdown;
full local/CLI renders pad the audio too. Returns
`countdown_padding_seconds`. Countdown previews get a salted hash so
pre-fix cached previews aren't reused.
- GCE preview path: pads the already-downloaded audio on the API side
and uploads it to `previews/{hash}_audio.flac` for the encoder, so **no
encoding-worker change / version skew**.
- `POST /preview-video` responses carry `countdown_padding_seconds`.
- `PreviewVideoSection`: video time = song time + offset, so the
overlaid instrumental stem, waveform playhead and audition seeks stay
aligned (stem parked at 0 during the intro).
- Corrections that already contain the countdown aren't shifted again
but still get padded audio (CodeRabbit finding).
## Testing
- New:
`tests/unit/lyrics_transcriber/output/test_countdown_preview_helpers.py`,
`tests/unit/lyrics_transcriber/correction/test_operations_preview_countdown.py`,
GCE-path endpoint test in `backend/tests/test_routes_review.py`, 4
offset tests in `PreviewVideoSection.test.tsx`.
- Frontend jest (lyrics-review + lib): 892 passed.
- Real render check: local preview of job 508b38b2 (Faithful and True)
from prod corrections + audio with real ffmpeg shows the countdown, and
silence runs to 3.80s, matching the delivered final video's first word
(3.795s).
- Local CodeRabbit review: 2 findings fixed, re-review clean.
@coderabbitai ignore
🤖 Generated with [Claude Code](https://claude.com/claude-code)
https://claude.ai/code/session_012RYLVVUdF2J1bNeVa2uLXA
---------
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ac844b3933
commit
13619b7cc9
12 files changed
+483
-23
No files matched your search
@@ -202,6 +202,7 @@ from karaoke_gen.lyrics_transcriber.types import CorrectionResult
|
||||
from karaoke_gen.lyrics_transcriber.core.config import OutputConfig
|
||||
from karaoke_gen.lyrics_transcriber.correction.operations import CorrectionOperations
|
||||
from karaoke_gen.lyrics_transcriber.output.timing_validation import LyricsTimingError
|
||||
from karaoke_gen.lyrics_transcriber.output.countdown_processor import CountdownProcessor
|
||||
|
||||
# Import from the unified style loader
|
||||
from karaoke_gen.style_loader import load_styles_from_gcs
|
||||
@@ -2058,6 +2059,14 @@ async def generate_preview_video(
|
||||
if untimed_line is not None:
|
||||
raise HTTPException(status_code=422, detail=t("en", "review.previewMissingTimings", line=untimed_line[:80]))
|
||||
|
||||
# Songs whose vocals start within 3s get a countdown intro in the preview,
|
||||
# exactly as in the final render, which shifts the preview's timeline by this
|
||||
# many seconds. The client needs it to keep the overlaid instrumental stem and
|
||||
# waveform playhead aligned with the video.
|
||||
countdown_padding_seconds = CountdownProcessor.padding_for_segments_data(
|
||||
updated_data.get("corrected_segments")
|
||||
)
|
||||
|
||||
# Check if GCE preview encoding is enabled
|
||||
use_gce_preview = encoding_service.is_preview_enabled
|
||||
|
||||
@@ -2123,7 +2132,7 @@ async def generate_preview_video(
|
||||
_record_preview_video(job_id, preview_hash, preview_gcs_path)
|
||||
span.set_attribute("preview_cache_hit", True)
|
||||
logger.info(f"Job {job_id}: Preview cache hit: {preview_hash}")
|
||||
return {"status": "success", "preview_hash": preview_hash}
|
||||
return {"status": "success", "preview_hash": preview_hash, "countdown_padding_seconds": countdown_padding_seconds}
|
||||
|
||||
# Upload ASS to GCS
|
||||
with create_span("upload-ass-to-gcs") as upload_ass_span:
|
||||
@@ -2141,6 +2150,23 @@ async def generate_preview_video(
|
||||
bucket_name = settings.gcs_bucket_name
|
||||
style_assets = job.style_assets or {}
|
||||
|
||||
# The ASS has the countdown intro (lyrics shifted +Ns);
|
||||
# the encoder needs audio padded by the same N seconds,
|
||||
# as the final render does. Pad here and hand the
|
||||
# encoder that file, so the worker needs no changes.
|
||||
audio_gcs_path = job.input_media_gcs_path
|
||||
if result.get("countdown_padding_seconds"):
|
||||
with create_span("pad-preview-audio-for-countdown"):
|
||||
padded_audio_path = await asyncio.to_thread(
|
||||
CountdownProcessor(
|
||||
cache_dir=output_config.cache_dir, logger=logger
|
||||
).create_padded_audio_only,
|
||||
audio_path,
|
||||
)
|
||||
audio_gcs_path = f"jobs/{job_id}/previews/{preview_hash}_audio.flac"
|
||||
await asyncio.to_thread(storage.upload_file, padded_audio_path, audio_gcs_path)
|
||||
span.set_attribute("countdown_padding_seconds", countdown_padding_seconds)
|
||||
|
||||
# Only use background image if user explicitly requested it.
|
||||
# Default is black background for faster preview generation.
|
||||
background_image_gcs_path = None
|
||||
@@ -2160,7 +2186,7 @@ async def generate_preview_video(
|
||||
encode_kwargs = {
|
||||
"job_id": f"preview_{job_id}_{preview_hash}",
|
||||
"ass_gcs_path": f"gs://{bucket_name}/{ass_gcs_path}",
|
||||
"audio_gcs_path": f"gs://{bucket_name}/{job.input_media_gcs_path}",
|
||||
"audio_gcs_path": f"gs://{bucket_name}/{audio_gcs_path}",
|
||||
"output_gcs_path": f"gs://{bucket_name}/{preview_gcs_path}",
|
||||
"background_color": "black",
|
||||
"background_image_gcs_path": background_image_gcs_path,
|
||||
@@ -2179,7 +2205,7 @@ async def generate_preview_video(
|
||||
task.add_done_callback(_background_tasks.discard)
|
||||
|
||||
logger.info(f"Job {job_id}: Preview encode running in background: {preview_hash}")
|
||||
return {"status": "generating", "preview_hash": preview_hash}
|
||||
return {"status": "generating", "preview_hash": preview_hash, "countdown_padding_seconds": countdown_padding_seconds}
|
||||
|
||||
except LyricsTimingError:
|
||||
raise
|
||||
@@ -2199,7 +2225,7 @@ async def generate_preview_video(
|
||||
span.set_attribute("preview_hash", preview_hash)
|
||||
span.set_attribute("success", True)
|
||||
|
||||
return {"status": "success", "preview_hash": preview_hash}
|
||||
return {"status": "success", "preview_hash": preview_hash, "countdown_padding_seconds": countdown_padding_seconds}
|
||||
|
||||
except LyricsTimingError as timing_err:
|
||||
# Untimed lyrics (e.g. custom lyrics not yet tap-synced) — this is a
|
||||
|
||||
@@ -1835,7 +1835,9 @@ class TestPreviewVideoAsyncFlow:
|
||||
job.title = "Test Song"
|
||||
return job
|
||||
|
||||
def _run_gce_preview(self, mock_job, *, mp4_exists, background_mock):
|
||||
def _run_gce_preview(self, mock_job, *, mp4_exists, background_mock,
|
||||
updated_data=None, countdown_padding_seconds=None,
|
||||
countdown_processor_cls=None):
|
||||
"""Run generate_preview_video with GCE enabled and heavy deps mocked."""
|
||||
import asyncio
|
||||
from backend.api.routes.review import generate_preview_video
|
||||
@@ -1855,7 +1857,7 @@ class TestPreviewVideoAsyncFlow:
|
||||
async def _run():
|
||||
result = await generate_preview_video(
|
||||
job_id=mock_job.job_id,
|
||||
updated_data={},
|
||||
updated_data=updated_data if updated_data is not None else {},
|
||||
auth_info=("user@test.com", "job_owner"),
|
||||
)
|
||||
# Let any create_task'd background coroutine start
|
||||
@@ -1877,6 +1879,14 @@ class TestPreviewVideoAsyncFlow:
|
||||
"preview_hash": "hash123",
|
||||
"ass_path": "/tmp/preview.ass",
|
||||
}
|
||||
if countdown_padding_seconds is not None:
|
||||
mock_co.generate_preview_video.return_value["countdown_padding_seconds"] = countdown_padding_seconds
|
||||
if countdown_processor_cls is not None:
|
||||
with patch("backend.api.routes.review.CountdownProcessor") as mock_cp:
|
||||
mock_cp.padding_for_segments_data.side_effect = countdown_processor_cls.padding_for_segments_data
|
||||
mock_cp.return_value.create_padded_audio_only.return_value = "/tmp/audio_padded.flac"
|
||||
result = asyncio.run(_run())
|
||||
return result, mock_storage, mock_co
|
||||
result = asyncio.run(_run())
|
||||
return result, mock_storage, mock_co
|
||||
|
||||
@@ -1886,7 +1896,7 @@ class TestPreviewVideoAsyncFlow:
|
||||
background_mock = AsyncMock()
|
||||
result, mock_storage, mock_co = self._run_gce_preview(
|
||||
mock_job, mp4_exists=True, background_mock=background_mock)
|
||||
assert result == {"status": "success", "preview_hash": "hash123"}
|
||||
assert result == {"status": "success", "preview_hash": "hash123", "countdown_padding_seconds": 0.0}
|
||||
background_mock.assert_not_called()
|
||||
# ASS is generated with ass_only=True (no local video render)
|
||||
assert mock_co.generate_preview_video.call_args.kwargs["ass_only"] is True
|
||||
@@ -1897,7 +1907,7 @@ class TestPreviewVideoAsyncFlow:
|
||||
background_mock = AsyncMock()
|
||||
result, mock_storage, _ = self._run_gce_preview(
|
||||
mock_job, mp4_exists=False, background_mock=background_mock)
|
||||
assert result == {"status": "generating", "preview_hash": "hash123"}
|
||||
assert result == {"status": "generating", "preview_hash": "hash123", "countdown_padding_seconds": 0.0}
|
||||
background_mock.assert_called_once()
|
||||
kwargs = background_mock.call_args.kwargs
|
||||
assert kwargs["preview_hash"] == "hash123"
|
||||
@@ -1907,6 +1917,29 @@ class TestPreviewVideoAsyncFlow:
|
||||
# Stale error marker from a previous failed attempt must be cleared
|
||||
mock_storage.delete_file.assert_called_once_with(
|
||||
"jobs/job-preview-async/previews/hash123.error.json", ignore_missing=True)
|
||||
# No countdown → the encoder uses the original input audio as-is.
|
||||
assert kwargs["encode_kwargs"]["audio_gcs_path"] == \
|
||||
"gs://test-bucket/jobs/job-preview-async/audio/input.flac"
|
||||
|
||||
def test_countdown_song_uploads_padded_audio_and_reports_offset(self, mock_job):
|
||||
"""Vocals within 3s: the preview's ASS carries the countdown intro (lyrics
|
||||
shifted +3s), so the encoder must get audio padded by the same 3s, and the
|
||||
client must be told the offset to keep the stem/playhead aligned."""
|
||||
from karaoke_gen.lyrics_transcriber.output.countdown_processor import CountdownProcessor
|
||||
updated_data = {"corrected_segments": [
|
||||
{"text": "hello", "words": [{"text": "hello", "start_time": 0.4, "end_time": 0.9}]},
|
||||
]}
|
||||
background_mock = AsyncMock()
|
||||
result, mock_storage, _ = self._run_gce_preview(
|
||||
mock_job, mp4_exists=False, background_mock=background_mock,
|
||||
updated_data=updated_data, countdown_padding_seconds=3.0,
|
||||
countdown_processor_cls=CountdownProcessor)
|
||||
assert result == {"status": "generating", "preview_hash": "hash123", "countdown_padding_seconds": 3.0}
|
||||
mock_storage.upload_file.assert_any_call(
|
||||
"/tmp/audio_padded.flac", "jobs/job-preview-async/previews/hash123_audio.flac")
|
||||
kwargs = background_mock.call_args.kwargs
|
||||
assert kwargs["encode_kwargs"]["audio_gcs_path"] == \
|
||||
"gs://test-bucket/jobs/job-preview-async/previews/hash123_audio.flac"
|
||||
|
||||
|
||||
class TestPreviewVideoStatusEndpoint:
|
||||
|
||||
@@ -477,6 +477,8 @@ Content-Type: application/json
|
||||
|
||||
Generates a preview video during review. Returns `{"status": "success", "preview_hash": "..."}` when the video is immediately available (cache hit — the hash is deterministic over the correction data — or local render), or `{"status": "generating", "preview_hash": "..."}` when encoding continues in the background on a GCE worker. In the generating case, poll the status endpoint below; the request never waits for the encoder (cold-starting a worker VM can take minutes, longer than Cloudflare's edge timeout).
|
||||
|
||||
Both responses also carry `countdown_padding_seconds` (3.0 or 0.0). When the first word starts within 3s the preview gets the same "3... 2... 1..." intro as the final render: lyrics shifted and audio padded by that many seconds, so video time = song time + `countdown_padding_seconds`. Clients syncing other audio (e.g. instrumental stems) to the preview must subtract it.
|
||||
|
||||
The `use_background_image` option controls whether the preview renders with the theme's background image (slower, ~30-60s) or a solid black background (faster, ~10s). Defaults to black background for speed.
|
||||
|
||||
#### Poll Preview Status
|
||||
|
||||
@@ -61,6 +61,7 @@
|
||||
|
||||
## Recent Changes
|
||||
|
||||
- **Review preview shows the countdown intro** (2026-11-11, v0.296.0). Songs whose vocals start within 3s got the "3... 2... 1..." intro (+3s lyrics shift, 3s leading silence) only in the final render, so the review preview opened straight on vocals and customers asked for a countdown that was already coming (Randy Vild, Nov 2026). `CorrectionOperations.generate_preview_video` now applies `CountdownProcessor.add_countdown_to_lyrics_if_needed` (local/CLI renders also pad the audio; the GCE preview path uploads padded audio to `previews/{hash}_audio.flac`, so the encoder worker is unchanged). The endpoint returns `countdown_padding_seconds`; `PreviewVideoSection` maps video time to song time so the instrumental stem and waveform playhead stay aligned. Countdown previews hash with a salt so pre-fix cached previews aren't reused.
|
||||
- **Guitar chords in karaoke videos** (2026-10-16, v0.277.0). `OutputGenerator.generate_outputs(chords=[ChordChange(...)])` draws each chord above the word it lands on (it lights up when played) in a row reserved above every line, plus a "now / next" chord box in the top-right corner that also covers intros and instrumental breaks. Chords come from `scripts/medley/detect_chords.py`, an ensemble of ChordMini models over the mix, instrumental and harmonic stems. It's used by the medley script for now (`"chords": true`); there's no job/UI option yet. Theme keys: `chord_color`, `chord_font_size`/`chord_font_ratio`, `chord_max_visible_lines`. See [MEDLEYS.md § Guitar chords](MEDLEYS.md#guitar-chords-optional).
|
||||
- **Customer chord charts + medley mastering fix** (2026-10-17, v0.278.5). `make_medley.py chart` turns a customer's chords-over-lyrics chart (.txt or .docx, capo shapes kept, `ChordChange.capo` shown in the chord box) into timed chords on the reviewed lyrics, with detected chords filling intros and breaks (`scripts/medley/chord_chart.py`). The medley master is now one static gain + limiter instead of single-pass `loudnorm`, which had boosted a near-silent a cappella intro into robotic artifacts. See [MEDLEYS.md](MEDLEYS.md#the-customers-own-chord-chart-chart-step).
|
||||
- **Local E2E stack + email-code sign-in** (2026-10-15, v0.275.0). Plan: [archive/2026-10-15-local-stack-e2e-plan.md](archive/2026-10-15-local-stack-e2e-plan.md).
|
||||
|
||||
@@ -30,6 +30,8 @@ interface ApiClient {
|
||||
status: string
|
||||
message?: string
|
||||
preview_hash?: string
|
||||
/** Seconds of countdown intro prepended to the preview (0 when none). */
|
||||
countdown_padding_seconds?: number
|
||||
}>
|
||||
getPreviewVideoStatus: (hash: string) => Promise<{
|
||||
status: string
|
||||
@@ -121,6 +123,11 @@ function PreviewVideoSection(
|
||||
}, [])
|
||||
|
||||
const internalVideoRef = useRef<HTMLVideoElement | null>(null)
|
||||
// Seconds of countdown intro ("3... 2... 1...") the backend prepended to this
|
||||
// preview, exactly as the final render does for songs whose vocals start within
|
||||
// 3s. The video's timeline is song time + this offset; the instrumental stem and
|
||||
// the waveform playhead run on song time, so every sync point converts.
|
||||
const introOffsetRef = useRef(0)
|
||||
const instrumentalAudioRef = useRef<HTMLAudioElement | null>(null)
|
||||
const isInstrumentalRef = useRef(false)
|
||||
isInstrumentalRef.current = isInstrumental
|
||||
@@ -174,7 +181,7 @@ function PreviewVideoSection(
|
||||
if (el && isInstrumentalRef.current) {
|
||||
const audio = instrumentalAudioRef.current
|
||||
if (audio && !audio.paused && audio.currentTime > 0) {
|
||||
el.currentTime = audio.currentTime
|
||||
el.currentTime = audio.currentTime + introOffsetRef.current
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -241,6 +248,7 @@ function PreviewVideoSection(
|
||||
|
||||
const response = await apiClient.generatePreviewVideo(dataToPreview, isDuet)
|
||||
if (cancelled) return
|
||||
introOffsetRef.current = Math.max(0, Number(response.countdown_padding_seconds) || 0)
|
||||
|
||||
if (response.status === 'error' || !response.preview_hash) {
|
||||
setPreviewState({
|
||||
@@ -362,6 +370,23 @@ function PreviewVideoSection(
|
||||
if (!video) return
|
||||
|
||||
const instrumentalActive = () => isInstrumentalRef.current && !!audio
|
||||
// Song time for the current video position (negative during the countdown intro).
|
||||
const songTime = () => video.currentTime - introOffsetRef.current
|
||||
// Put the stem where the video is: parked at 0 during the intro (silence in the
|
||||
// video too), playing from the matching song position after it.
|
||||
const syncAudio = (forceSeek: boolean) => {
|
||||
if (!audio) return
|
||||
const target = songTime()
|
||||
if (target < 0) {
|
||||
if (!audio.paused) audio.pause()
|
||||
if (audio.currentTime !== 0) audio.currentTime = 0
|
||||
return
|
||||
}
|
||||
if (forceSeek || Math.abs(audio.currentTime - target) > AUDIO_SYNC_TOLERANCE_S) {
|
||||
audio.currentTime = target
|
||||
}
|
||||
if (!video.paused && audio.paused) audio.play()?.catch(() => {})
|
||||
}
|
||||
|
||||
const applyMode = () => {
|
||||
const active = instrumentalActive()
|
||||
@@ -370,10 +395,7 @@ function PreviewVideoSection(
|
||||
audio.muted = !active
|
||||
if (active) {
|
||||
audio.playbackRate = video.playbackRate
|
||||
if (Math.abs(audio.currentTime - video.currentTime) > AUDIO_SYNC_TOLERANCE_S) {
|
||||
audio.currentTime = video.currentTime
|
||||
}
|
||||
if (!video.paused) audio.play()?.catch(() => {})
|
||||
syncAudio(false)
|
||||
} else {
|
||||
audio.pause()
|
||||
}
|
||||
@@ -381,22 +403,21 @@ function PreviewVideoSection(
|
||||
|
||||
const onPlay = () => {
|
||||
if (!audio || !instrumentalActive()) return
|
||||
audio.currentTime = video.currentTime
|
||||
audio.play()?.catch(() => {})
|
||||
syncAudio(true)
|
||||
}
|
||||
const onPause = () => audio?.pause()
|
||||
const onSeeked = () => {
|
||||
if (audio) audio.currentTime = video.currentTime
|
||||
if (!audio) return
|
||||
if (instrumentalActive()) syncAudio(true)
|
||||
else audio.currentTime = Math.max(0, songTime())
|
||||
}
|
||||
const onRateChange = () => {
|
||||
if (audio) audio.playbackRate = video.playbackRate
|
||||
}
|
||||
const onTimeUpdateEvt = () => {
|
||||
onTimeUpdate?.(video.currentTime)
|
||||
onTimeUpdate?.(Math.max(0, songTime()))
|
||||
if (!audio || !instrumentalActive()) return
|
||||
if (Math.abs(audio.currentTime - video.currentTime) > AUDIO_SYNC_TOLERANCE_S) {
|
||||
audio.currentTime = video.currentTime
|
||||
}
|
||||
syncAudio(false)
|
||||
}
|
||||
|
||||
applyMode()
|
||||
@@ -453,7 +474,7 @@ function PreviewVideoSection(
|
||||
const video = internalVideoRef.current
|
||||
if (video) {
|
||||
if (seekTo != null) {
|
||||
video.currentTime = seekTo
|
||||
video.currentTime = seekTo + introOffsetRef.current
|
||||
video.play()?.catch(() => {})
|
||||
}
|
||||
return
|
||||
|
||||
@@ -613,4 +613,91 @@ describe('PreviewVideoSection', () => {
|
||||
})
|
||||
expect(audio.src).toBe('http://test/backing.ogg')
|
||||
})
|
||||
|
||||
describe('countdown intro (vocals start within 3s)', () => {
|
||||
// The backend prepends the same "3... 2... 1..." intro the final render adds and
|
||||
// reports its length; the video timeline is song time + that offset, while the
|
||||
// stem and waveform playhead stay on song time.
|
||||
const renderWithCountdown = (onTimeUpdate?: jest.Mock) => {
|
||||
const apiClient = makeApiClient({
|
||||
generatePreviewVideo: jest.fn().mockResolvedValue({
|
||||
status: 'success',
|
||||
preview_hash: 'hash-countdown',
|
||||
countdown_padding_seconds: 3,
|
||||
}),
|
||||
})
|
||||
const ref = createRef<PreviewVideoHandle>()
|
||||
render(
|
||||
<PreviewVideoSection
|
||||
ref={ref}
|
||||
apiClient={apiClient}
|
||||
isModalOpen={true}
|
||||
updatedData={data}
|
||||
instrumentalOptions={instrumentalOptions as any}
|
||||
autoSelection="with_backing"
|
||||
onTimeUpdate={onTimeUpdate}
|
||||
/>
|
||||
)
|
||||
return ref
|
||||
}
|
||||
|
||||
it('seeks the video past the intro when auditioning at a song time', async () => {
|
||||
const ref = renderWithCountdown()
|
||||
await flush()
|
||||
act(() => {
|
||||
ref.current!.auditionInstrumental('clean', 5)
|
||||
})
|
||||
await flush()
|
||||
const video = document.querySelector('video') as HTMLVideoElement
|
||||
expect(video.currentTime).toBe(8)
|
||||
})
|
||||
|
||||
it('reports song time (video time minus the intro) to the playhead', async () => {
|
||||
const onTimeUpdate = jest.fn()
|
||||
renderWithCountdown(onTimeUpdate)
|
||||
await flush()
|
||||
const video = document.querySelector('video') as HTMLVideoElement
|
||||
video.currentTime = 10
|
||||
fireEvent.timeUpdate(video)
|
||||
expect(onTimeUpdate).toHaveBeenLastCalledWith(7)
|
||||
// During the intro the playhead stays at the start of the song.
|
||||
video.currentTime = 1
|
||||
fireEvent.timeUpdate(video)
|
||||
expect(onTimeUpdate).toHaveBeenLastCalledWith(0)
|
||||
})
|
||||
|
||||
it('keeps the stem on song time when the video seeks, parked at 0 during the intro', async () => {
|
||||
const ref = renderWithCountdown()
|
||||
await flush()
|
||||
act(() => {
|
||||
ref.current!.auditionInstrumental('with_backing')
|
||||
})
|
||||
await flush()
|
||||
const video = document.querySelector('video') as HTMLVideoElement
|
||||
const audio = document.querySelector('audio') as HTMLAudioElement
|
||||
video.currentTime = 11
|
||||
fireEvent.seeked(video)
|
||||
expect(audio.currentTime).toBe(8)
|
||||
video.currentTime = 2
|
||||
fireEvent.seeked(video)
|
||||
expect(audio.currentTime).toBe(0)
|
||||
})
|
||||
|
||||
it('uses no offset when the backend reports no countdown', async () => {
|
||||
const onTimeUpdate = jest.fn()
|
||||
render(
|
||||
<PreviewVideoSection
|
||||
apiClient={makeApiClient()}
|
||||
isModalOpen={true}
|
||||
updatedData={data}
|
||||
onTimeUpdate={onTimeUpdate}
|
||||
/>
|
||||
)
|
||||
await flush()
|
||||
const video = document.querySelector('video') as HTMLVideoElement
|
||||
video.currentTime = 10
|
||||
fireEvent.timeUpdate(video)
|
||||
expect(onTimeUpdate).toHaveBeenLastCalledWith(10)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -4459,6 +4459,8 @@ export function createLyricsReviewApiClient(jobId: string): LyricsReviewApiClien
|
||||
status: string
|
||||
message?: string
|
||||
preview_hash?: string
|
||||
/** Seconds of countdown intro prepended to the preview (0 when none). */
|
||||
countdown_padding_seconds?: number
|
||||
}> {
|
||||
const body: Record<string, unknown> = { ...data }
|
||||
if (isDuet !== undefined) {
|
||||
|
||||
@@ -28,6 +28,7 @@ from karaoke_gen.lyrics_transcriber.types import (
|
||||
LyricsData
|
||||
)
|
||||
from karaoke_gen.lyrics_transcriber.correction.corrector import LyricsCorrector
|
||||
from karaoke_gen.lyrics_transcriber.output.countdown_processor import CountdownProcessor
|
||||
from karaoke_gen.lyrics_transcriber.lyrics.user_input_provider import UserInputProvider
|
||||
from karaoke_gen.lyrics_transcriber.lyrics.base_lyrics_provider import LyricsProviderConfig, BaseLyricsProvider
|
||||
from karaoke_gen.lyrics_transcriber.output.generator import OutputGenerator
|
||||
@@ -536,8 +537,27 @@ class CorrectionOperations:
|
||||
correction_result, updated_data
|
||||
)
|
||||
|
||||
# Generate a unique hash for this preview
|
||||
# Apply the same countdown intro the final render adds (CountdownProcessor
|
||||
# in the encoding worker): songs whose vocals start within 3s get the
|
||||
# lyrics shifted +3s and a "3... 2... 1..." line. Without this the preview
|
||||
# starts singing immediately while the final video has a cue, so reviewers
|
||||
# think the countdown is missing. Audio padding is done here for the full
|
||||
# render; ass_only callers get countdown_padding_seconds and must pad the
|
||||
# audio they hand to the encoder themselves.
|
||||
countdown_processor = CountdownProcessor(cache_dir=output_config.cache_dir, logger=logger)
|
||||
temp_correction, countdown_padding_seconds = countdown_processor.add_countdown_to_lyrics_if_needed(
|
||||
temp_correction
|
||||
)
|
||||
if countdown_padding_seconds:
|
||||
if not ass_only:
|
||||
audio_filepath = countdown_processor.create_padded_audio_only(audio_filepath)
|
||||
logger.info(f"Preview: added {countdown_padding_seconds}s countdown intro")
|
||||
|
||||
# Generate a unique hash for this preview. Salted when a countdown is
|
||||
# applied so previews cached before countdown support aren't reused.
|
||||
preview_data = json.dumps(updated_data, sort_keys=True).encode("utf-8")
|
||||
if countdown_padding_seconds:
|
||||
preview_data += b"|countdown-v1"
|
||||
preview_hash = hashlib.md5(preview_data).hexdigest()[:12]
|
||||
|
||||
# Propagate duet flag from the outer config OR the request body.
|
||||
@@ -589,6 +609,7 @@ class CorrectionOperations:
|
||||
"status": "success",
|
||||
"preview_hash": preview_hash,
|
||||
"ass_path": preview_outputs.ass,
|
||||
"countdown_padding_seconds": countdown_padding_seconds,
|
||||
}
|
||||
|
||||
if not preview_outputs.video:
|
||||
@@ -600,4 +621,5 @@ class CorrectionOperations:
|
||||
"status": "success",
|
||||
"preview_hash": preview_hash,
|
||||
"video_path": preview_outputs.video,
|
||||
"countdown_padding_seconds": countdown_padding_seconds,
|
||||
}
|
||||
@@ -90,6 +90,54 @@ class CountdownProcessor:
|
||||
|
||||
return modified_result, padded_audio_path, True, self.COUNTDOWN_PADDING_SECONDS
|
||||
|
||||
@classmethod
|
||||
def padding_for_segments_data(cls, segments: Optional[List[dict]]) -> float:
|
||||
"""
|
||||
Countdown padding (seconds) a render of these serialized segments gets.
|
||||
|
||||
Same rule as ``process``/``add_countdown_to_lyrics_if_needed`` but on the
|
||||
plain ``corrected_segments`` dicts a review client sends, so an API can
|
||||
tell the client how far the preview's timeline is shifted without
|
||||
rebuilding a CorrectionResult. A countdown that is already present still
|
||||
means a shifted timeline (and padded audio). 0.0 if no countdown applies.
|
||||
"""
|
||||
if not segments:
|
||||
return 0.0
|
||||
if (segments[0] or {}).get("text") == cls.COUNTDOWN_TEXT:
|
||||
return cls.COUNTDOWN_PADDING_SECONDS
|
||||
for segment in segments:
|
||||
words = (segment or {}).get("words") or []
|
||||
if words:
|
||||
try:
|
||||
first_start = float(words[0].get("start_time"))
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
return cls.COUNTDOWN_PADDING_SECONDS if first_start < cls.COUNTDOWN_THRESHOLD_SECONDS else 0.0
|
||||
return 0.0
|
||||
|
||||
def add_countdown_to_lyrics_if_needed(
|
||||
self, correction_result: CorrectionResult
|
||||
) -> Tuple[CorrectionResult, float]:
|
||||
"""
|
||||
Lyrics-only half of ``process``: shift timings and add the countdown
|
||||
segment if the song needs one, without touching any audio.
|
||||
|
||||
Used by the review preview, which must match the final render but pads
|
||||
its audio separately (or has the encoder use pre-padded audio).
|
||||
Already-countdown-processed results are returned unchanged, but still
|
||||
report the padding: their timings are already shifted, so the audio
|
||||
must be padded to match.
|
||||
|
||||
Returns:
|
||||
(possibly modified CorrectionResult, padding seconds the caller must
|
||||
prepend to the audio — 0.0 when the song has no countdown intro)
|
||||
"""
|
||||
if self.has_countdown(correction_result):
|
||||
return correction_result, self.COUNTDOWN_PADDING_SECONDS
|
||||
if not self._needs_countdown(correction_result):
|
||||
return correction_result, 0.0
|
||||
return self._add_countdown_to_result(correction_result), self.COUNTDOWN_PADDING_SECONDS
|
||||
|
||||
def _needs_countdown(self, correction_result: CorrectionResult) -> bool:
|
||||
"""
|
||||
Check if the song needs a countdown intro.
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.295.1"
|
||||
version = "0.296.0"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
"""The review preview gets the same countdown intro as the final render.
|
||||
|
||||
Before this, a song whose vocals start within 3s showed a countdown in the
|
||||
final video (CountdownProcessor in the encoding worker) but none in the review
|
||||
preview, so reviewers asked for a countdown that was already coming.
|
||||
"""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from karaoke_gen.lyrics_transcriber.correction.operations import CorrectionOperations
|
||||
from karaoke_gen.lyrics_transcriber.core.config import OutputConfig
|
||||
from karaoke_gen.lyrics_transcriber.output.countdown_processor import CountdownProcessor
|
||||
from tests.unit.lyrics_transcriber.correction.test_operations_multisinger import _empty_correction_result
|
||||
|
||||
|
||||
def _updated_data(first_word_start: float) -> dict:
|
||||
return {
|
||||
"corrections": [],
|
||||
"corrected_segments": [{
|
||||
"id": "s1", "text": "hello", "start_time": first_word_start, "end_time": first_word_start + 0.5,
|
||||
"words": [{"id": "w1", "text": "hello", "start_time": first_word_start,
|
||||
"end_time": first_word_start + 0.5}],
|
||||
}],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_preview(monkeypatch, tmp_path):
|
||||
captured = {}
|
||||
|
||||
class _StubGenerator:
|
||||
def __init__(self, config, preview_mode=False, **kwargs):
|
||||
pass
|
||||
|
||||
def generate_outputs(self, transcription_corrected, audio_filepath, **kwargs):
|
||||
captured["correction"] = transcription_corrected
|
||||
captured["audio"] = audio_filepath
|
||||
out = MagicMock()
|
||||
out.video = str(tmp_path / "preview.mp4")
|
||||
out.ass = str(tmp_path / "preview.ass")
|
||||
return out
|
||||
|
||||
monkeypatch.setattr(
|
||||
"karaoke_gen.lyrics_transcriber.correction.operations.OutputGenerator", _StubGenerator
|
||||
)
|
||||
padded = str(tmp_path / "audio_padded.flac")
|
||||
pad_calls = []
|
||||
monkeypatch.setattr(
|
||||
CountdownProcessor, "create_padded_audio_only",
|
||||
lambda self, path: pad_calls.append(path) or padded,
|
||||
)
|
||||
(tmp_path / "previews").mkdir(exist_ok=True)
|
||||
cfg = OutputConfig(output_styles_json="", output_dir=str(tmp_path), cache_dir=str(tmp_path))
|
||||
|
||||
def _run(updated_data, ass_only):
|
||||
result = CorrectionOperations.generate_preview_video(
|
||||
correction_result=_empty_correction_result(),
|
||||
updated_data=updated_data,
|
||||
output_config=cfg,
|
||||
audio_filepath=str(tmp_path / "song.flac"),
|
||||
logger=MagicMock(),
|
||||
ass_only=ass_only,
|
||||
)
|
||||
return result, captured, pad_calls, padded
|
||||
|
||||
return _run
|
||||
|
||||
|
||||
def test_early_vocals_get_countdown_in_ass_only_preview(run_preview):
|
||||
result, captured, pad_calls, _ = run_preview(_updated_data(0.4), ass_only=True)
|
||||
assert result["countdown_padding_seconds"] == 3.0
|
||||
segs = captured["correction"].corrected_segments
|
||||
assert segs[0].text == CountdownProcessor.COUNTDOWN_TEXT
|
||||
assert segs[1].words[0].start_time == pytest.approx(3.4)
|
||||
# ass_only callers pad the encoder's audio themselves.
|
||||
assert pad_calls == []
|
||||
|
||||
|
||||
def test_full_local_render_pads_audio(run_preview):
|
||||
result, captured, pad_calls, padded = run_preview(_updated_data(0.4), ass_only=False)
|
||||
assert result["countdown_padding_seconds"] == 3.0
|
||||
assert len(pad_calls) == 1
|
||||
assert captured["audio"] == padded
|
||||
|
||||
|
||||
def test_late_vocals_untouched(run_preview):
|
||||
result, captured, pad_calls, _ = run_preview(_updated_data(10.0), ass_only=False)
|
||||
assert result["countdown_padding_seconds"] == 0.0
|
||||
assert captured["correction"].corrected_segments[0].text == "hello"
|
||||
assert pad_calls == []
|
||||
assert captured["audio"].endswith("song.flac")
|
||||
|
||||
|
||||
def test_countdown_previews_get_a_fresh_hash(run_preview):
|
||||
"""Previews cached before countdown support must not be served again."""
|
||||
import hashlib, json
|
||||
data = _updated_data(0.4)
|
||||
result, *_ = run_preview(data, ass_only=True)
|
||||
legacy = hashlib.md5(json.dumps(data, sort_keys=True).encode("utf-8")).hexdigest()[:12]
|
||||
assert result["preview_hash"] != legacy
|
||||
|
||||
late = _updated_data(10.0)
|
||||
result_late, *_ = run_preview(late, ass_only=True)
|
||||
legacy_late = hashlib.md5(json.dumps(late, sort_keys=True).encode("utf-8")).hexdigest()[:12]
|
||||
assert result_late["preview_hash"] == legacy_late
|
||||
|
||||
|
||||
def test_submitted_countdown_is_not_doubled_but_audio_is_padded(run_preview):
|
||||
"""Corrections that already carry the countdown (timings shifted) must not be
|
||||
shifted again, but the audio must still be padded or lyrics run 3s early."""
|
||||
data = {
|
||||
"corrections": [],
|
||||
"corrected_segments": [
|
||||
{"id": "c", "text": CountdownProcessor.COUNTDOWN_TEXT, "start_time": 0.1, "end_time": 2.9,
|
||||
"words": [{"id": "cw", "text": CountdownProcessor.COUNTDOWN_TEXT, "start_time": 0.1, "end_time": 2.9}]},
|
||||
{"id": "s1", "text": "hello", "start_time": 3.4, "end_time": 3.9,
|
||||
"words": [{"id": "w1", "text": "hello", "start_time": 3.4, "end_time": 3.9}]},
|
||||
],
|
||||
}
|
||||
result, captured, pad_calls, padded = run_preview(data, ass_only=False)
|
||||
assert result["countdown_padding_seconds"] == 3.0
|
||||
segs = captured["correction"].corrected_segments
|
||||
assert [s.text for s in segs] == [CountdownProcessor.COUNTDOWN_TEXT, "hello"]
|
||||
assert segs[1].words[0].start_time == pytest.approx(3.4)
|
||||
assert captured["audio"] == padded
|
||||
@@ -0,0 +1,92 @@
|
||||
"""CountdownProcessor helpers used by the review preview.
|
||||
|
||||
The review preview must show the same countdown intro the final render adds
|
||||
(CountdownProcessor.process in the encoding worker): a song whose vocals start
|
||||
within 3s gets lyrics shifted +3s and a "3... 2... 1..." line. These helpers
|
||||
split that into a lyrics-only step and a dict-level offset check.
|
||||
"""
|
||||
from karaoke_gen.lyrics_transcriber.output.countdown_processor import CountdownProcessor
|
||||
from karaoke_gen.lyrics_transcriber.types import CorrectionResult, LyricsSegment, Word
|
||||
|
||||
|
||||
def _result(first_word_start: float) -> CorrectionResult:
|
||||
word = Word(id="w1", text="hello", start_time=first_word_start, end_time=first_word_start + 0.5)
|
||||
seg = LyricsSegment(id="s1", text="hello", words=[word], start_time=first_word_start,
|
||||
end_time=first_word_start + 0.5)
|
||||
return CorrectionResult(
|
||||
original_segments=[seg], corrected_segments=[seg], corrections=[], corrections_made=0,
|
||||
confidence=1.0, reference_lyrics={}, anchor_sequences=[], gap_sequences=[],
|
||||
resized_segments=[], metadata={}, correction_steps=[], word_id_map={}, segment_id_map={},
|
||||
)
|
||||
|
||||
|
||||
class TestPaddingForSegmentsData:
|
||||
def test_early_vocals_need_padding(self):
|
||||
segs = [{"text": "hi", "words": [{"text": "hi", "start_time": 0.5}]}]
|
||||
assert CountdownProcessor.padding_for_segments_data(segs) == 3.0
|
||||
|
||||
def test_late_vocals_need_none(self):
|
||||
segs = [{"text": "hi", "words": [{"text": "hi", "start_time": 12.0}]}]
|
||||
assert CountdownProcessor.padding_for_segments_data(segs) == 0.0
|
||||
|
||||
def test_threshold_is_exclusive(self):
|
||||
segs = [{"text": "hi", "words": [{"text": "hi", "start_time": 3.0}]}]
|
||||
assert CountdownProcessor.padding_for_segments_data(segs) == 0.0
|
||||
|
||||
def test_skips_wordless_segments(self):
|
||||
segs = [{"text": "", "words": []}, {"text": "hi", "words": [{"text": "hi", "start_time": 1.0}]}]
|
||||
assert CountdownProcessor.padding_for_segments_data(segs) == 3.0
|
||||
|
||||
def test_already_counted_down_still_reports_offset(self):
|
||||
"""Timings are already shifted, so the timeline offset is still 3s."""
|
||||
segs = [{"text": CountdownProcessor.COUNTDOWN_TEXT,
|
||||
"words": [{"text": CountdownProcessor.COUNTDOWN_TEXT, "start_time": 0.1}]},
|
||||
{"text": "hi", "words": [{"text": "hi", "start_time": 3.4}]}]
|
||||
assert CountdownProcessor.padding_for_segments_data(segs) == 3.0
|
||||
|
||||
def test_empty_or_malformed(self):
|
||||
assert CountdownProcessor.padding_for_segments_data(None) == 0.0
|
||||
assert CountdownProcessor.padding_for_segments_data([]) == 0.0
|
||||
assert CountdownProcessor.padding_for_segments_data(
|
||||
[{"text": "x", "words": [{"text": "x", "start_time": None}]}]) == 0.0
|
||||
|
||||
|
||||
class TestAddCountdownToLyricsIfNeeded:
|
||||
def test_shifts_and_prepends_countdown(self, tmp_path):
|
||||
cp = CountdownProcessor(cache_dir=str(tmp_path))
|
||||
original = _result(0.4)
|
||||
result, padding = cp.add_countdown_to_lyrics_if_needed(original)
|
||||
assert padding == 3.0
|
||||
assert result.corrected_segments[0].text == CountdownProcessor.COUNTDOWN_TEXT
|
||||
assert result.corrected_segments[1].words[0].start_time == 3.4
|
||||
# The input is not mutated.
|
||||
assert original.corrected_segments[0].words[0].start_time == 0.4
|
||||
|
||||
def test_late_vocals_unchanged(self, tmp_path):
|
||||
cp = CountdownProcessor(cache_dir=str(tmp_path))
|
||||
original = _result(8.0)
|
||||
result, padding = cp.add_countdown_to_lyrics_if_needed(original)
|
||||
assert padding == 0.0
|
||||
assert result is original
|
||||
|
||||
def test_idempotent_but_still_needs_padded_audio(self, tmp_path):
|
||||
"""A result that already has the countdown isn't shifted again, but its
|
||||
audio must still be padded to match the shifted timings."""
|
||||
cp = CountdownProcessor(cache_dir=str(tmp_path))
|
||||
once, _ = cp.add_countdown_to_lyrics_if_needed(_result(0.4))
|
||||
twice, padding = cp.add_countdown_to_lyrics_if_needed(once)
|
||||
assert padding == 3.0
|
||||
assert twice is once
|
||||
assert twice.corrected_segments[1].words[0].start_time == 3.4
|
||||
|
||||
def test_agrees_with_dict_helper(self, tmp_path):
|
||||
cp = CountdownProcessor(cache_dir=str(tmp_path))
|
||||
for start in (0.0, 2.99, 3.0, 7.5):
|
||||
r = _result(start)
|
||||
out, padding = cp.add_countdown_to_lyrics_if_needed(r)
|
||||
segs = [s.to_dict() for s in r.corrected_segments]
|
||||
assert CountdownProcessor.padding_for_segments_data(segs) == padding
|
||||
# ...and on the already-processed result too.
|
||||
_, again = cp.add_countdown_to_lyrics_if_needed(out)
|
||||
assert CountdownProcessor.padding_for_segments_data(
|
||||
[s.to_dict() for s in out.corrected_segments]) == again
|
||||
Reference in new issue
Block a user