Files
karaoke-gen/tests/unit/test_video_generator_unicode.py
Andrew Beveridge ☄️andClaude Sonnet 4.5 d9bb8b3181 fix: sanitize Unicode in temp file paths (FFmpeg exit 183) (#375)
## Summary
Fixes job 1eab1172 which failed with FFmpeg exit code 183 due to
Cyrillic characters in temp subtitle file paths. Python's
`str.isalnum()` returns True for ALL Unicode alphanumeric characters
(Cyrillic, Arabic, Chinese, emoji, etc.), not just ASCII.

## Root Cause
- Job `1eab1172` with Russian title "я куплю тебе дом" failed
- Temp subtitle path contained Cyrillic:
`temp_subtitles_zivert___я_куплю_тебе_дом_1770150285852.ass`
- FFmpeg cannot handle Unicode in filter argument paths → exit code 183

## Changes
- **karaoke_gen/lyrics_transcriber/output/video.py** (lines 222, 282):
- Changed `c.isalnum()` to `c.isascii() and c.isalnum()` for temp
subtitle paths
  - Affects both `generate_video()` and `generate_preview_video()`

- **karaoke_gen/utils/remote_cli.py** (line 1970):
- Changed `c.isalnum()` to `c.isascii() and c.isalnum()` for output
folder names
- Added fallback to `job_{job_id}` if folder name is empty or contains
no alphanumeric chars

- **Tests added** (21 comprehensive tests):
- `test_video_generator_unicode.py` (10 tests): Cyrillic, Chinese,
Arabic, emoji, accented chars
- `test_remote_cli_unicode_folders.py` (11 tests): Folder sanitization,
edge cases, real failure case

- **Documentation**:
- Updated `docs/LESSONS-LEARNED.md` with "Unicode in File Paths (Feb
2026)" section
  - Updated `docs/README.md` with recent changes
  - Bumped version 0.115.1 → 0.115.2

## Testing
- [x] All 1088 unit tests pass (68.91% coverage, target 69%)
- [x] 21 new comprehensive Unicode tests covering:
  - Cyrillic, Chinese, Arabic, emoji, accented characters
  - Empty folder name edge case
  - Separator-only folder name edge case
  - Exact real-world failure case from job 1eab1172
- [x] Verified temp file paths are ASCII-only
- [x] Verified folder names safe for all filesystems

## Review
- [x] CodeRabbit CLI review completed locally (2 cycles)
- [x] All identified issues addressed:
  - Empty folder name fallback added
  - Separator-only folder name guard enhanced
- [x] Test coverage maintained at target

@coderabbitai ignore

---
🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>
2026-02-05 09:13:33 -08:00

284 lines
12 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Unit tests for VideoGenerator Unicode filename handling.
Tests the fix for job 1eab1172 where Cyrillic characters in temp subtitle
filenames caused FFmpeg to fail with exit code 183.
"""
import os
import tempfile
import pytest
from unittest.mock import MagicMock, patch
from karaoke_gen.lyrics_transcriber.output.video import VideoGenerator
class TestVideoGeneratorUnicodeSanitization:
"""Test that VideoGenerator properly sanitizes Unicode characters in temp filenames."""
@pytest.fixture
def video_generator(self):
"""Create a VideoGenerator instance for testing."""
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = os.path.join(temp_dir, "output")
cache_dir = os.path.join(temp_dir, "cache")
os.makedirs(output_dir)
os.makedirs(cache_dir)
styles = {
"karaoke": {
"background_color": "black",
"font_path": None,
}
}
generator = VideoGenerator(
output_dir=output_dir,
cache_dir=cache_dir,
video_resolution=(1920, 1080),
styles=styles,
logger=MagicMock(),
)
yield generator
def test_generate_video_sanitizes_cyrillic_characters(self, video_generator):
"""Test that Cyrillic characters are replaced with underscores in temp filename.
This was the bug in job 1eab1172 with Russian title "я куплю тебе дом".
"""
output_prefix = "zivert - я куплю тебе дом"
# Create mock ASS and audio files
with tempfile.NamedTemporaryFile(suffix=".ass", delete=False) as ass_file:
ass_path = ass_file.name
ass_file.write(b"[Script Info]\nTitle: Test\n")
with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as audio_file:
audio_path = audio_file.name
try:
with patch.object(video_generator, '_run_ffmpeg_command') as mock_ffmpeg:
# This should not raise an exception
video_generator.generate_video(ass_path, audio_path, output_prefix)
# Verify FFmpeg was called
assert mock_ffmpeg.called
# Get the temp file path that was created
cmd = mock_ffmpeg.call_args[0][0]
ass_filter_arg = None
for i, arg in enumerate(cmd):
if arg == "-vf":
ass_filter_arg = cmd[i + 1]
break
assert ass_filter_arg is not None, "Should have ASS filter in FFmpeg command"
# Extract the temp ASS path from the filter
# Format: ass=/path/to/temp_subtitles_...:fontsdir=...
temp_ass_path = ass_filter_arg.split("=")[1].split(":")[0]
# Verify the temp filename only contains ASCII characters
temp_filename = os.path.basename(temp_ass_path)
assert temp_filename.isascii(), f"Temp filename should be ASCII-only: {temp_filename}"
# Verify Cyrillic characters were replaced with underscores
assert "я" not in temp_filename
assert "куплю" not in temp_filename
assert "дом" not in temp_filename
assert "_" in temp_filename # Should have underscores as replacements
finally:
# Cleanup
if os.path.exists(ass_path):
os.unlink(ass_path)
if os.path.exists(audio_path):
os.unlink(audio_path)
def test_generate_preview_video_sanitizes_unicode(self, video_generator):
"""Test that preview video generation also sanitizes Unicode characters."""
output_prefix = "artist - 中文标题" # Chinese characters
# Create mock ASS and audio files
with tempfile.NamedTemporaryFile(suffix=".ass", delete=False) as ass_file:
ass_path = ass_file.name
ass_file.write(b"[Script Info]\nTitle: Test\n")
with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as audio_file:
audio_path = audio_file.name
try:
# Mock the LocalPreviewEncodingService
with patch('backend.services.local_preview_encoding_service.LocalPreviewEncodingService') as mock_service_class:
mock_service = MagicMock()
mock_result = MagicMock()
mock_result.success = True
mock_service.encode_preview.return_value = mock_result
mock_service_class.return_value = mock_service
# This should not raise an exception
video_generator.generate_preview_video(ass_path, audio_path, output_prefix)
# Verify the service was called
assert mock_service.encode_preview.called
# Get the config that was passed
config = mock_service.encode_preview.call_args[0][0]
temp_ass_path = config.ass_path
# Verify the temp filename only contains ASCII characters
temp_filename = os.path.basename(temp_ass_path)
assert temp_filename.isascii(), f"Temp filename should be ASCII-only: {temp_filename}"
# Verify Chinese characters were replaced
assert "中文标题" not in temp_filename
finally:
# Cleanup
if os.path.exists(ass_path):
os.unlink(ass_path)
if os.path.exists(audio_path):
os.unlink(audio_path)
@pytest.mark.parametrize("output_prefix,unicode_chars,should_contain_ascii", [
("artist - я куплю тебе дом", ["я", "к", "у", "п", "л", "ю", "т", "е", "б", "д", "о", "м"], "artist"), # Cyrillic
("artist - 中文标题", ["中", "文", "标", "题"], "artist"), # Chinese
("artist - العربية", ["ا", "ل", "ع", "ر", "ب", "ي", "ة"], "artist"), # Arabic
("artist - 😀🎵", ["😀", "🎵"], "artist"), # Emoji
("artist - título", ["í"], "tulo"), # Spanish with accent - 't' and 'ulo' should remain
("normal title", [], "normal_title"), # ASCII-only (control)
])
def test_various_unicode_characters_sanitized(self, video_generator, output_prefix, unicode_chars, should_contain_ascii):
"""Test that various Unicode character sets are properly sanitized."""
import re
# Create mock ASS and audio files
with tempfile.NamedTemporaryFile(suffix=".ass", delete=False) as ass_file:
ass_path = ass_file.name
ass_file.write(b"[Script Info]\nTitle: Test\n")
with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as audio_file:
audio_path = audio_file.name
try:
with patch.object(video_generator, '_run_ffmpeg_command') as mock_ffmpeg:
video_generator.generate_video(ass_path, audio_path, output_prefix)
# Get the temp file path
cmd = mock_ffmpeg.call_args[0][0]
ass_filter_arg = None
for i, arg in enumerate(cmd):
if arg == "-vf":
ass_filter_arg = cmd[i + 1]
break
temp_ass_path = ass_filter_arg.split("=")[1].split(":")[0]
temp_filename = os.path.basename(temp_ass_path)
# Verify filename only contains ASCII
assert temp_filename.isascii(), f"Should be ASCII-only: {temp_filename}"
# Verify all Unicode characters were removed
for unicode_char in unicode_chars:
assert unicode_char not in temp_filename, \
f"Unicode character '{unicode_char}' should not be in filename: {temp_filename}"
# Verify ASCII parts were preserved
if should_contain_ascii:
assert should_contain_ascii in temp_filename, \
f"Expected '{should_contain_ascii}' to be in filename: {temp_filename}"
# Verify it matches the general pattern (starts with temp_subtitles_, ends with .ass)
assert temp_filename.startswith("temp_subtitles_"), \
f"Filename should start with 'temp_subtitles_': {temp_filename}"
assert temp_filename.endswith(".ass"), \
f"Filename should end with '.ass': {temp_filename}"
finally:
if os.path.exists(ass_path):
os.unlink(ass_path)
if os.path.exists(audio_path):
os.unlink(audio_path)
def test_sanitization_preserves_ascii_alphanumeric(self, video_generator):
"""Test that ASCII alphanumeric characters are preserved."""
output_prefix = "ABC123xyz"
# Create mock ASS and audio files
with tempfile.NamedTemporaryFile(suffix=".ass", delete=False) as ass_file:
ass_path = ass_file.name
ass_file.write(b"[Script Info]\nTitle: Test\n")
with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as audio_file:
audio_path = audio_file.name
try:
with patch.object(video_generator, '_run_ffmpeg_command') as mock_ffmpeg:
video_generator.generate_video(ass_path, audio_path, output_prefix)
# Get the temp file path
cmd = mock_ffmpeg.call_args[0][0]
ass_filter_arg = None
for i, arg in enumerate(cmd):
if arg == "-vf":
ass_filter_arg = cmd[i + 1]
break
temp_ass_path = ass_filter_arg.split("=")[1].split(":")[0]
temp_filename = os.path.basename(temp_ass_path)
# Verify ASCII alphanumeric characters are preserved
assert "ABC123xyz" in temp_filename
finally:
if os.path.exists(ass_path):
os.unlink(ass_path)
if os.path.exists(audio_path):
os.unlink(audio_path)
def test_special_characters_replaced_with_underscores(self, video_generator):
"""Test that special characters (spaces, hyphens, etc.) are replaced."""
output_prefix = "artist - title (version) [2024]"
# Create mock ASS and audio files
with tempfile.NamedTemporaryFile(suffix=".ass", delete=False) as ass_file:
ass_path = ass_file.name
ass_file.write(b"[Script Info]\nTitle: Test\n")
with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as audio_file:
audio_path = audio_file.name
try:
with patch.object(video_generator, '_run_ffmpeg_command') as mock_ffmpeg:
video_generator.generate_video(ass_path, audio_path, output_prefix)
# Get the temp file path
cmd = mock_ffmpeg.call_args[0][0]
ass_filter_arg = None
for i, arg in enumerate(cmd):
if arg == "-vf":
ass_filter_arg = cmd[i + 1]
break
temp_ass_path = ass_filter_arg.split("=")[1].split(":")[0]
temp_filename = os.path.basename(temp_ass_path)
# Verify special characters were replaced
assert " " not in temp_filename
assert "-" not in temp_filename
assert "(" not in temp_filename
assert ")" not in temp_filename
assert "[" not in temp_filename
assert "]" not in temp_filename
# But alphanumeric should remain
assert "artist" in temp_filename
assert "title" in temp_filename
assert "version" in temp_filename
assert "2024" in temp_filename
finally:
if os.path.exists(ass_path):
os.unlink(ass_path)
if os.path.exists(audio_path):
os.unlink(audio_path)