FIxed tests, fixed modal web implementation (at least for a first end to end successful job with upload to youtube, dropbox and google drive etc.)

This commit is contained in:
Andrew Beveridge committed 2025-07-08 02:22:05 -04:00
1 parent 7c8c57cd26
commit 3776f4d803
8 files changed
+338 -196

No files matched your search

+33 -2
View File
@@ -269,7 +269,12 @@ class JobLogHandler(logging.Handler):
"""Filter out noisy logs from Modal's internal libraries and operations."""
# Filter by logger name (module)
noisy_modules = ['hpack']
noisy_modules = [
'hpack',
'ssa',
'byteflow',
'PngImagePlugin'
]
if any(record.name.startswith(module) for module in noisy_modules):
return True
@@ -353,6 +358,9 @@ def setup_job_logging(job_id: str):
# Suppress noisy debug logs from Modal's internal libraries
noisy_loggers = [
'hpack',
'ssa',
'byteflow',
'PngImagePlugin',
'modal', # Suppress most Modal internal logs
'urllib3', # HTTP request logs
'requests', # HTTP request logs
@@ -4203,7 +4211,30 @@ async def complete_review(job_id: str, request: Request):
# Get the corrected lyrics data from request
request_data = await request.json()
corrected_data = request_data.get("corrected_data", {})
# Handle both formats:
# 1. Direct format (React frontend): { corrections: ..., corrected_segments: ... }
# 2. Wrapped format (Web frontend): { corrected_data: { corrections: ..., corrected_segments: ... } }
if "corrected_data" in request_data:
# Wrapped format from web frontend
corrected_data = request_data.get("corrected_data", {})
log_message(job_id, "DEBUG", "Received corrected data in wrapped format (web frontend)")
elif "corrections" in request_data or "corrected_segments" in request_data:
# Direct format from React frontend
corrected_data = request_data
log_message(job_id, "DEBUG", "Received corrected data in direct format (React frontend)")
else:
# No corrected data provided
corrected_data = {}
log_message(job_id, "DEBUG", "No corrected data provided in request")
# Log the corrected data details for debugging
if corrected_data:
corrections_count = len(corrected_data.get("corrections", {}))
segments_count = len(corrected_data.get("corrected_segments", []))
log_message(job_id, "INFO", f"Received corrected data: {corrections_count} corrections, {segments_count} segments")
else:
log_message(job_id, "INFO", "No corrected data received - will use original transcription")
log_message(job_id, "INFO", "Review completed, starting Phase 2 (video generation only)")
-119
View File
@@ -633,10 +633,6 @@ class ServerlessKaraokeProcessor:
if font_files:
self._fix_font_names_in_styles(output_styles_path, font_files)
# Run test render to verify font rendering works
test_output_dir = str(Path(output_styles_path).parent)
self.test_font_render(font_files, test_output_dir)
# Log the content of the updated file for debugging
self.logger.debug("Updated styles file content:")
with open(output_styles_path, 'r') as f:
@@ -908,121 +904,6 @@ class ServerlessKaraokeProcessor:
self.logger.error(f"Traceback: {traceback.format_exc()}")
# Don't raise - this shouldn't stop the job
def test_font_render(self, font_files: List[str], output_dir: str) -> None:
"""
Create a test video with ASS subtitles to verify font rendering works correctly.
"""
import subprocess
import tempfile
try:
test_dir = Path(output_dir) / "font_tests"
test_dir.mkdir(exist_ok=True)
self.logger.info("=== Starting Font Render Test ===")
for font_file in font_files:
try:
# Get actual font name
result = subprocess.run([
'fc-query', '--format=%{family}\\n', font_file
], capture_output=True, text=True, timeout=10)
if result.returncode == 0:
font_name = result.stdout.strip()
# Create test ASS file
ass_content = f"""[Script Info]
Title: Font Test
ScriptType: v4.00+
[V4+ Styles]
Format: Name, Fontname, Fontpath, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
Style: TestStyle,{font_name},{font_file},48,&H00FFFFFF,&H000000FF,&H00000000,&H80000000,0,0,0,0,100,100,0,0,1,2,0,2,10,10,10,1
[Events]
Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
Dialogue: 0,0:00:00.00,0:00:03.00,TestStyle,,0,0,0,,Font Test: {font_name}
Dialogue: 0,0:00:03.00,0:00:06.00,TestStyle,,0,0,0,,The quick brown fox jumps
Dialogue: 0,0:00:06.00,0:00:09.00,TestStyle,,0,0,0,,ABCDEFGHIJKLMNOP
"""
ass_file = test_dir / f"test_{Path(font_file).stem}.ass"
with open(ass_file, 'w') as f:
f.write(ass_content)
# Create test video
output_video = test_dir / f"test_{Path(font_file).stem}.mp4"
cmd = [
'ffmpeg', '-y', '-hide_banner', '-loglevel', 'error',
'-f', 'lavfi', '-i', 'color=black:size=640x360:duration=10:rate=25',
'-vf', f"ass={ass_file}",
'-c:v', 'libx264', '-preset', 'ultrafast',
str(output_video)
]
result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
if result.returncode == 0:
self.logger.info(f"✓ Test video created successfully: {output_video}")
self.logger.info(f" Font: {font_name} ({Path(font_file).name})")
else:
self.logger.error(f"✗ Test video failed for {font_name}: {result.stderr}")
except Exception as e:
self.logger.error(f"Test render failed for {font_file}: {str(e)}")
self.logger.info("=== Font Render Test Complete ===")
except Exception as e:
self.logger.error(f"Font test render failed: {str(e)}")
# Don't raise - this is just a test
def debug_ass_file(self, ass_file_path: str) -> None:
"""
Debug ASS file content to see what font is actually being used.
"""
try:
if not Path(ass_file_path).exists():
self.logger.warning(f"ASS file not found: {ass_file_path}")
return
self.logger.info(f"=== Debugging ASS File: {ass_file_path} ===")
with open(ass_file_path, 'r') as f:
lines = f.readlines()
# Look for Style lines
for i, line in enumerate(lines):
if line.startswith('Style:'):
self.logger.info(f"Line {i+1}: {line.strip()}")
# Parse the style line
parts = line.strip().split(',')
if len(parts) >= 4:
style_name = parts[0].split(':')[1] if ':' in parts[0] else parts[0]
fontname = parts[1] if len(parts) > 1 else "N/A"
fontpath = parts[2] if len(parts) > 2 else "N/A"
fontsize = parts[3] if len(parts) > 3 else "N/A"
self.logger.info(f" Style Name: {style_name}")
self.logger.info(f" Font Name: {fontname}")
self.logger.info(f" Font Path: {fontpath}")
self.logger.info(f" Font Size: {fontsize}")
# Check if font path exists
if fontpath.startswith('/') and fontpath != "N/A":
exists = Path(fontpath).exists()
self.logger.info(f" Font File Exists: {exists}")
if not exists:
self.logger.warning(f" ⚠ Font file missing: {fontpath}")
self.logger.info("=== End ASS File Debug ===")
except Exception as e:
self.logger.error(f"Error debugging ASS file: {str(e)}")
def ensure_fonts_available(self, styles_file_path: str) -> None:
"""
Ensure fonts are available in the current environment by re-installing them if needed.
@@ -690,9 +690,46 @@ class KaraokeFinalise:
return base_name, artist, title
def execute_command(self, command, description):
"""Legacy method for backwards compatibility - delegates to execute_command_with_fallback"""
# For backwards compatibility, treat as CPU command
self.execute_command_with_fallback(command, command, description)
"""Execute a shell command and log the output. For general commands (rclone, etc.)"""
self.logger.info(f"{description}")
self.logger.debug(f"Executing command: {command}")
if self.dry_run:
self.logger.info(f"DRY RUN: Would execute: {command}")
return
try:
result = subprocess.run(command, shell=True, capture_output=True, text=True, timeout=600)
# Log command output for debugging
if result.stdout and result.stdout.strip():
self.logger.debug(f"Command STDOUT: {result.stdout.strip()}")
if result.stderr and result.stderr.strip():
self.logger.debug(f"Command STDERR: {result.stderr.strip()}")
if result.returncode != 0:
error_msg = f"Command failed with exit code {result.returncode}"
self.logger.error(error_msg)
self.logger.error(f"Command: {command}")
if result.stdout:
self.logger.error(f"STDOUT: {result.stdout}")
if result.stderr:
self.logger.error(f"STDERR: {result.stderr}")
raise Exception(f"{error_msg}: {command}")
else:
self.logger.info(f"✓ Command completed successfully")
except subprocess.TimeoutExpired:
error_msg = f"Command timed out after 600 seconds"
self.logger.error(error_msg)
raise Exception(f"{error_msg}: {command}")
except Exception as e:
if "Command failed" not in str(e):
error_msg = f"Command failed with exception: {e}"
self.logger.error(error_msg)
raise Exception(f"{error_msg}: {command}")
else:
raise
def remux_with_instrumental(self, with_vocals_file, instrumental_audio, output_file):
"""Remux the video with instrumental audio to create karaoke version"""
@@ -1027,7 +1064,7 @@ class KaraokeFinalise:
os.remove(file_path)
self.logger.info(f"Deleted .DS_Store file: {file_path}")
rclone_cmd = f"rclone copy -v '{self.public_share_dir}' '{self.rclone_destination}'"
rclone_cmd = f"rclone copy -v {shlex.quote(self.public_share_dir)} {shlex.quote(self.rclone_destination)}"
self.execute_command(rclone_cmd, "Copying to cloud destination")
def post_discord_notification(self):
@@ -1062,6 +1099,13 @@ class KaraokeFinalise:
try:
self.logger.info(f"Running command: {rclone_link_cmd}")
result = subprocess.run(rclone_link_cmd, shell=True, check=True, capture_output=True, text=True)
# Log command output for debugging
if result.stdout and result.stdout.strip():
self.logger.debug(f"Command STDOUT: {result.stdout.strip()}")
if result.stderr and result.stderr.strip():
self.logger.debug(f"Command STDERR: {result.stderr.strip()}")
self.brand_code_dir_sharing_link = result.stdout.strip()
self.logger.info(f"Got organised folder sharing link: {self.brand_code_dir_sharing_link}")
except subprocess.CalledProcessError as e:
@@ -1084,23 +1128,28 @@ class KaraokeFinalise:
pattern = re.compile(rf"^{re.escape(self.brand_prefix)}-(\d{{4}})")
# Use rclone lsf --dirs-only for clean, machine-readable directory listing
rclone_list_cmd = f"rclone lsf --dirs-only '{self.organised_dir_rclone_root}'"
rclone_list_cmd = f"rclone lsf --dirs-only {shlex.quote(self.organised_dir_rclone_root)}"
if self.dry_run:
self.logger.info(f"DRY RUN: Would run: {rclone_list_cmd}")
return f"{self.brand_prefix}-0001"
try:
self.logger.debug(f"Running command: {rclone_list_cmd}")
self.logger.info(f"Running command: {rclone_list_cmd}")
result = subprocess.run(rclone_list_cmd, shell=True, check=True, capture_output=True, text=True)
# Log command output for debugging
if result.stdout and result.stdout.strip():
self.logger.debug(f"Command STDOUT: {result.stdout.strip()}")
if result.stderr and result.stderr.strip():
self.logger.debug(f"Command STDERR: {result.stderr.strip()}")
# Parse the output to find matching directories
matching_dirs = []
for line_num, line in enumerate(result.stdout.strip().split('\n')):
if line.strip():
# Remove trailing slash and whitespace
dir_name = line.strip().rstrip('/')
self.logger.debug(f"Processing directory {line_num}: '{dir_name}'")
# Check if directory matches our brand pattern
match = pattern.match(dir_name)
@@ -1108,7 +1157,6 @@ class KaraokeFinalise:
num = int(match.group(1))
max_num = max(max_num, num)
matching_dirs.append((dir_name, num))
self.logger.debug(f"Found matching directory: '{dir_name}' with number {num}")
self.logger.info(f"Found {len(matching_dirs)} matching directories with pattern {self.brand_prefix}-XXXX")
@@ -1141,7 +1189,7 @@ class KaraokeFinalise:
current_dir = os.getcwd()
# Use rclone copy to upload the entire current directory to the remote destination
rclone_upload_cmd = f"rclone copy -v '{current_dir}' '{remote_dest}'"
rclone_upload_cmd = f"rclone copy -v {shlex.quote(current_dir)} {shlex.quote(remote_dest)}"
if self.dry_run:
self.logger.info(f"DRY RUN: Would upload current directory to: {remote_dest}")
@@ -1156,7 +1204,7 @@ class KaraokeFinalise:
"""Generate a sharing link for the remote organized folder using rclone."""
self.logger.info(f"Getting sharing link for remote organized folder: {remote_path}")
rclone_link_cmd = f"rclone link '{remote_path}'"
rclone_link_cmd = f"rclone link {shlex.quote(remote_path)}"
if self.dry_run:
self.logger.info(f"DRY RUN: Would get sharing link with: {rclone_link_cmd}")
@@ -1170,6 +1218,13 @@ class KaraokeFinalise:
try:
self.logger.info(f"Running command: {rclone_link_cmd}")
result = subprocess.run(rclone_link_cmd, shell=True, check=True, capture_output=True, text=True)
# Log command output for debugging
if result.stdout and result.stdout.strip():
self.logger.debug(f"Command STDOUT: {result.stdout.strip()}")
if result.stderr and result.stderr.strip():
self.logger.debug(f"Command STDERR: {result.stderr.strip()}")
self.brand_code_dir_sharing_link = result.stdout.strip()
self.logger.info(f"Got organized folder sharing link: {self.brand_code_dir_sharing_link}")
except subprocess.CalledProcessError as e:
Generated
+19 -18
View File
@@ -1536,14 +1536,14 @@ readthedocs = ["sphinx-rtd-theme"]
[[package]]
name = "fastapi"
version = "0.115.14"
version = "0.116.0"
description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production"
optional = false
python-versions = ">=3.8"
groups = ["main"]
files = [
{file = "fastapi-0.115.14-py3-none-any.whl", hash = "sha256:6c0c8bf9420bd58f565e585036d971872472b4f7d3f6c73b698e10cffdefb3ca"},
{file = "fastapi-0.115.14.tar.gz", hash = "sha256:b1de15cdc1c499a4da47914db35d0e4ef8f1ce62b624e94e0e5824421df99739"},
{file = "fastapi-0.116.0-py3-none-any.whl", hash = "sha256:fdcc9ed272eaef038952923bef2b735c02372402d1203ee1210af4eea7a78d2b"},
{file = "fastapi-0.116.0.tar.gz", hash = "sha256:80dc0794627af0390353a6d1171618276616310d37d24faba6648398e57d687a"},
]
[package.dependencies]
@@ -1552,8 +1552,9 @@ starlette = ">=0.40.0,<0.47.0"
typing-extensions = ">=4.8.0"
[package.extras]
all = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.5)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=3.1.5)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.18)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"]
standard = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.5)", "httpx (>=0.23.0)", "jinja2 (>=3.1.5)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"]
all = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.8)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=3.1.5)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.18)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"]
standard = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.8)", "httpx (>=0.23.0)", "jinja2 (>=3.1.5)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"]
standard-no-fastapi-cloud-cli = ["email-validator (>=2.0.0)", "fastapi-cli[standard-no-fastapi-cloud-cli] (>=0.0.8)", "httpx (>=0.23.0)", "jinja2 (>=3.1.5)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"]
[[package]]
name = "fastjsonschema"
@@ -2942,14 +2943,14 @@ files = [
[[package]]
name = "lyrics-transcriber"
version = "0.60.0"
version = "0.61.0"
description = "Automatically create synchronised lyrics files in ASS and MidiCo LRC formats with word-level timestamps, using Whisper and lyrics from Genius and Spotify"
optional = false
python-versions = "<3.14,>=3.10"
groups = ["main"]
files = [
{file = "lyrics_transcriber-0.60.0-py3-none-any.whl", hash = "sha256:bcc42b221427a4cd8e95942240eac4ef6b52d1b8e98c1bbfefb2e24918c423cb"},
{file = "lyrics_transcriber-0.60.0.tar.gz", hash = "sha256:84f8affa27bed5c30502168e57b36b8c6ec42a06ab581c3489618e961e0baa13"},
{file = "lyrics_transcriber-0.61.0-py3-none-any.whl", hash = "sha256:21242400ddd9263e170613b6f54da769e4237da2dede2c2f338c3f47415e30ae"},
{file = "lyrics_transcriber-0.61.0.tar.gz", hash = "sha256:ef13651089b8c8dffb8e1cccb281c9e0824f70c6bcff01aa7293f806ff3c85de"},
]
[package.dependencies]
@@ -4156,14 +4157,14 @@ sympy = "*"
[[package]]
name = "openai"
version = "1.93.0"
version = "1.93.1"
description = "The official Python library for the openai API"
optional = false
python-versions = ">=3.8"
groups = ["main"]
files = [
{file = "openai-1.93.0-py3-none-any.whl", hash = "sha256:3d746fe5498f0dd72e0d9ab706f26c91c0f646bf7459e5629af8ba7c9dbdf090"},
{file = "openai-1.93.0.tar.gz", hash = "sha256:988f31ade95e1ff0585af11cc5a64510225e4f5cd392698c675d0a9265b8e337"},
{file = "openai-1.93.1-py3-none-any.whl", hash = "sha256:a2c2946c4f21346d4902311a7440381fd8a33466ee7ca688133d1cad29a9357c"},
{file = "openai-1.93.1.tar.gz", hash = "sha256:11eb8932965d0f79ecc4cb38a60a0c4cef4bcd5fcf08b99fc9a399fa5f1e50ab"},
]
[package.dependencies]
@@ -5079,21 +5080,21 @@ dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests
[[package]]
name = "pytest-asyncio"
version = "0.23.8"
version = "1.0.0"
description = "Pytest support for asyncio"
optional = false
python-versions = ">=3.8"
python-versions = ">=3.9"
groups = ["main", "dev"]
files = [
{file = "pytest_asyncio-0.23.8-py3-none-any.whl", hash = "sha256:50265d892689a5faefb84df80819d1ecef566eb3549cf915dfb33569359d1ce2"},
{file = "pytest_asyncio-0.23.8.tar.gz", hash = "sha256:759b10b33a6dc61cce40a8bd5205e302978bbbcc00e279a8b61d9a6a3c82e4d3"},
{file = "pytest_asyncio-1.0.0-py3-none-any.whl", hash = "sha256:4f024da9f1ef945e680dc68610b52550e36590a67fd31bb3b4943979a1f90ef3"},
{file = "pytest_asyncio-1.0.0.tar.gz", hash = "sha256:d15463d13f4456e1ead2594520216b225a16f781e144f8fdf6c5bb4667c48b3f"},
]
[package.dependencies]
pytest = ">=7.0.0,<9"
pytest = ">=8.2,<9"
[package.extras]
docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"]
docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1)"]
testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"]
[[package]]
@@ -7844,4 +7845,4 @@ cffi = ["cffi (>=1.11)"]
[metadata]
lock-version = "2.1"
python-versions = ">=3.10,<3.14"
content-hash = "f5c4872052d7a397501c7daba51aaec4525014a9e37bf2b513dd0682788cea10"
content-hash = "79161280bd0e2bac6d75dddb0dcedc5b4cb57a67694b61fad620ccd7a13fa978"
+3 -3
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.56.0"
version = "0.57.0"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"
@@ -36,7 +36,7 @@ toml = ">=0.10"
argparse = ">=1.4.0"
psutil = "^7.0.0"
pyperclip = "*"
pytest-asyncio = "^0.23.5"
pytest-asyncio = "*"
ffmpeg-python = "^0.2.0"
modal = "^1.0.5"
python-multipart = "^0.0.20"
@@ -47,7 +47,7 @@ poetry = "*"
pytest = ">=7.0"
pytest-cov = ">=4.0"
pytest-mock = ">=3.10"
pytest-asyncio = ">=0.21.0"
pytest-asyncio = "*"
[tool.poetry.scripts]
+209 -41
View File
@@ -297,29 +297,54 @@ async def test_full_cli_integration(tmp_path, mocker):
if cmd_str_decoded.strip().startswith("ffmpeg"):
should_execute = True
# Special handling for ffmpeg WAV conversion command - simulate creating WAV file instead of running actual ffmpeg
if isinstance(cmd_arg, list) and len(cmd_arg) > 3 and cmd_arg[0] == "ffmpeg" and any(".wav" in arg for arg in cmd_arg):
print(f"SIDE_EFFECT: Simulating ffmpeg WAV conversion instead of executing: {cmd_str}")
# Extract the output WAV path
output_wav_path = None
for i, arg in enumerate(cmd_arg):
if arg.endswith(".wav") and i > 0 and cmd_arg[i-1] != "-i":
output_wav_path = arg
break
# Special handling for ALL ffmpeg commands - create dummy output files instead of running actual ffmpeg
if isinstance(cmd_arg, list) and len(cmd_arg) > 0 and cmd_arg[0] == "ffmpeg":
print(f"SIDE_EFFECT: Simulating ffmpeg command instead of executing: {cmd_str}")
# Handle WAV conversion
if any(".wav" in arg for arg in cmd_arg):
# Extract the output WAV path
output_wav_path = None
for i, arg in enumerate(cmd_arg):
if arg.endswith(".wav") and i > 0 and cmd_arg[i-1] != "-i":
output_wav_path = arg
break
if output_wav_path:
# Create the output directory if it doesn't exist
os.makedirs(os.path.dirname(output_wav_path), exist_ok=True)
# Create a dummy WAV file with minimal header (44 bytes, 8000Hz, mono, PCM)
with open(output_wav_path, 'wb') as f:
# Simple WAV header + minimal audio data
f.write(b'RIFF\x24\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00@\x1f\x00\x00@\x1f\x00\x00\x01\x00\x08\x00data\x00\x00\x00\x00')
# Add some silence (1 second at 8000Hz)
f.write(b'\x80' * 8000)
print(f"SIDE_EFFECT: Created dummy WAV file at {output_wav_path}")
return subprocess.CompletedProcess(args=cmd_arg, returncode=0, stdout="", stderr="")
# Handle video conversion (MP4, MKV output)
elif any(arg.endswith((".mp4", ".mkv")) for arg in cmd_arg):
# Extract the output video path
output_video_path = None
for i, arg in enumerate(cmd_arg):
if arg.endswith((".mp4", ".mkv")) and i > 0 and cmd_arg[i-1] != "-i":
output_video_path = arg
break
if output_wav_path:
# Create the output directory if it doesn't exist
os.makedirs(os.path.dirname(output_wav_path), exist_ok=True)
# Create a dummy WAV file with minimal header (44 bytes, 8000Hz, mono, PCM)
with open(output_wav_path, 'wb') as f:
# Simple WAV header + minimal audio data
f.write(b'RIFF\x24\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00@\x1f\x00\x00@\x1f\x00\x00\x01\x00\x08\x00data\x00\x00\x00\x00')
# Add some silence (1 second at 8000Hz)
f.write(b'\x80' * 8000)
print(f"SIDE_EFFECT: Created dummy WAV file at {output_wav_path}")
return subprocess.CompletedProcess(args=cmd_arg, returncode=0, stdout="", stderr="")
if output_video_path:
# Create the output directory if it doesn't exist
os.makedirs(os.path.dirname(output_video_path), exist_ok=True)
# Create a dummy video file
with open(output_video_path, 'w') as f:
f.write('dummy video content for testing')
print(f"SIDE_EFFECT: Created dummy video file at {output_video_path}")
return subprocess.CompletedProcess(args=cmd_arg, returncode=0, stdout="", stderr="")
# Default ffmpeg mock - just return success
print(f"SIDE_EFFECT: Default ffmpeg mock - returning success for: {cmd_str}")
return subprocess.CompletedProcess(args=cmd_arg, returncode=0, stdout="", stderr="")
# For other ffmpeg commands, or if we couldn't extract the WAV path, proceed to normal execution
# For other non-ffmpeg commands that should be executed, proceed to normal execution
if should_execute:
print(f"SIDE_EFFECT: Executing original: {cmd_str}")
# Pass original args and kwargs to the original function
@@ -334,6 +359,14 @@ async def test_full_cli_integration(tmp_path, mocker):
if '_os_system_call' in run_kwargs:
del run_kwargs['_os_system_call']
# Handle shell parameter to avoid conflicts
if 'shell' in run_kwargs:
# If shell was already specified, use that value
use_shell = run_kwargs['shell']
else:
# Otherwise set it based on command type
run_kwargs['shell'] = use_shell
# Determine if output should be captured based on original kwargs or if it's ffmpeg/ffprobe
should_capture = False
if isinstance(cmd_arg, list) and cmd_arg[0] in ["ffmpeg", "ffprobe"]:
@@ -358,8 +391,9 @@ async def test_full_cli_integration(tmp_path, mocker):
current_mocked_dir = mock_current_dir_state()
print(f"SIDE_EFFECT: Executing {cmd_arg[0]} in mocked CWD: {current_mocked_dir}")
# Call the original subprocess run within the mocked CWD
result = original_subprocess_run(*args, shell=use_shell, cwd=current_mocked_dir, **run_kwargs)
# Set CWD in kwargs and call original subprocess run
run_kwargs['cwd'] = current_mocked_dir
result = original_subprocess_run(*args, **run_kwargs)
# Log results if captured
if should_capture and hasattr(result, 'returncode'): # Check if it's a CompletedProcess
@@ -772,21 +806,160 @@ async def test_full_cli_integration(tmp_path, mocker):
# Mock all FFmpeg command execution in finalisation to prevent system dependencies
execute_command_calls = [] # Track calls for testing
def mock_execute_command(command, description):
"""Mock execute_command to prevent actual FFmpeg execution and create dummy output files"""
print(f"MOCK execute_command: {description}")
print(f"MOCK execute_command: Would run: {command[:100]}...") # Show first 100 chars
# Mock the specific video conversion methods that call FFmpeg directly
def mock_convert_mov_to_mp4(*args, **kwargs):
"""Mock MOV to MP4 conversion - create dummy output file"""
print("MOCK convert_mov_to_mp4: Creating dummy MP4 output")
# The method signature is convert_mov_to_mp4(self, input_file, output_file)
if len(args) >= 3: # self, input_file, output_file
input_file = args[1]
output_file = args[2]
else:
input_file = kwargs.get('input_file', "ABBA - Waterloo (With Vocals).mkv")
output_file = kwargs.get('output_file', f"{input_file.replace('.mkv', '').replace('.mov', '')}.mp4")
# Track this call for testing
execute_command_calls.append((command, description))
with open(output_file, "w") as f:
f.write("dummy mp4 content for testing")
print(f"MOCK convert_mov_to_mp4: Created {output_file}")
return output_file
def mock_execute_command_with_fallback(*args, **kwargs):
"""Mock execute_command_with_fallback to avoid FFmpeg calls and create expected output files"""
print("MOCK execute_command_with_fallback: Mocking hardware/software fallback")
# The arguments are (self, gpu_command, cpu_command, description)
if len(args) >= 4:
description = args[3]
else:
description = kwargs.get('description', 'Unknown FFmpeg operation')
print(f"MOCK execute_command_with_fallback: {description}")
# Create dummy output files that the finalisation process expects
if "Creating MP4 version with PCM audio" in description:
output_file = "ABBA - Waterloo (Final Karaoke Lossless 4k).mp4"
with open(output_file, "w") as f:
f.write("dummy lossless mp4 content")
print(f"MOCK execute_command: Created {output_file}")
print(f"MOCK execute_command_with_fallback: Created {output_file}")
elif "Creating MP4 version with AAC audio" in description:
output_file = "ABBA - Waterloo (Final Karaoke Lossy 4k).mp4"
with open(output_file, "w") as f:
f.write("dummy lossy mp4 content")
print(f"MOCK execute_command_with_fallback: Created {output_file}")
elif "Creating MKV version with FLAC audio" in description:
output_file = "ABBA - Waterloo (Final Karaoke Lossless 4k).mkv"
with open(output_file, "w") as f:
f.write("dummy mkv content")
print(f"MOCK execute_command_with_fallback: Created {output_file}")
elif "Encoding 720p version" in description:
output_file = "ABBA - Waterloo (Final Karaoke Lossy 720p).mp4"
with open(output_file, "w") as f:
f.write("dummy 720p mp4 content")
print(f"MOCK execute_command_with_fallback: Created {output_file}")
elif "Converting MOV video to MP4" in description:
output_file = "ABBA - Waterloo (With Vocals).mp4"
with open(output_file, "w") as f:
f.write("dummy with vocals mp4 content")
print(f"MOCK execute_command_with_fallback: Created {output_file}")
return
def mock_remux_with_instrumental(*args, **kwargs):
"""Mock remux_with_instrumental to create dummy output file"""
print("MOCK remux_with_instrumental: Creating dummy karaoke MP4")
# The method signature is remux_with_instrumental(self, with_vocals_file, instrumental_audio, output_file)
if len(args) >= 4: # self, with_vocals_file, instrumental_audio, output_file
output_file = args[3]
else:
output_file = kwargs.get('output_file', "ABBA - Waterloo (Karaoke).mp4")
with open(output_file, "w") as f:
f.write("dummy karaoke mp4 content for testing")
print(f"MOCK remux_with_instrumental: Created {output_file}")
return output_file
# Mock the specific encoding methods that create the output files we need
def mock_encode_lossless_mp4(*args, **kwargs):
"""Mock encode_lossless_mp4 to create expected output file"""
print("MOCK encode_lossless_mp4: Creating lossless MP4 output")
output_file = "ABBA - Waterloo (Final Karaoke Lossless 4k).mp4"
with open(output_file, "w") as f:
f.write("dummy lossless mp4 content for testing")
print(f"MOCK encode_lossless_mp4: Created {output_file}")
return
def mock_encode_lossy_mp4(*args, **kwargs):
"""Mock encode_lossy_mp4 to create expected output file"""
print("MOCK encode_lossy_mp4: Creating lossy MP4 output")
output_file = "ABBA - Waterloo (Final Karaoke Lossy 4k).mp4"
with open(output_file, "w") as f:
f.write("dummy lossy mp4 content for testing")
print(f"MOCK encode_lossy_mp4: Created {output_file}")
return
def mock_encode_lossless_mkv(*args, **kwargs):
"""Mock encode_lossless_mkv to create expected output file"""
print("MOCK encode_lossless_mkv: Creating lossless MKV output")
output_file = "ABBA - Waterloo (Final Karaoke Lossless 4k).mkv"
with open(output_file, "w") as f:
f.write("dummy mkv content for testing")
print(f"MOCK encode_lossless_mkv: Created {output_file}")
return
def mock_encode_720p_version(*args, **kwargs):
"""Mock encode_720p_version to create expected output file"""
print("MOCK encode_720p_version: Creating 720p MP4 output")
output_file = "ABBA - Waterloo (Final Karaoke Lossy 720p).mp4"
with open(output_file, "w") as f:
f.write("dummy 720p mp4 content for testing")
print(f"MOCK encode_720p_version: Created {output_file}")
return
# Mock the video conversion methods in the finalisation module
mocker.patch('karaoke_gen.karaoke_finalise.karaoke_finalise.KaraokeFinalise.convert_mov_to_mp4',
side_effect=mock_convert_mov_to_mp4)
mocker.patch('karaoke_gen.karaoke_finalise.karaoke_finalise.KaraokeFinalise.remux_with_instrumental',
side_effect=mock_remux_with_instrumental)
# Mock the specific encoding methods that create the output files
mocker.patch('karaoke_gen.karaoke_finalise.karaoke_finalise.KaraokeFinalise.encode_lossless_mp4',
side_effect=mock_encode_lossless_mp4)
mocker.patch('karaoke_gen.karaoke_finalise.karaoke_finalise.KaraokeFinalise.encode_lossy_mp4',
side_effect=mock_encode_lossy_mp4)
mocker.patch('karaoke_gen.karaoke_finalise.karaoke_finalise.KaraokeFinalise.encode_lossless_mkv',
side_effect=mock_encode_lossless_mkv)
mocker.patch('karaoke_gen.karaoke_finalise.karaoke_finalise.KaraokeFinalise.encode_720p_version',
side_effect=mock_encode_720p_version)
def mock_execute_command_general(*args, **kwargs):
"""Mock execute_command for general shell commands and create expected output files"""
print("MOCK execute_command: Mocking general command execution")
# Debug the arguments
print(f"MOCK execute_command: args = {args}")
print(f"MOCK execute_command: kwargs = {kwargs}")
# The arguments are (command, description) when called directly
if len(args) >= 2:
command = args[0]
description = args[1]
elif len(args) == 1:
command = args[0]
description = kwargs.get('description', 'Unknown description')
else:
command = kwargs.get('command', 'Unknown command')
description = kwargs.get('description', 'Unknown description')
print(f"MOCK execute_command: {description}")
print(f"MOCK execute_command: Command: {command}")
# Track this call for testing
execute_command_calls.append((command, description))
# Create dummy output files that the finalisation process expects
if "Creating MP4 version with AAC audio" in description:
output_file = "ABBA - Waterloo (Final Karaoke Lossy 4k).mp4"
with open(output_file, "w") as f:
f.write("dummy lossy mp4 content")
@@ -796,16 +969,11 @@ async def test_full_cli_integration(tmp_path, mocker):
with open(output_file, "w") as f:
f.write("dummy mkv content")
print(f"MOCK execute_command: Created {output_file}")
elif "Encoding 720p version" in description:
output_file = "ABBA - Waterloo (Final Karaoke Lossy 720p).mp4"
with open(output_file, "w") as f:
f.write("dummy 720p mp4 content")
print(f"MOCK execute_command: Created {output_file}")
return # Just return without executing
return
mocker.patch('karaoke_gen.karaoke_finalise.karaoke_finalise.KaraokeFinalise.execute_command',
side_effect=mock_execute_command)
side_effect=mock_execute_command_general)
# Mock convert_file for lyrics format conversion
def mock_convert_file(*args, **kwargs):
@@ -984,10 +1152,10 @@ async def test_full_cli_integration(tmp_path, mocker):
mock_requests_post.assert_called_once_with(discord_webhook_url, json=mocker.ANY)
assert 'https://www.youtube.com/watch?v=manual_mock_video_id' in mock_requests_post.call_args[1]['json']['content']
# Rclone Sync - check if execute_command was called with rclone sync
# The rclone sync is executed through the mocked execute_command method
# Rclone Sync - check if execute_command was called with rclone copy
# The rclone copy is executed through the mocked execute_command method
rclone_sync_called = any(
'rclone sync' in command and str(public_share_dir) in command and rclone_destination in command
'rclone copy' in command and str(public_share_dir) in command and rclone_destination in command
for command, description in execute_command_calls
)
@@ -73,7 +73,8 @@ def test_execute_command_runs_command(mock_subprocess_run, finaliser_with_aac):
mock_subprocess_run.assert_called_once_with(command, shell=True, capture_output=True, text=True, timeout=600)
finaliser_with_aac.logger.info.assert_any_call(description)
finaliser_with_aac.logger.debug.assert_any_call(f"Running software encoding: {command}")
finaliser_with_aac.logger.debug.assert_any_call(f"Executing command: {command}")
finaliser_with_aac.logger.info.assert_any_call("✓ Command completed successfully")
@patch('subprocess.run')
def test_execute_command_dry_run(mock_subprocess_run, finaliser_with_aac):
@@ -85,7 +86,7 @@ def test_execute_command_dry_run(mock_subprocess_run, finaliser_with_aac):
mock_subprocess_run.assert_not_called()
finaliser_with_aac.logger.info.assert_any_call(description)
finaliser_with_aac.logger.info.assert_any_call(f"DRY RUN: Would run CPU command: {command}")
finaliser_with_aac.logger.info.assert_any_call(f"DRY RUN: Would execute: {command}")
# --- prepare_concat_filter Tests ---
@@ -235,7 +235,8 @@ def test_copy_final_files_dry_run(mock_copy, mock_makedirs, mock_isdir, finalise
@patch('os.remove')
@patch('os.path.join', side_effect=os.path.join)
@patch.object(KaraokeFinalise, 'execute_command')
def test_sync_public_share_dir_to_rclone(mock_execute, mock_join, mock_remove, mock_walk, finaliser_for_org):
@patch('shlex.quote', side_effect=lambda x: f"'{x}'") # Mock shlex.quote for predictable output
def test_sync_public_share_dir_to_rclone(mock_quote, mock_execute, mock_join, mock_remove, mock_walk, finaliser_for_org):
"""Test syncing public share dir, including .DS_Store removal."""
# Simulate finding a .DS_Store file
mock_walk.return_value = [
@@ -252,6 +253,10 @@ def test_sync_public_share_dir_to_rclone(mock_execute, mock_join, mock_remove, m
finaliser_for_org.logger.info.assert_any_call(f"Deleted .DS_Store file: {ds_store_path}")
expected_cmd = f"rclone copy -v '{PUBLIC_SHARE_DIR}' '{RCLONE_DEST}'"
mock_execute.assert_called_once_with(expected_cmd, "Copying to cloud destination")
mock_quote.assert_has_calls([
call(PUBLIC_SHARE_DIR),
call(RCLONE_DEST)
], any_order=True)
@patch('time.sleep')
@patch('subprocess.run')