mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
1. _validate_url now handles None and non-string inputs safely - Added early guard: if url is None or not isinstance(url, str) - Updated function signature to accept Optional[str] 2. Fixed cookie tempfile leak in file_handler.py - Use context manager pattern to safely write cookies - Track cookie_file_path instead of cookie_file object - Guaranteed cleanup in finally block 3. Removed unused style_params_path from submit_job_from_url - URL-based jobs don't support custom styles (would need signed URL upload) - Added warning when user provides --style_params_json with URL input - Updated docstring to clarify limitation Also merged latest main which added job_logging_context for log isolation.
533 lines
23 KiB
Python
533 lines
23 KiB
Python
"""
|
|
Audio separation worker.
|
|
|
|
Handles the audio processing track of parallel processing:
|
|
1. Stage 1: Clean instrumental separation (Modal API, 3-5 min)
|
|
2. Stage 2: Backing vocals separation (Modal API, 2-3 min)
|
|
3. Post-processing: Combine instrumentals, normalize
|
|
|
|
Re-uses karaoke_gen.audio_processor.AudioProcessor for remote GPU separation.
|
|
"""
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import tempfile
|
|
from typing import Optional, Dict, Any
|
|
from pathlib import Path
|
|
|
|
from backend.models.job import JobStatus
|
|
from backend.services.job_manager import JobManager
|
|
from backend.services.storage_service import StorageService
|
|
from backend.config import get_settings
|
|
from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
|
|
|
|
# Import from karaoke_gen package
|
|
from karaoke_gen.audio_processor import AudioProcessor
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# Loggers to capture for audio worker
|
|
AUDIO_WORKER_LOGGERS = [
|
|
"karaoke_gen.audio_processor",
|
|
]
|
|
|
|
|
|
async def download_from_url(url: str, temp_dir: str, artist: str, title: str, job_manager: JobManager = None, job_id: str = None) -> Optional[str]:
|
|
"""
|
|
Download audio from a URL (YouTube, etc.) using karaoke_gen.FileHandler.
|
|
|
|
Uses the battle-tested FileHandler from karaoke_gen which includes:
|
|
- Anti-detection options (user agent, headers, delays)
|
|
- Cookie support for authenticated downloads
|
|
- Retry logic
|
|
|
|
If artist and/or title are not provided, attempts to extract them from
|
|
the URL metadata.
|
|
|
|
Args:
|
|
url: URL to download from
|
|
temp_dir: Temporary directory to save to
|
|
artist: Artist name for filename (can be None for auto-detection)
|
|
title: Song title for filename (can be None for auto-detection)
|
|
job_manager: Optional JobManager to update job with detected metadata
|
|
job_id: Optional job ID to update
|
|
|
|
Returns:
|
|
Path to downloaded audio file, or None if failed
|
|
"""
|
|
try:
|
|
from karaoke_gen.file_handler import FileHandler
|
|
from karaoke_gen.utils import sanitize_filename
|
|
|
|
# Create FileHandler instance
|
|
file_handler = FileHandler(
|
|
logger=logger,
|
|
ffmpeg_base_command="ffmpeg -hide_banner -loglevel error -nostats -y",
|
|
create_track_subfolders=False,
|
|
dry_run=False
|
|
)
|
|
|
|
# Try to extract metadata if artist or title not provided
|
|
if not artist or not title:
|
|
logger.info(f"Extracting metadata from URL: {url}")
|
|
metadata = file_handler.extract_metadata_from_url(url)
|
|
|
|
if metadata:
|
|
if not artist:
|
|
artist = metadata.get('artist', 'Unknown')
|
|
logger.info(f"Auto-detected artist: {artist}")
|
|
if not title:
|
|
title = metadata.get('title', 'Unknown')
|
|
logger.info(f"Auto-detected title: {title}")
|
|
|
|
# Update job with detected metadata if job_manager provided
|
|
if job_manager and job_id:
|
|
update_data = {}
|
|
if artist:
|
|
update_data['artist'] = artist
|
|
if title:
|
|
update_data['title'] = title
|
|
if update_data:
|
|
job_manager.update_job(job_id, update_data)
|
|
logger.info(f"Updated job {job_id} with detected metadata")
|
|
else:
|
|
logger.warning("Could not extract metadata from URL, using defaults")
|
|
artist = artist or "Unknown"
|
|
title = title or "Unknown"
|
|
|
|
# Create output filename (without extension)
|
|
safe_artist = sanitize_filename(artist) if artist else "Unknown"
|
|
safe_title = sanitize_filename(title) if title else "Unknown"
|
|
output_filename_no_extension = os.path.join(temp_dir, f"{safe_artist} - {safe_title}")
|
|
|
|
# Get YouTube cookies from environment variable if available
|
|
# This helps bypass "Sign in to confirm you're not a bot" errors
|
|
cookies_str = os.environ.get("YOUTUBE_COOKIES")
|
|
if cookies_str:
|
|
logger.info("Using YouTube cookies for download authentication")
|
|
else:
|
|
logger.info("No YOUTUBE_COOKIES env var set - attempting download without cookies")
|
|
|
|
# Download using FileHandler (includes anti-detection features)
|
|
logger.info(f"Downloading from URL: {url}")
|
|
downloaded_file = file_handler.download_video(
|
|
url=url,
|
|
output_filename_no_extension=output_filename_no_extension,
|
|
cookies_str=cookies_str
|
|
)
|
|
|
|
if downloaded_file and os.path.exists(downloaded_file):
|
|
logger.info(f"Downloaded video: {downloaded_file}")
|
|
|
|
# Convert to WAV for processing
|
|
wav_file = file_handler.convert_to_wav(
|
|
input_filename=downloaded_file,
|
|
output_filename_no_extension=output_filename_no_extension
|
|
)
|
|
|
|
if wav_file and os.path.exists(wav_file):
|
|
logger.info(f"Converted to WAV: {wav_file}")
|
|
return wav_file
|
|
else:
|
|
logger.error("WAV conversion failed")
|
|
return None
|
|
else:
|
|
logger.error("Download failed - no file returned")
|
|
return None
|
|
|
|
except ImportError as e:
|
|
logger.error(f"Import error: {e}. Check karaoke_gen installation.")
|
|
return None
|
|
except Exception as e:
|
|
logger.error(f"Failed to download from URL {url}: {e}", exc_info=True)
|
|
return None
|
|
|
|
|
|
def create_audio_processor(
|
|
temp_dir: str,
|
|
clean_instrumental_model: Optional[str] = None,
|
|
backing_vocals_models: Optional[list] = None,
|
|
other_stems_models: Optional[list] = None
|
|
) -> AudioProcessor:
|
|
"""
|
|
Create an AudioProcessor instance configured for remote API processing.
|
|
|
|
This reuses the karaoke_gen AudioProcessor with settings optimized for Cloud Run:
|
|
- Uses remote Modal API (via AUDIO_SEPARATOR_API_URL env var)
|
|
- No local models needed (model_file_dir=None)
|
|
- FLAC output format for quality
|
|
- Model configurations from job or CLI defaults
|
|
|
|
Args:
|
|
temp_dir: Temporary directory for processing
|
|
clean_instrumental_model: Model for clean instrumental separation (optional, uses default if not provided)
|
|
backing_vocals_models: List of models for backing vocals separation (optional, uses default if not provided)
|
|
other_stems_models: List of models for other stems separation (optional, uses default if not provided)
|
|
|
|
Returns:
|
|
Configured AudioProcessor instance
|
|
"""
|
|
# Configure logger for AudioProcessor
|
|
audio_logger = logging.getLogger("karaoke_gen.audio_processor")
|
|
audio_logger.setLevel(logging.INFO)
|
|
|
|
# Model configurations - use provided values or CLI defaults
|
|
effective_clean_model = clean_instrumental_model or "model_bs_roformer_ep_317_sdr_12.9755.ckpt"
|
|
effective_backing_models = backing_vocals_models or ["mel_band_roformer_karaoke_aufr33_viperx_sdr_10.1956.ckpt"]
|
|
effective_other_models = other_stems_models or ["htdemucs_6s.yaml"] # For 6-stem separation (bass, drums, etc.)
|
|
|
|
# FFmpeg command for combining audio files (must be a string, not a list)
|
|
ffmpeg_base_command = "ffmpeg -hide_banner -loglevel error -nostats -y"
|
|
|
|
return AudioProcessor(
|
|
logger=audio_logger,
|
|
log_level=logging.INFO,
|
|
log_formatter=None, # Not needed for our use case
|
|
model_file_dir=None, # No local models, using remote API
|
|
lossless_output_format="FLAC",
|
|
clean_instrumental_model=effective_clean_model,
|
|
backing_vocals_models=effective_backing_models,
|
|
other_stems_models=effective_other_models,
|
|
ffmpeg_base_command=ffmpeg_base_command
|
|
)
|
|
|
|
|
|
async def process_audio_separation(job_id: str) -> bool:
|
|
"""
|
|
Process audio separation for a job using karaoke_gen.AudioProcessor.
|
|
|
|
This is the main entry point for the audio worker.
|
|
Called asynchronously from the job submission endpoint.
|
|
|
|
Workflow:
|
|
1. Download audio from GCS
|
|
2. Stage 1: Separate with clean instrumental + other stems models (Modal API)
|
|
3. Stage 2: Separate vocals for backing vocals (Modal API)
|
|
4. Post-process: Combine instrumentals, normalize audio
|
|
5. Upload all stems to GCS
|
|
6. Mark job as AUDIO_COMPLETE
|
|
|
|
Args:
|
|
job_id: Job ID to process
|
|
|
|
Returns:
|
|
True if successful, False otherwise
|
|
"""
|
|
job_manager = JobManager()
|
|
storage = StorageService()
|
|
settings = get_settings()
|
|
|
|
# Create job logger for remote debugging FIRST
|
|
job_log = create_job_logger(job_id, "audio")
|
|
|
|
# Log immediately to verify logging is working
|
|
job_log.info("=== AUDIO WORKER STARTED ===")
|
|
job_log.info(f"Job ID: {job_id}")
|
|
|
|
# Set up log capture for AudioProcessor
|
|
log_handler = setup_job_logging(job_id, "audio", *AUDIO_WORKER_LOGGERS)
|
|
job_log.info(f"Log handler attached for {len(AUDIO_WORKER_LOGGERS)} loggers")
|
|
|
|
job = job_manager.get_job(job_id)
|
|
if not job:
|
|
logger.error(f"Job {job_id} not found")
|
|
job_log.error(f"Job {job_id} not found in Firestore!")
|
|
return False
|
|
|
|
# Create temporary working directory
|
|
temp_dir = tempfile.mkdtemp(prefix=f"karaoke_{job_id}_")
|
|
job_log.info(f"Created temp directory: {temp_dir}")
|
|
|
|
try:
|
|
# Use job_logging_context for proper log isolation when multiple jobs run concurrently
|
|
# This ensures logs from third-party libraries (karaoke_gen.audio_processor) are
|
|
# only captured by this job's handler, not handlers from other concurrent jobs
|
|
with job_logging_context(job_id):
|
|
job_log.info(f"Starting audio separation for {job.artist} - {job.title}")
|
|
logger.info(f"Starting audio separation for job {job_id}")
|
|
|
|
# Ensure AUDIO_SEPARATOR_API_URL is set
|
|
api_url = os.environ.get("AUDIO_SEPARATOR_API_URL")
|
|
if not api_url:
|
|
raise Exception("AUDIO_SEPARATOR_API_URL environment variable not set. "
|
|
"Cannot perform audio separation without remote API access.")
|
|
job_log.info(f"Audio separator API: {api_url}")
|
|
|
|
# Download audio file from GCS or URL
|
|
job_log.info("Downloading audio file...")
|
|
audio_path = await download_audio(job_id, temp_dir, storage, job, job_manager_instance=job_manager)
|
|
if not audio_path:
|
|
raise Exception("Failed to download audio file")
|
|
job_log.info(f"Audio downloaded: {os.path.basename(audio_path)}")
|
|
|
|
# Update progress using state_data (don't change status during parallel processing)
|
|
# The status is managed at a higher level - workers just track their progress
|
|
job_manager.update_state_data(job_id, 'audio_progress', {
|
|
'stage': 'separating_stage1',
|
|
'progress': 10,
|
|
'message': 'Starting audio separation (Stage 1: Clean instrumental)'
|
|
})
|
|
|
|
# Create AudioProcessor instance (reuses karaoke_gen code)
|
|
# Use model configuration from job if provided, otherwise use defaults
|
|
job_log.info("Creating AudioProcessor instance...")
|
|
if job.clean_instrumental_model:
|
|
job_log.info(f" Using clean instrumental model: {job.clean_instrumental_model}")
|
|
if job.backing_vocals_models:
|
|
job_log.info(f" Using backing vocals models: {job.backing_vocals_models}")
|
|
if job.other_stems_models:
|
|
job_log.info(f" Using other stems models: {job.other_stems_models}")
|
|
|
|
audio_processor = create_audio_processor(
|
|
temp_dir,
|
|
clean_instrumental_model=job.clean_instrumental_model,
|
|
backing_vocals_models=job.backing_vocals_models,
|
|
other_stems_models=job.other_stems_models
|
|
)
|
|
|
|
# Format artist-title for file naming (matches CLI behavior)
|
|
artist_title = f"{job.artist} - {job.title}"
|
|
|
|
# Run audio separation (calls Modal API internally)
|
|
# This returns a dict with paths to all separated stems
|
|
job_log.info("Starting audio separation (this may take 5-10 minutes)...")
|
|
job_log.info(" Stage 1: Clean instrumental separation (MDX models)")
|
|
job_log.info(" Stage 2: Backing vocals separation (Demucs model)")
|
|
logger.info(f"Job {job_id}: Calling audio_processor.process_audio_separation()")
|
|
separation_result = audio_processor.process_audio_separation(
|
|
audio_file=audio_path,
|
|
artist_title=artist_title,
|
|
track_output_dir=temp_dir
|
|
)
|
|
|
|
job_log.info("Audio separation complete!")
|
|
job_log.info(f" Generated {len(separation_result)} stem files")
|
|
logger.info(f"Job {job_id}: Audio separation complete, organizing results")
|
|
|
|
# Update progress using state_data (don't change status during parallel processing)
|
|
job_manager.update_state_data(job_id, 'audio_progress', {
|
|
'stage': 'audio_complete',
|
|
'progress': 45,
|
|
'message': 'Audio separation complete, uploading stems'
|
|
})
|
|
|
|
# Upload all stems to GCS
|
|
await upload_separation_results(job_id, separation_result, storage, job_manager)
|
|
|
|
logger.info(f"Job {job_id}: All stems uploaded successfully")
|
|
|
|
# Mark audio processing complete
|
|
# This will check if lyrics are also complete and transition to next stage if so
|
|
job_manager.mark_audio_complete(job_id)
|
|
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.error(f"Job {job_id}: Audio separation failed: {e}", exc_info=True)
|
|
job_manager.mark_job_failed(
|
|
job_id=job_id,
|
|
error_message=f"Audio separation failed: {str(e)}",
|
|
error_details={"stage": "audio_separation", "error": str(e)}
|
|
)
|
|
return False
|
|
|
|
finally:
|
|
# Remove log handler to avoid duplicate logging on future runs
|
|
for logger_name in AUDIO_WORKER_LOGGERS:
|
|
try:
|
|
logging.getLogger(logger_name).removeHandler(log_handler)
|
|
except Exception:
|
|
pass
|
|
|
|
# Cleanup temporary directory
|
|
if os.path.exists(temp_dir):
|
|
shutil.rmtree(temp_dir)
|
|
logger.debug(f"Cleaned up temp directory: {temp_dir}")
|
|
|
|
|
|
async def download_audio(
|
|
job_id: str,
|
|
temp_dir: str,
|
|
storage: StorageService,
|
|
job,
|
|
job_manager_instance: JobManager = None
|
|
) -> Optional[str]:
|
|
"""
|
|
Download or fetch audio file to local temp directory.
|
|
|
|
Handles two cases:
|
|
1. Uploaded file: Download from GCS using input_media_gcs_path
|
|
2. URL (YouTube, etc.): Download using yt-dlp or other tools
|
|
|
|
Args:
|
|
job_id: Job ID
|
|
temp_dir: Temporary directory to save to
|
|
storage: StorageService instance
|
|
job: Job object with URL or GCS path
|
|
job_manager_instance: Optional JobManager instance for updating job metadata
|
|
|
|
Returns:
|
|
Path to downloaded audio file, or None if failed
|
|
"""
|
|
try:
|
|
# Case 1: File was uploaded to GCS
|
|
if job.input_media_gcs_path:
|
|
logger.info(f"Job {job_id}: Downloading uploaded file from GCS: {job.input_media_gcs_path}")
|
|
local_path = os.path.join(temp_dir, job.filename or "input.flac")
|
|
storage.download_file(job.input_media_gcs_path, local_path)
|
|
logger.info(f"Job {job_id}: Downloaded uploaded file to {local_path}")
|
|
return local_path
|
|
|
|
# Case 2: URL download (from file_urls if already downloaded, or from job.url)
|
|
if job.file_urls and job.file_urls.get('input'):
|
|
# Already downloaded and stored in GCS
|
|
input_url = job.file_urls.get('input')
|
|
local_path = os.path.join(temp_dir, "input.flac")
|
|
storage.download_file(input_url, local_path)
|
|
logger.info(f"Job {job_id}: Downloaded audio from GCS: {input_url}")
|
|
return local_path
|
|
|
|
# Case 3: Fresh URL that needs downloading
|
|
if job.url:
|
|
logger.info(f"Job {job_id}: Downloading from URL: {job.url}")
|
|
|
|
# Use provided job_manager or create new one
|
|
jm = job_manager_instance or JobManager()
|
|
|
|
local_path = await download_from_url(
|
|
job.url,
|
|
temp_dir,
|
|
job.artist,
|
|
job.title,
|
|
job_manager=jm,
|
|
job_id=job_id
|
|
)
|
|
|
|
if local_path and os.path.exists(local_path):
|
|
# Upload to GCS and update job
|
|
gcs_path = f"jobs/{job_id}/input/{os.path.basename(local_path)}"
|
|
url = storage.upload_file(local_path, gcs_path)
|
|
|
|
# Update job with GCS path for lyrics worker
|
|
jm.update_job(job_id, {'input_media_gcs_path': gcs_path})
|
|
jm.update_file_url(job_id, 'input', 'audio', url)
|
|
|
|
logger.info(f"Job {job_id}: Downloaded and uploaded audio to GCS: {gcs_path}")
|
|
return local_path
|
|
else:
|
|
logger.error(f"Job {job_id}: Failed to download from URL: {job.url}")
|
|
return None
|
|
|
|
logger.error(f"Job {job_id}: No input source found (no GCS path, file_urls, or URL)")
|
|
return None
|
|
|
|
except Exception as e:
|
|
logger.error(f"Job {job_id}: Failed to download audio: {e}", exc_info=True)
|
|
return None
|
|
|
|
|
|
async def upload_separation_results(
|
|
job_id: str,
|
|
separation_result: Dict[str, Any],
|
|
storage: StorageService,
|
|
job_manager: JobManager
|
|
) -> None:
|
|
"""
|
|
Upload all audio separation results to GCS.
|
|
|
|
The separation_result dict from AudioProcessor.process_audio_separation() contains:
|
|
- clean_instrumental: Dict with 'vocals' and 'instrumental' paths
|
|
- other_stems: Dict with stem paths (bass, drums, guitar, piano, other)
|
|
- backing_vocals: Dict with model-keyed paths to lead_vocals and backing_vocals
|
|
- combined_instrumentals: Dict with model-keyed paths to instrumental+BV files
|
|
|
|
Args:
|
|
job_id: Job ID
|
|
separation_result: Result dict from AudioProcessor
|
|
storage: StorageService instance
|
|
job_manager: JobManager instance
|
|
"""
|
|
logger.info(f"Job {job_id}: Uploading separation results to GCS")
|
|
|
|
# Upload clean instrumental + vocals (Stage 1, clean model)
|
|
if separation_result.get("clean_instrumental"):
|
|
clean = separation_result["clean_instrumental"]
|
|
|
|
if clean.get("instrumental") and os.path.exists(clean["instrumental"]):
|
|
gcs_path = f"jobs/{job_id}/stems/instrumental_clean.flac"
|
|
url = storage.upload_file(clean["instrumental"], gcs_path)
|
|
job_manager.update_file_url(job_id, 'stems', 'instrumental_clean', url)
|
|
logger.info(f"Job {job_id}: Uploaded clean instrumental")
|
|
|
|
if clean.get("vocals") and os.path.exists(clean["vocals"]):
|
|
gcs_path = f"jobs/{job_id}/stems/vocals_clean.flac"
|
|
url = storage.upload_file(clean["vocals"], gcs_path)
|
|
job_manager.update_file_url(job_id, 'stems', 'vocals_clean', url)
|
|
logger.info(f"Job {job_id}: Uploaded clean vocals")
|
|
|
|
# Upload other stems (Stage 1, htdemucs 6-stem)
|
|
if separation_result.get("other_stems"):
|
|
for stem_name, stem_value in separation_result["other_stems"].items():
|
|
# Handle both string paths and nested dicts
|
|
if isinstance(stem_value, str):
|
|
stem_path = stem_value
|
|
elif isinstance(stem_value, dict):
|
|
# Some models return nested dicts like {"path": "/path/to/file"}
|
|
stem_path = stem_value.get("path") or stem_value.get("file")
|
|
logger.debug(f"Job {job_id}: other_stems[{stem_name}] is dict: {stem_value}")
|
|
else:
|
|
logger.warning(f"Job {job_id}: Unexpected type for other_stems[{stem_name}]: {type(stem_value)}")
|
|
continue
|
|
|
|
if stem_path and isinstance(stem_path, str) and os.path.exists(stem_path):
|
|
gcs_path = f"jobs/{job_id}/stems/{stem_name.lower()}.flac"
|
|
url = storage.upload_file(stem_path, gcs_path)
|
|
job_manager.update_file_url(job_id, 'stems', stem_name.lower(), url)
|
|
logger.info(f"Job {job_id}: Uploaded {stem_name} stem")
|
|
|
|
# Upload backing vocals separation (Stage 2)
|
|
if separation_result.get("backing_vocals"):
|
|
# backing_vocals is a dict keyed by model name
|
|
for model_name, bv_stems in separation_result["backing_vocals"].items():
|
|
if bv_stems.get("lead_vocals") and os.path.exists(bv_stems["lead_vocals"]):
|
|
gcs_path = f"jobs/{job_id}/stems/lead_vocals.flac"
|
|
url = storage.upload_file(bv_stems["lead_vocals"], gcs_path)
|
|
job_manager.update_file_url(job_id, 'stems', 'lead_vocals', url)
|
|
logger.info(f"Job {job_id}: Uploaded lead vocals")
|
|
|
|
if bv_stems.get("backing_vocals") and os.path.exists(bv_stems["backing_vocals"]):
|
|
gcs_path = f"jobs/{job_id}/stems/backing_vocals.flac"
|
|
url = storage.upload_file(bv_stems["backing_vocals"], gcs_path)
|
|
job_manager.update_file_url(job_id, 'stems', 'backing_vocals', url)
|
|
logger.info(f"Job {job_id}: Uploaded backing vocals")
|
|
|
|
# Only process first model (we typically only use one backing vocals model)
|
|
break
|
|
|
|
# Upload combined instrumentals (instrumental + backing vocals)
|
|
if separation_result.get("combined_instrumentals"):
|
|
# combined_instrumentals is a dict keyed by model name
|
|
for model_name, combined_path in separation_result["combined_instrumentals"].items():
|
|
if combined_path and os.path.exists(combined_path):
|
|
gcs_path = f"jobs/{job_id}/stems/instrumental_with_backing.flac"
|
|
url = storage.upload_file(combined_path, gcs_path)
|
|
job_manager.update_file_url(job_id, 'stems', 'instrumental_with_backing', url)
|
|
logger.info(f"Job {job_id}: Uploaded instrumental with backing vocals")
|
|
|
|
# Only process first model
|
|
break
|
|
|
|
# Store instrumental options in state_data for later selection
|
|
instrumental_options = {}
|
|
if separation_result.get("clean_instrumental", {}).get("instrumental"):
|
|
instrumental_options["clean"] = f"jobs/{job_id}/stems/instrumental_clean.flac"
|
|
if separation_result.get("combined_instrumentals"):
|
|
instrumental_options["with_backing"] = f"jobs/{job_id}/stems/instrumental_with_backing.flac"
|
|
|
|
if instrumental_options:
|
|
job_manager.update_state_data(job_id, 'instrumental_options', instrumental_options)
|
|
logger.info(f"Job {job_id}: Stored instrumental options: {list(instrumental_options.keys())}")
|
|
|