Added python services to log (debug or higher). Reworked live lyric preview to change custom background only for preview window and fixed mobile display.

This commit is contained in:
translator committed 2026-03-05 17:49:50 +09:00
1 parent 0e382caf47
commit 42891006de
22 files changed
+756 -219

No files matched your search

+51 -10
View File
@@ -95,13 +95,24 @@ _flags_lock = threading.Lock()
# ---------------------------------------------------------------------------
# Thread pools
# Two executors:
# _general_executor — multi-worker for CPU-bound tasks (e.g. ABC conversion)
# _gpu_executor — single-worker for GPU-exclusive tasks (PyTorch + TF)
# serialises Demucs/BS-RoFormer and Omnizart so they
# never compete for VRAM; FIFO by submission order.
# _general_executor — multi-worker for CPU-bound tasks (e.g. ABC conversion)
# _gpu_executor — single-worker for GPU-exclusive tasks (PyTorch + TF)
# serialises Demucs/BS-RoFormer and Omnizart so they
# never compete for VRAM; FIFO by submission order.
# _transcription_executor — single-worker queue ONLY for lyric transcription and
# LRC alignment. Kept separate from _gpu_executor so
# stem separation can run without blocking, but still
# prevents two transcription/alignment jobs from
# competing for VRAM. Jobs are submitted here instead
# of _gpu_executor for these job types.
# ---------------------------------------------------------------------------
_general_executor = ThreadPoolExecutor(max_workers=4)
_gpu_executor = ThreadPoolExecutor(max_workers=1)
_general_executor = ThreadPoolExecutor(max_workers=4)
_gpu_executor = ThreadPoolExecutor(max_workers=1)
_transcription_executor = ThreadPoolExecutor(max_workers=1)
# Queue-depth counter for the transcription executor (includes the running job).
_tx_queue_lock = threading.Lock()
_tx_queue_depth = 0
# ---------------------------------------------------------------------------
@@ -207,14 +218,44 @@ def submit(fn: Callable, *args, **kwargs):
def submit_gpu(fn: Callable, *args, **kwargs):
"""Submit a GPU-exclusive background task (single-worker FIFO queue).
Use for any job that loads PyTorch or TensorFlow models onto the GPU:
stem separation (Demucs / BS-RoFormer), MIDI transcription (Omnizart),
lyrics alignment (wav2vec2 + Demucs vocal extraction). Tasks run
strictly one-at-a-time in the order they were submitted.
Use for stem separation (Demucs / BS-RoFormer) and MIDI transcription.
Tasks run strictly one-at-a-time in the order they were submitted.
"""
_gpu_executor.submit(fn, *args, **kwargs)
def submit_transcription(job_id: str, fn: Callable, *args, **kwargs):
"""Submit a lyric transcription or LRC alignment job.
Uses a dedicated single-worker executor so transcription jobs are
serialised independently of stem separation, preventing VRAM contention.
Shows queue position in the job's progress field while waiting.
"""
global _tx_queue_depth
with _tx_queue_lock:
_tx_queue_depth += 1
position = _tx_queue_depth
if position > 1:
update_job(job_id, progress=f"Queued (position {position} of {position})…")
def _wrapper():
global _tx_queue_depth
# Recalculate position at start time (earlier jobs may have finished).
with _tx_queue_lock:
pos = _tx_queue_depth
if pos > 1:
update_job(job_id, progress=f"Queued — waiting for {pos - 1} job(s) to finish…")
try:
fn(*args, **kwargs)
finally:
with _tx_queue_lock:
_tx_queue_depth -= 1
_transcription_executor.submit(_wrapper)
def _record_to_dict(record: JobRecord) -> dict:
return {
"id": record.id,
+41
View File
@@ -0,0 +1,41 @@
# StemEngine Python logging configuration.
# Edit `root.level` to control verbosity:
# DEBUG — full trace: job steps, model loads, GPU memory events, alignment details
# INFO — normal operational messages
# WARNING — warnings and errors only (quiet production mode)
#
# Individual loggers can be overridden in the `loggers` section below.
version: 1
disable_existing_loggers: false
formatters:
standard:
format: "%(asctime)s %(levelname)-8s %(name)s — %(message)s"
datefmt: "%H:%M:%S"
handlers:
console:
class: logging.StreamHandler
formatter: standard
stream: ext://sys.stdout
root:
level: WARNING
handlers: [console]
loggers:
# ── StemEngine application loggers ──────────────────────────────────────────
backend:
level: WARNING # change to DEBUG to see all app-level debug output
propagate: true
# ── Third-party: kept at WARNING to avoid noise ───────────────────────────
httpx: { level: WARNING, propagate: true }
httpcore: { level: WARNING, propagate: true }
urllib3: { level: WARNING, propagate: true }
multipart: { level: WARNING, propagate: true }
watchfiles: { level: WARNING, propagate: true }
asyncio: { level: WARNING, propagate: true }
h11: { level: WARNING, propagate: true }
uvicorn.access: { level: WARNING, propagate: true }
+10
View File
@@ -2,9 +2,19 @@
StemEngine — FastAPI application entry point.
Mounts all API routers and serves the built React frontend as static files.
"""
import logging
import logging.config
from pathlib import Path
from fastapi import FastAPI
# Load logging config from the discrete YAML file next to this package.
# Edit backend/logging.yaml to change log levels without touching code.
_log_cfg = Path(__file__).parent / "logging.yaml"
if _log_cfg.exists():
import yaml
with open(_log_cfg) as _f:
logging.config.dictConfig(yaml.safe_load(_f))
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from fastapi.responses import FileResponse
+3 -2
View File
@@ -51,9 +51,10 @@ cryptography>=42.0
# MIDI transcription via basic-pitch
# Installed separately in Dockerfile with --no-deps to avoid TF/numpy conflict.
# Runtime inference uses onnxruntime (above) instead of TensorFlow.
# pretty_midi is basic-pitch's MIDI output library; must be installed separately
# since it's normally pulled in as a dep of basic-pitch.
# pretty_midi and mir_eval are basic-pitch deps that must be installed separately
# since --no-deps skips them along with tensorflow.
pretty_midi>=0.2.10
mir_eval>=0.7
# Syllable hyphenation (pure-Python, for per-syllable ELRC timing)
pyphen>=0.14.0
+6 -2
View File
@@ -78,6 +78,7 @@ class LyricTranscribeRequest(BaseModel):
diarize: bool = False # run speaker diarization (WhisperX only)
lrc_power_preset: str = "standard" # 'eco' | 'standard' | 'performance'
phoneme_align: bool = False # use phonemizer + phoneme model for syllable-level timestamps
two_pass: bool = False # run a second VAD pass to fill gaps > 2 s
lyrics_source: str = "transcribe" # 'transcribe' | 'auto' | 'tag' | 'file' — when not 'transcribe',
# skip Whisper and force-align existing lyrics instead
existing_lyrics: Optional[str] = None # lyrics text (plain or LRC) to align; overrides lyrics_source
@@ -407,7 +408,8 @@ def lrc_project(project_id: str, body: LrcRequest):
job_id = jm.create_job("lrc_align")
set_project_lrc_job(project_id, body.stem_name, job_id)
jm.submit_gpu(
jm.submit_transcription(
job_id,
run_lrc_alignment,
job_id,
audio_path,
@@ -481,7 +483,8 @@ def lyric_transcribe_project(project_id: str, body: LyricTranscribeRequest):
# frontend always shows the most-recent AI transcription for that stem.
set_project_lrc_job(project_id, f"{body.stem_name}_whisper", job_id)
jm.submit_gpu(
jm.submit_transcription(
job_id,
run_lyric_transcription,
job_id,
stem_path,
@@ -497,6 +500,7 @@ def lyric_transcribe_project(project_id: str, body: LyricTranscribeRequest):
diarize=body.diarize,
power_preset=body.lrc_power_preset,
phoneme_align=body.phoneme_align,
two_pass=body.two_pass,
existing_lyrics=resolved_lyrics,
)
return {"job_id": job_id, "output_dir": str(output_dir)}
+19 -18
View File
@@ -1,8 +1,12 @@
from music21 import environment, converter, metadata, stream
from pathlib import Path
import subprocess
import os
import logging
import traceback
import os
import subprocess
from pathlib import Path
from music21 import environment, converter, metadata, stream
log = logging.getLogger(__name__)
def _post_process_abc_content(abc_text: str, title: str) -> str:
@@ -47,7 +51,7 @@ def convert_midi_to_abc(
for midi_path in midi_paths:
part_name = Path(midi_path).stem
part_title = f"{song_title} ({part_name})"
print(f" -> Processing part: {part_name}.mid")
log.debug(" -> Processing part: %s.mid", part_name)
temp_midi_path = temp_dir / f"{part_title}.mid"
temp_abc_path = temp_dir / f"{part_title}.abc"
@@ -55,7 +59,7 @@ def convert_midi_to_abc(
try:
score_part = converter.parse(midi_path, forceSource=True)
if not score_part or not score_part.flatten().notesAndRests:
print(f" -> Skipping empty MIDI: {Path(midi_path).name}") # noqa
log.debug(" -> Skipping empty MIDI: %s", Path(midi_path).name)
continue
single_part_score = stream.Score()
@@ -71,12 +75,11 @@ def convert_midi_to_abc(
result = subprocess.run(cmd, capture_output=True, text=True, check=False)
if result.returncode != 0:
print(f" -> ERROR: midi2abc failed for {part_name}: "
f"{result.stderr}")
log.debug(" -> ERROR: midi2abc failed for %s: %s", part_name, result.stderr)
continue
if not temp_abc_path.exists() or temp_abc_path.stat().st_size == 0:
print(f" -> ERROR: midi2abc created an empty file for {part_name}.")
log.debug(" -> ERROR: midi2abc created an empty file for %s.", part_name)
continue
with open(temp_abc_path, 'r', encoding='utf-8') as f:
@@ -84,12 +87,11 @@ def convert_midi_to_abc(
abc_results[part_name] = _post_process_abc_content(
abc_text_content, part_title)
print(f" -> Successfully converted {part_name} to ABC.")
log.debug(" -> Successfully converted %s to ABC.", part_name)
except Exception as e:
print(
f" -> ERROR during ABC conversion for {part_name}: {e}")
traceback.print_exc()
log.debug(" -> ERROR during ABC conversion for %s: %s", part_name, e)
log.debug(traceback.format_exc())
finally:
if temp_midi_path.exists():
os.remove(temp_midi_path)
@@ -108,21 +110,20 @@ def generate_mix_abc(mix_midi_path: str, song_title: str) -> str | None:
temp_abc_path = temp_dir / f"{mix_title}.abc"
try:
print("\n -> Creating combined 'Mix' ABC score...")
log.debug(" -> Creating combined 'Mix' ABC score...")
cmd = ['midi2abc', mix_midi_path, '-o', str(temp_abc_path)]
result = subprocess.run(cmd, capture_output=True, text=True, check=False)
if result.returncode == 0 and temp_abc_path.exists():
raw_mix_abc = temp_abc_path.read_text(encoding='utf-8')
cleaned_mix_abc = _post_process_abc_content(raw_mix_abc, mix_title)
print(" -> Successfully created 'Mix' ABC.")
log.debug(" -> Successfully created 'Mix' ABC.")
return cleaned_mix_abc
else:
print(
f" -> ERROR: midi2abc failed for Mix: {result.stderr}")
log.debug(" -> ERROR: midi2abc failed for Mix: %s", result.stderr)
return None
except Exception as e:
print(f" -> ERROR during Mix ABC conversion: {e}")
log.debug(" -> ERROR during Mix ABC conversion: %s", e)
return None
finally:
if temp_abc_path.exists():
+7 -1
View File
@@ -2,6 +2,7 @@
LRC alignment service for creating enhanced ELRC files with word-level timestamps.
Uses the lyrics-sync module for vocal extraction and forced alignment.
"""
import logging
import os
import re
import sys
@@ -12,6 +13,8 @@ from typing import Optional, List, Tuple
from .. import job_manager as jm
from ..config import JOBS_DIR, LRC_SYNC_DIR
log = logging.getLogger(__name__)
from .metadata_util import (
extract_lyrics_from_file,
lyrics_to_plaintext,
@@ -154,7 +157,9 @@ def run_lrc_alignment(
else:
out_path = LRC_SYNC_DIR / job_id
out_path.mkdir(parents=True, exist_ok=True)
log.debug("[%s] run_lrc_alignment start — source=%s vocals=%s precision=%s",
job_id, lyrics_source, is_vocals, timing_precision)
try:
# Import lyrics-sync here to handle any import issues gracefully
try:
@@ -235,6 +240,7 @@ def run_lrc_alignment(
)
except Exception as exc:
log.debug("[%s] lrc_alignment failed — %s: %s", job_id, type(exc).__name__, exc)
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
+29 -10
View File
@@ -49,9 +49,13 @@ try:
except Exception:
pass
import logging
from .. import job_manager as jm
from ..config import JOBS_DIR, LRC_SYNC_DIR
log = logging.getLogger(__name__)
@dataclass
class WordTimestamp:
@@ -1034,6 +1038,7 @@ def run_lyric_transcription(
diarize: bool = False,
power_preset: str = "standard",
phoneme_align: bool = False,
two_pass: bool = False,
existing_lyrics: Optional[str] = None,
):
"""
@@ -1064,7 +1069,9 @@ def run_lyric_transcription(
else:
out_path = LRC_SYNC_DIR / job_id
out_path.mkdir(parents=True, exist_ok=True)
log.debug("[%s] run_lyric_transcription start — model=%s engine=%s phoneme=%s two_pass=%s",
job_id, model, engine, phoneme_align, two_pass)
try:
if engine == "whisperx":
# ── WhisperX path: faster-whisper backend + phoneme-level alignment ──
@@ -1087,11 +1094,13 @@ def run_lyric_transcription(
return
device = "cuda" if torch.cuda.is_available() else "cpu"
log.debug("[%s] device=%s power_preset=%s", job_id, device, power_preset)
_wx_preset = _WHISPERX_POWER_PRESETS.get(power_preset, _WHISPERX_POWER_PRESETS['standard'])
batch_size = _wx_preset['batch_size']
compute_type = _wx_preset['compute_type_cuda'] if device == "cuda" else "int8"
# Auto-ensure vocals stem for best transcription quality.
log.debug("[%s] ensuring vocals stem from %s", job_id, audio_path)
audio_path = _ensure_vocals(audio_path, job_id)
# Music-tuned ASR options:
@@ -1148,16 +1157,18 @@ def run_lyric_transcription(
vad_options=_vad_opts,
)
jm.update_job(job_id, progress=f"Pass 1: Transcribing with WhisperX ({model}, {power_preset})...")
jm.update_job(job_id, progress=f"Transcribing with WhisperX ({model}, {power_preset})...")
wx_result = wx_model.transcribe(audio_arr, batch_size=batch_size)
detected_lang = wx_result.get("language") or language or "unknown"
log.debug("[%s] transcription done — lang=%s segments=%d",
job_id, detected_lang, len(wx_result.get("segments", [])))
# Pass 2: re-transcribe any gaps > 2 s that VAD may have missed
# (e.g. quieter verse sections after long instrumentals).
wx_result["segments"] = _fill_transcript_gaps(
wx_model, audio_arr, wx_result.get("segments", []),
batch_size, job_id, min_gap=2.0,
)
if two_pass:
# Re-transcribe any gaps > 2 s that VAD may have missed
wx_result["segments"] = _fill_transcript_gaps(
wx_model, audio_arr, wx_result.get("segments", []),
batch_size, job_id, min_gap=2.0,
)
# Free the Whisper model from VRAM before loading the alignment model.
# large-v2/v3 consume 3–6 GB; keeping both models resident simultaneously
@@ -1173,6 +1184,7 @@ def run_lyric_transcription(
return
# ── Alignment (word-level or phoneme-level) ─────────────────────────
log.debug("[%s] starting alignment — phoneme=%s", job_id, phoneme_align)
_use_phoneme = phoneme_align
_phoneme_orig_words: list = []
@@ -1438,6 +1450,8 @@ def run_lyric_transcription(
"word_count": sum(len(seg.words) for seg in result.segments),
}
log.debug("[%s] complete — %d output files: %s",
job_id, len(output_files), [f["name"] for f in output_files])
jm.update_job(
job_id,
status=jm.JobStatus.DONE,
@@ -1445,11 +1459,16 @@ def run_lyric_transcription(
files=output_files,
result=result_summary
)
except Exception as exc:
log.debug("[%s] failed — %s: %s", job_id, type(exc).__name__, exc)
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
# Best-effort GPU flush so a failed job doesn't strand VRAM.
# Best-effort GPU flush so a failed job doesn't strand VRAM for the next
# queued job. gc.collect() releases live tensor refs before empty_cache()
# can reclaim the CUDA memory they hold.
try:
import gc
gc.collect()
import torch
if torch.cuda.is_available():
torch.cuda.empty_cache()
+7
View File
@@ -2,12 +2,15 @@
Mix service: create static or dynamic mixes from separated stems.
CPU-bound (pydub) — uses the general executor, not the GPU queue.
"""
import logging
from pathlib import Path
from typing import Dict, List, Optional
from .. import job_manager as jm
from .preset_service import make_static, make_dynamic
log = logging.getLogger(__name__)
def run_mix(
job_id: str,
@@ -19,6 +22,8 @@ def run_mix(
dynamic_levels: Optional[Dict[str, float]] = None,
):
"""Background task: create a mix from separated stems."""
log.debug("[%s] run_mix start — preset=%s format=%s output=%s",
job_id, preset_type, output_format, output_path)
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Creating mix...")
try:
out = Path(output_path)
@@ -33,6 +38,7 @@ def run_mix(
else:
raise ValueError(f"Unknown preset type: {preset_type}")
log.debug("[%s] mix complete — output=%s", job_id, result_path)
jm.update_job(
job_id,
status=jm.JobStatus.DONE,
@@ -40,6 +46,7 @@ def run_mix(
files=[{"name": out.name, "path": result_path, "type": preset_type}],
)
except Exception as exc:
log.debug("[%s] mix failed — %s: %s", job_id, type(exc).__name__, exc)
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
+9
View File
@@ -3,6 +3,7 @@ Full pipeline: separate → transcribe → abc-convert.
Supports both Demucs (CLI) and BS-RoFormer (Python API) for the separation stage.
Uses project-based folder structure for organized output.
"""
import logging
import sys
import subprocess
import time
@@ -16,6 +17,8 @@ from .project_namer import project_name, make_project_dirs
from .separator_service import BSR_PREFIX, BSR_STEM_MODE
from . import abc_generator
log = logging.getLogger(__name__)
def run_pipeline(
job_id: str,
@@ -34,6 +37,9 @@ def run_pipeline(
Creates organized project folder structure under MEDIA_DIR.
"""
log.debug("[%s] run_pipeline start — model=%s format=%s transcribe_mode=%s input=%s",
job_id, sep_model, sep_format, transcribe_mode, input_path)
# Read metadata for project naming
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Reading metadata...")
metadata = read_metadata(input_path)
@@ -78,6 +84,7 @@ def run_pipeline(
job_id, input_path, sep_model, sep_format, stems_dir
)
except Exception as exc:
log.debug("[%s] pipeline separation failed — %s: %s", job_id, type(exc).__name__, exc)
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Separation failed.")
return
@@ -152,6 +159,8 @@ def run_pipeline(
except Exception as exc:
jm.update_job(job_id, progress=f"[3/3] ABC conversion error: {exc}")
log.debug("[%s] pipeline complete — project=%s stems=%d files=%d",
job_id, proj_name, len(stem_files), len(all_files))
jm.update_job(
job_id, status=jm.JobStatus.DONE,
progress="Pipeline complete.",
+10
View File
@@ -4,6 +4,7 @@ Stem separation service with project-based folder structure and preset mixing.
- BS-RoFormer → Python API (in-process, no subprocess needed)
- Presets → Instrumental, Static, Dynamic mixing via preset_service
"""
import logging
import os
import subprocess
import sys
@@ -24,6 +25,8 @@ from .project_namer import project_name, make_project_dirs
from .preset_service import make_instrumental, make_static, make_dynamic
from .project_metadata import write_project_metadata
log = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Stem file naming helper
# ---------------------------------------------------------------------------
@@ -229,6 +232,9 @@ def run_separation(
Static/ (if preset is static)
Dynamic/ (if preset is dynamic)
"""
log.debug("[%s] run_separation start — model=%s format=%s preset=%s input=%s",
job_id, model, output_format, preset_type, input_path)
# Read metadata for project naming
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Reading metadata...")
metadata = read_metadata(input_path)
@@ -565,6 +571,8 @@ def run_separation(
jm.append_job_files(parent_job_id, stem_entries)
project_id_result = None
log.debug("[%s] separation complete — project=%s stems=%d files=%d",
job_id, proj_name, len(stem_files), len(all_files))
jm.update_job(
job_id,
status=jm.JobStatus.DONE,
@@ -600,8 +608,10 @@ def run_separation(
"The GPU was automatically reset — please retry the separation. "
"If the error keeps occurring, restart the container."
)
log.debug("[%s] separation GPU error — %s: %s", job_id, type(exc).__name__, err_str[:200])
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=user_msg, progress="GPU error — retry to use reset GPU.")
else:
log.debug("[%s] separation failed — %s: %s", job_id, type(exc).__name__, exc)
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=err_str, progress="Error.")
+6
View File
@@ -4,11 +4,14 @@ Supports two modes mapped to inference parameter presets:
polyphonic — general multi-instrument transcription
melody — single-voice / vocal melody (melodia trick, min frequency)
"""
import logging
from pathlib import Path
from .. import job_manager as jm
from ..config import JOBS_DIR, TRANSCRIBE_PRESETS
log = logging.getLogger(__name__)
def run_transcription(job_id: str, input_path: str, mode: str):
"""
@@ -18,6 +21,7 @@ def run_transcription(job_id: str, input_path: str, mode: str):
from basic_pitch.inference import predict
from basic_pitch import ICASSP_2022_MODEL_PATH
log.debug("[%s] run_transcription start — mode=%s input=%s", job_id, mode, input_path)
output_dir = JOBS_DIR / job_id
output_dir.mkdir(parents=True, exist_ok=True)
@@ -45,6 +49,7 @@ def run_transcription(job_id: str, input_path: str, mode: str):
dest = output_dir / output_name
midi_data.write(str(dest))
log.debug("[%s] transcription complete — output=%s", job_id, dest.name)
jm.update_job(
job_id,
status=jm.JobStatus.DONE,
@@ -53,4 +58,5 @@ def run_transcription(job_id: str, input_path: str, mode: str):
)
except Exception as exc:
log.debug("[%s] transcription failed — %s: %s", job_id, type(exc).__name__, exc)
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.")
+2 -2
View File
@@ -19,8 +19,8 @@
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<meta name="apple-mobile-web-app-title" content="StemEngine" />
<link rel="apple-touch-icon" href="/icon.svg" />
<script type="module" crossorigin src="/assets/index-BPMVHjDg.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-DBmWYVxf.css">
<script type="module" crossorigin src="/assets/index-Dydw3m3a.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-DVL4P0ML.css">
</head>
<body>
<div id="root"></div>
+11
View File
@@ -10,6 +10,7 @@
"dependencies": {
"@tanstack/react-query": "^5.0.0",
"@tonejs/midi": "^2.0.28",
"abcjs": "^6.6.2",
"lucide-react": "^0.294.0",
"react": "^18.2.0",
"react-dom": "^18.2.0",
@@ -1214,6 +1215,16 @@
"vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0"
}
},
"node_modules/abcjs": {
"version": "6.6.2",
"resolved": "https://registry.npmjs.org/abcjs/-/abcjs-6.6.2.tgz",
"integrity": "sha512-YLbp5lYUq0uOywWZx9EuTdm0TcflKZi7hOzz366A/LFl3qoAXSYIjznJQmr/VeHg8NcLxZYoN8dLi7PqCpxKEA==",
"license": "MIT",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/paulrosen"
}
},
"node_modules/any-promise": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz",
+3 -2
View File
@@ -11,10 +11,11 @@
"dependencies": {
"@tanstack/react-query": "^5.0.0",
"@tonejs/midi": "^2.0.28",
"tone": "^14.7.0",
"abcjs": "^6.6.2",
"lucide-react": "^0.294.0",
"react": "^18.2.0",
"react-dom": "^18.2.0"
"react-dom": "^18.2.0",
"tone": "^14.7.0"
},
"devDependencies": {
"@types/react": "^18.2.37",
+2
View File
@@ -389,6 +389,7 @@ export const lyricTranscribeProject = (
diarize?: boolean
lrcPowerPreset?: string
phonemeAlign?: boolean
twoPass?: boolean
}
) => {
const wordTs = opts?.wordTimestamps ?? true
@@ -414,6 +415,7 @@ export const lyricTranscribeProject = (
diarize: opts?.diarize ?? false,
lrc_power_preset: opts?.lrcPowerPreset ?? 'standard',
phoneme_align: phonemeAlign,
two_pass: opts?.twoPass ?? false,
}),
})
}
@@ -185,6 +185,7 @@ export default function AutoPipelineDialog({ target, onClose, onComplete, onOpen
wordTimestamps: wordTs,
diarize: step.diarize,
phonemeAlign: step.phonemeAlign,
twoPass: step.twoPass ?? false,
})
await pollUntilDone(transcribeJobId, ctrl.signal)
}, ctrl.signal, retry)
+37 -19
View File
@@ -843,6 +843,7 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
// ── Fullscreen ────────────────────────────────────────────────────────────
const fullscreenRef = useRef<HTMLDivElement | null>(null)
const [isFullscreen, setIsFullscreen] = useState(false)
const [isNarrow, setIsNarrow] = useState(typeof window !== 'undefined' && window.innerWidth < 640)
// Scale font size up in fullscreen so lyrics fill the larger canvas
const fontSize = isFullscreen ? Math.round(baseFontSize * 1.65) : baseFontSize
@@ -853,6 +854,12 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
return () => document.removeEventListener('fullscreenchange', onFsChange)
}, [])
useEffect(() => {
function check() { setIsNarrow(window.innerWidth < 640) }
window.addEventListener('resize', check)
return () => window.removeEventListener('resize', check)
}, [])
function toggleFullscreen() {
if (!isFullscreen) fullscreenRef.current?.requestFullscreen()
else document.exitFullscreen()
@@ -1537,8 +1544,7 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
function renderTransport() {
return (
<div
style={{ backgroundColor: effectiveBg }}
className="flex items-center gap-1.5 px-2.5 py-2 border-t border-white/10"
className={`flex items-center gap-1.5 px-2.5 py-2 border-t border-white/10 ${isFullscreen ? 'bg-black/40' : 'bg-slate-100/80 dark:bg-gray-800/80'}`}
>
<button
onClick={handleReset}
@@ -1604,8 +1610,7 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
if (!isMultiStem || !stems) return null
return (
<div
style={{ backgroundColor: effectiveBg }}
className="space-y-1.5 px-3 pb-2.5 pt-1.5 border-t border-white/10"
className={`space-y-1.5 px-3 pb-2.5 pt-1.5 border-t border-white/10 ${isFullscreen ? 'bg-black/40' : 'bg-slate-100/80 dark:bg-gray-800/80'}`}
>
{stems.map(stem => (
<div key={stem.name} className="flex items-center gap-2 min-h-[36px]">
@@ -1664,7 +1669,7 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
style={isFullscreen ? { backgroundColor: effectiveBg } : undefined}
>
{/* Visual style tabs + toolbar — scrollable on narrow screens */}
<div className={`flex items-center gap-1 px-2 py-1 border-b overflow-x-auto [&::-webkit-scrollbar]:hidden ${isFullscreen ? 'border-white/10' : 'bg-slate-100/80 dark:bg-gray-800/80 border-slate-200/30 dark:border-gray-700/50'}`}>
<div className={`flex items-center gap-1 px-2 py-1 border-b overflow-x-auto [&::-webkit-scrollbar]:hidden ${isFullscreen ? 'bg-black/40 border-white/10' : 'bg-slate-100/80 dark:bg-gray-800/80 border-slate-200/30 dark:border-gray-700/50'}`}>
{VISUAL_STYLES.map(s => (
<button
key={s.id}
@@ -1716,7 +1721,7 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
{/* Style picker panel */}
{showStylePicker && (
<div style={{ backgroundColor: effectiveBg }}>
<div className={isFullscreen ? 'bg-black/50' : 'bg-slate-100/80 dark:bg-gray-800/80'}>
<StylePicker style={style} onChange={patchStyle} hasCoverArt={!!coverArtUrl && artLoaded} />
{/* Speaker colour swatches — always shown so user knows if diarization is missing */}
<div className="px-3 py-2 border-b border-white/10 flex items-center gap-3 flex-wrap">
@@ -1792,35 +1797,48 @@ export default function LyricsLivePreview({ lrcText, audioUrl, stems, title, sty
/>
)}
{/* Content area: fullscreen with art → left panel + right column; otherwise stacked */}
{/* Content area: fullscreen with art → side-by-side on wide screens, stacked on mobile */}
{isFullscreen && coverArtUrl && artLoaded ? (
<div style={{ flex: '1 1 0', display: 'flex', flexDirection: 'row', overflow: 'hidden' }}>
{/* Left panel: large album art + artist + title */}
<div style={{ flex: '1 1 0', display: 'flex', flexDirection: isNarrow ? 'column' : 'row', overflow: 'hidden' }}>
{/* Left/top panel: album art + artist + title */}
<div style={{
width: '38%', flexShrink: 0, backgroundColor: effectiveBg,
display: 'flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center',
padding: '2rem', gap: '1.25rem', borderRight: '1px solid rgba(255,255,255,0.07)',
width: isNarrow ? '100%' : '38%',
height: isNarrow ? 'auto' : undefined,
flexShrink: 0,
display: 'flex', flexDirection: isNarrow ? 'row' : 'column',
alignItems: 'center', justifyContent: isNarrow ? 'flex-start' : 'center',
padding: isNarrow ? '0.75rem 1rem' : '2rem',
gap: isNarrow ? '0.75rem' : '1.25rem',
borderRight: isNarrow ? 'none' : '1px solid rgba(255,255,255,0.07)',
borderBottom: isNarrow ? '1px solid rgba(255,255,255,0.07)' : 'none',
backgroundColor: 'rgba(0,0,0,0.3)',
}}>
<img
src={coverArtUrl}
alt="Cover"
style={{
width: '100%', maxWidth: 240, aspectRatio: '1/1',
objectFit: 'cover', borderRadius: 12,
width: isNarrow ? 52 : '100%',
maxWidth: isNarrow ? 52 : 240,
aspectRatio: '1/1',
objectFit: 'cover',
borderRadius: isNarrow ? 8 : 12,
boxShadow: '0 20px 60px rgba(0,0,0,0.6)',
flexShrink: 0,
}}
onError={e => { (e.currentTarget as HTMLImageElement).style.display = 'none' }}
/>
<div style={{ textAlign: 'center', maxWidth: '100%' }}>
<div style={{ textAlign: isNarrow ? 'left' : 'center', maxWidth: '100%', minWidth: 0 }}>
{artist && (
<p style={{ margin: '0 0 4px 0', fontSize: 16, color: '#e2e8f0',
fontWeight: 700, lineHeight: 1.3, wordBreak: 'break-word' }}>
<p style={{ margin: isNarrow ? 0 : '0 0 4px 0', fontSize: isNarrow ? 13 : 16,
color: '#e2e8f0', fontWeight: 700, lineHeight: 1.3,
wordBreak: 'break-word', whiteSpace: 'nowrap', overflow: 'hidden', textOverflow: 'ellipsis' }}>
{artist}
</p>
)}
{(songTitle ?? title) && (
<p style={{ margin: 0, fontSize: 13, color: '#94a3b8',
lineHeight: 1.3, wordBreak: 'break-word' }}>
<p style={{ margin: 0, fontSize: isNarrow ? 11 : 13, color: '#94a3b8',
lineHeight: 1.3, wordBreak: 'break-word', whiteSpace: 'nowrap',
overflow: 'hidden', textOverflow: 'ellipsis' }}>
{songTitle ?? title}
</p>
)}
+490 -151
View File
@@ -4,7 +4,7 @@ import {
Music2, Mic2, FileCode, FileText, Download, Loader2, Trash2,
CheckCircle, XCircle, Clock, AlertCircle, ChevronDown, ChevronUp,
ListMusic, Edit2, Check, X as XIcon, Wand2, RotateCcw, Copy, Users,
Play, Pause, Square,
Play, Pause, Square, Eye, EyeOff,
} from 'lucide-react'
import { getProjectDetail, transcribeProject, abcProject, mixProject, lrcProject, lyricTranscribeProject, downloadUrl, reprocessStem, renameStem, getPerInstrumentModels } from '../api/client'
import type { ProjectDetail as PD, Job, JobFile } from '../types'
@@ -13,6 +13,12 @@ import LyricsLivePreview from '../components/LyricsLivePreview'
// ─── helpers ────────────────────────────────────────────────────────────────
function formatTime(s: number): string {
const m = Math.floor(s / 60)
const sec = Math.floor(s % 60)
return `${m}:${sec.toString().padStart(2, '0')}`
}
const TRANSCRIBE_MODES = [
{ id: 'polyphonic', label: 'Polyphonic' },
{ id: 'melody', label: 'Melody / Vocal' },
@@ -98,23 +104,26 @@ const SALAMANDER_URLS: Record<string, string> = {
}
function PianoRoll({
midi,
midiList,
hiddenStems,
playheadRef,
}: {
midi: any
midiList: Array<{midi: any; stemIdx: number}>
hiddenStems: Record<number, boolean>
playheadRef: React.RefObject<SVGLineElement | null>
}) {
const allNotes = midi.tracks
.flatMap((t: any, ti: number) => t.notes.map((n: any) => ({ ...n, ti })))
.filter((n: any) => n.duration > 0)
const allNotes = midiList
.filter(({ stemIdx }) => !hiddenStems[stemIdx])
.flatMap(({ midi, stemIdx }) => midi.tracks.flatMap((t: any) => t.notes.map((n: any) => ({ note: n, stemIdx }))))
.filter(({ note }) => note.duration > 0)
if (!allNotes.length) {
return <p className="px-3 py-4 text-xs text-gray-500 italic text-center">No notes found.</p>
}
const totalTime = midi.duration || 1
const minP = Math.max(0, Math.min(...allNotes.map((n: any) => n.midi)) - 2)
const maxP = Math.min(127, Math.max(...allNotes.map((n: any) => n.midi)) + 2)
const totalTime = Math.max(...midiList.map(({ midi }) => midi.duration || 1))
const minP = Math.max(0, Math.min(...allNotes.map(({ note }) => note.midi)) - 2)
const maxP = Math.min(127, Math.max(...allNotes.map(({ note }) => note.midi)) + 2)
const range = maxP - minP + 1
const svgH = range * NOTE_PX
const isBlack = (p: number) => [1, 3, 6, 8, 10].includes(p % 12)
@@ -142,19 +151,19 @@ function PianoRoll({
<line key={i} x1={0} y1={i * NOTE_PX} x2={SVG_W} y2={i * NOTE_PX} stroke="rgba(255,255,255,0.2)" strokeWidth={0.5} />
) : null
})}
{/* Notes */}
{allNotes.map((n: any, i: number) => (
{/* Notes — color-coded by stem */}
{allNotes.map(({ note, stemIdx }, i) => (
<rect
key={i}
x={(n.time / totalTime) * SVG_W}
y={(maxP - n.midi) * NOTE_PX + 0.4}
width={Math.max((n.duration / totalTime) * SVG_W, 1.5)}
x={(note.time / totalTime) * SVG_W}
y={(maxP - note.midi) * NOTE_PX + 0.4}
width={Math.max((note.duration / totalTime) * SVG_W, 1.5)}
height={NOTE_PX - 0.8}
fill={TRACK_COLORS[n.ti % TRACK_COLORS.length]}
fill={TRACK_COLORS[stemIdx % TRACK_COLORS.length]}
rx={0.5}
opacity={0.45 + n.velocity * 0.55}
opacity={0.45 + note.velocity * 0.55}
>
<title>{n.name} {n.time.toFixed(2)}s</title>
<title>{note.name} {note.time.toFixed(2)}s</title>
</rect>
))}
{/* Playhead — position updated directly via ref by the RAF loop */}
@@ -170,23 +179,113 @@ function PianoRoll({
)
}
function MidiPreview({ file, jobId }: { file: JobFile; jobId: string }) {
// ─── GM instrument data ───────────────────────────────────────────────────────
const GM_FAMILIES: Record<string, number[]> = {
Piano: [0,1,2,3,4,5,6,7],
'Chrom. Perc': [8,9,10,11,12,13,14,15],
Organ: [16,17,18,19,20,21,22,23],
Guitar: [24,25,26,27,28,29,30,31],
Bass: [32,33,34,35,36,37,38,39],
Strings: [40,41,42,43,44,45,46,47],
Ensemble: [48,49,50,51,52,53,54,55],
Brass: [56,57,58,59,60,61,62,63],
Reed: [64,65,66,67,68,69,70,71],
Pipe: [72,73,74,75,76,77,78,79],
'Synth Lead': [80,81,82,83,84,85,86,87],
'Synth Pad': [88,89,90,91,92,93,94,95],
'Synth FX': [96,97,98,99,100,101,102,103],
Ethnic: [104,105,106,107,108,109,110,111],
Percussive: [112,113,114,115,116,117,118,119],
SFX: [120,121,122,123,124,125,126,127],
}
const GM_NAMES: string[] = [
'Acoustic Grand Piano','Bright Acoustic Piano','Electric Grand Piano','Honky-tonk Piano','Electric Piano 1','Electric Piano 2','Harpsichord','Clavinet',
'Celesta','Glockenspiel','Music Box','Vibraphone','Marimba','Xylophone','Tubular Bells','Dulcimer',
'Drawbar Organ','Percussive Organ','Rock Organ','Church Organ','Reed Organ','Accordion','Harmonica','Tango Accordion',
'Nylon Guitar','Steel Guitar','Jazz Guitar','Clean Guitar','Muted Guitar','Overdriven Guitar','Distortion Guitar','Guitar Harmonics',
'Acoustic Bass','Electric Bass (finger)','Electric Bass (pick)','Fretless Bass','Slap Bass 1','Slap Bass 2','Synth Bass 1','Synth Bass 2',
'Violin','Viola','Cello','Contrabass','Tremolo Strings','Pizzicato Strings','Orchestral Harp','Timpani',
'String Ensemble 1','String Ensemble 2','SynthStrings 1','SynthStrings 2','Choir Aahs','Voice Oohs','Synth Voice','Orchestra Hit',
'Trumpet','Trombone','Tuba','Muted Trumpet','French Horn','Brass Section','SynthBrass 1','SynthBrass 2',
'Soprano Sax','Alto Sax','Tenor Sax','Baritone Sax','Oboe','English Horn','Bassoon','Clarinet',
'Piccolo','Flute','Recorder','Pan Flute','Blown Bottle','Shakuhachi','Whistle','Ocarina',
'Lead 1 (Square)','Lead 2 (Sawtooth)','Lead 3 (Calliope)','Lead 4 (Chiff)','Lead 5 (Charang)','Lead 6 (Voice)','Lead 7 (Fifths)','Lead 8 (Bass+Lead)',
'Pad 1 (New Age)','Pad 2 (Warm)','Pad 3 (Polysynth)','Pad 4 (Choir)','Pad 5 (Bowed)','Pad 6 (Metallic)','Pad 7 (Halo)','Pad 8 (Sweep)',
'FX 1 (Rain)','FX 2 (Soundtrack)','FX 3 (Crystal)','FX 4 (Atmosphere)','FX 5 (Brightness)','FX 6 (Goblins)','FX 7 (Echoes)','FX 8 (Sci-fi)',
'Sitar','Banjo','Shamisen','Koto','Kalimba','Bagpipe','Fiddle','Shanai',
'Tinkle Bell','Agogo','Steel Drums','Woodblock','Taiko Drum','Melodic Tom','Synth Drum','Reverse Cymbal',
'Guitar Fret Noise','Breath Noise','Seashore','Bird Tweet','Telephone Ring','Helicopter','Applause','Gunshot',
]
function createSynthForProgram(Tone: any, program: number): any {
if (program <= 7) {
return new Tone.Sampler({ urls: SALAMANDER_URLS, baseUrl: SALAMANDER_BASE, release: 1 })
}
const opts = (
program <= 15 ? { oscillator: { type: 'sine' }, envelope: { attack: 0.001, decay: 0.5, sustain: 0, release: 0.5 } } :
program <= 23 ? { oscillator: { type: 'sine' }, envelope: { attack: 0.01, decay: 0, sustain: 1, release: 0.05 } } :
program <= 31 ? { oscillator: { type: 'triangle' }, envelope: { attack: 0.005, decay: 0.4, sustain: 0.1, release: 1.2 } } :
program <= 39 ? { oscillator: { type: 'sawtooth' }, envelope: { attack: 0.01, decay: 0.3, sustain: 0.5, release: 0.5 } } :
program <= 47 ? { oscillator: { type: 'sawtooth' }, envelope: { attack: 0.4, decay: 0.1, sustain: 0.9, release: 1.5 } } :
program <= 55 ? { oscillator: { type: 'sine' }, envelope: { attack: 0.5, decay: 0.1, sustain: 0.8, release: 2 } } :
program <= 63 ? { oscillator: { type: 'sawtooth' }, envelope: { attack: 0.08, decay: 0.05, sustain: 0.9, release: 0.2 } } :
program <= 71 ? { oscillator: { type: 'sawtooth' }, envelope: { attack: 0.05, decay: 0.05, sustain: 0.8, release: 0.3 } } :
program <= 79 ? { oscillator: { type: 'sine' }, envelope: { attack: 0.1, decay: 0.05, sustain: 0.8, release: 0.5 } } :
program <= 87 ? { oscillator: { type: 'square' }, envelope: { attack: 0.01, decay: 0.05, sustain: 0.7, release: 0.3 } } :
program <= 95 ? { oscillator: { type: 'sine' }, envelope: { attack: 0.5, decay: 0.1, sustain: 0.9, release: 2 } } :
{}
) as any
return new Tone.PolySynth(Tone.Synth, opts)
}
interface MidiGroupStem {
label: string
file: JobFile
jobId: string
}
function MidiGroupPlayer({ stems }: { stems: MidiGroupStem[] }) {
const [open, setOpen] = useState(false)
const [midi, setMidi] = useState<any>(null)
const [midis, setMidis] = useState<Record<number, any>>({})
const [loading, setLoading] = useState(false)
const [err, setErr] = useState<string | null>(null)
const [playing, setPlaying] = useState(false)
// 'idle' → first play loads sampler; 'loading' → fetching samples; 'ready' → can play
const [sfState, setSfState] = useState<'idle' | 'loading' | 'ready' | 'error'>('idle')
const [currentTime, setCurrentTime] = useState(0)
const [duration, setDuration] = useState(0)
const [stemInstruments, setStemInstruments] = useState<Record<number, number>>({})
const [mutedStems, setMutedStems] = useState<Record<number, boolean>>({})
const [soloedStems, setSoloedStems] = useState<Record<number, boolean>>({})
const [hiddenStems, setHiddenStems] = useState<Record<number, boolean>>({})
const ToneRef = useRef<any>(null)
const samplerRef = useRef<any>(null)
const partRef = useRef<any>(null)
const rafRef = useRef<number | null>(null)
const playheadRef = useRef<SVGLineElement | null>(null)
const ToneRef = useRef<any>(null)
const synthRefs = useRef<Record<number, any>>({})
const partRefs = useRef<Record<number, any>>({})
const volRefs = useRef<Record<number, any>>({})
const rafRef = useRef<number | null>(null)
const playheadRef = useRef<SVGLineElement | null>(null)
const durationRef = useRef(0)
// Clean up Tone.js on unmount
useEffect(() => () => { cleanupAudio() }, [])
useEffect(() => () => { cleanupAll() }, [])
// Set default instruments + duration once midis are loaded
useEffect(() => {
if (!Object.keys(midis).length) return
const instruments: Record<number, number> = {}
let maxDur = 0
stems.forEach((_, si) => {
const midi = midis[si]
if (!midi) return
const first = midi.tracks.find((t: any) => t.notes.length > 0)
instruments[si] = first?.instrument?.number ?? 0
maxDur = Math.max(maxDur, midi.duration || 0)
})
setStemInstruments(instruments)
durationRef.current = maxDur
setDuration(maxDur)
}, [midis])
function cancelRaf() {
if (rafRef.current !== null) { cancelAnimationFrame(rafRef.current); rafRef.current = null }
@@ -199,15 +298,20 @@ function MidiPreview({ file, jobId }: { file: JobFile; jobId: string }) {
playheadRef.current.setAttribute('x2', '0')
}
function cleanupAudio() {
function cleanupAll() {
cancelRaf()
try {
ToneRef.current?.Transport.stop()
ToneRef.current?.Transport.cancel()
partRef.current?.dispose(); partRef.current = null
samplerRef.current?.dispose(); samplerRef.current = null
} catch { /* ignore */ }
Object.values(partRefs.current).forEach(p => { try { p?.dispose() } catch {} })
Object.values(synthRefs.current).forEach(s => { try { s?.dispose() } catch {} })
Object.values(volRefs.current).forEach(v => { try { v?.dispose() } catch {} })
partRefs.current = {}
synthRefs.current = {}
volRefs.current = {}
} catch {}
resetPlayhead()
setCurrentTime(0)
setPlaying(false)
}
@@ -215,6 +319,7 @@ function MidiPreview({ file, jobId }: { file: JobFile; jobId: string }) {
cancelRaf()
function loop() {
const t = ToneRef.current?.Transport.seconds ?? 0
setCurrentTime(t)
if (playheadRef.current) {
const pct = Math.min(1, t / (totalDuration || 1))
playheadRef.current.style.display = 'block'
@@ -227,72 +332,102 @@ function MidiPreview({ file, jobId }: { file: JobFile; jobId: string }) {
rafRef.current = requestAnimationFrame(loop)
}
function buildPart(Tone: any, midiData: any) {
const allNotes = midiData.tracks.flatMap((t: any) =>
t.notes.map((n: any) => ({ time: n.time, name: n.name, duration: n.duration, velocity: n.velocity }))
)
// One Part per stem — merges all tracks in that stem's MIDI into one synth
function buildStemPart(Tone: any, si: number, synth: any) {
const midi = midis[si]
if (!midi) return
const allNotes: Array<[number, any]> = []
midi.tracks.forEach((track: any) => {
track.notes.forEach((n: any) => allNotes.push([n.time, n]))
})
if (!allNotes.length) return
const part = new Tone.Part((time: number, note: any) => {
samplerRef.current?.triggerAttackRelease(note.name, note.duration, time, note.velocity)
}, allNotes.map((n: any) => [n.time, n]))
synth.triggerAttackRelease(note.name, note.duration, time, note.velocity)
}, allNotes)
part.start(0)
return part
partRefs.current[si] = part
}
async function toggle() {
if (!open && midi === null && !err) {
function applyMutes(overrides: {
mutedStems?: Record<number, boolean>
soloedStems?: Record<number, boolean>
} = {}) {
const mutes = overrides.mutedStems ?? mutedStems
const solos = overrides.soloedStems ?? soloedStems
const anySolo = Object.values(solos).some(Boolean)
stems.forEach((_, si) => {
const vol = volRefs.current[si]
if (!vol) return
vol.mute = (mutes[si] ?? false) || (anySolo && !(solos[si] ?? false))
})
}
async function handleToggle() {
if (open) {
cleanupAll()
setOpen(false)
return
}
setOpen(true)
if (!Object.keys(midis).length && !loading) {
setLoading(true)
try {
const { Midi } = await import('@tonejs/midi')
const res = await fetch(downloadUrl(jobId, file.name))
const buf = await res.arrayBuffer()
setMidi(new Midi(buf))
const results = await Promise.all(
stems.map(async (stem, si) => {
const res = await fetch(downloadUrl(stem.jobId, stem.file.name))
const buf = await res.arrayBuffer()
return { si, midi: new Midi(buf) }
})
)
const newMidis: Record<number, any> = {}
results.forEach(({ si, midi }) => { newMidis[si] = midi })
setMidis(newMidis)
} catch {
setErr('Failed to parse MIDI file.')
setErr('Failed to load MIDI.')
} finally {
setLoading(false)
}
}
if (open) cleanupAudio()
setOpen(v => !v)
}
async function handlePlay() {
if (!midi) return
// First play: lazy-load Tone.js and fetch Salamander samples
if (!Object.keys(midis).length) return
if (sfState === 'idle') {
setSfState('loading')
try {
const Tone = await import('tone')
ToneRef.current = Tone
await Tone.start() // unlock AudioContext — must happen inside user gesture
const sampler = new Tone.Sampler({
urls: SALAMANDER_URLS,
baseUrl: SALAMANDER_BASE,
release: 1,
onload: () => {
setSfState('ready')
// Auto-start once all samples are buffered
Tone.Transport.start()
setPlaying(true)
startRaf(midi.duration || 1)
},
}).toDestination()
samplerRef.current = sampler
partRef.current = buildPart(Tone, midi)
await Tone.start()
const pianoPromises: Promise<any>[] = []
stems.forEach((_, si) => {
const midi = midis[si]
if (!midi) return
const program = stemInstruments[si] ?? 0
const synth = createSynthForProgram(Tone, program)
const vol = new Tone.Volume(0).toDestination()
synth.connect(vol)
synthRefs.current[si] = synth
volRefs.current[si] = vol
if (program <= 7) pianoPromises.push(synth.loaded)
buildStemPart(Tone, si, synth)
})
applyMutes()
if (pianoPromises.length) await Promise.all(pianoPromises)
setSfState('ready')
Tone.Transport.start()
setPlaying(true)
startRaf(durationRef.current)
} catch {
setSfState('error')
}
return
}
// Sampler already ready — resume or restart
const Tone = ToneRef.current
await Tone.start()
Tone.Transport.start()
setPlaying(true)
startRaf(midi.duration || 1)
startRaf(durationRef.current)
}
function handlePause() {
@@ -304,93 +439,230 @@ function MidiPreview({ file, jobId }: { file: JobFile; jobId: string }) {
function handleStop() {
cancelRaf()
const Tone = ToneRef.current
if (Tone && midi) {
if (Tone) {
Tone.Transport.stop()
Tone.Transport.cancel()
Tone.Transport.position = 0
// Rebuild the Part so the next play starts from the beginning
partRef.current?.dispose()
partRef.current = buildPart(Tone, midi)
Tone.Transport.seconds = 0
Object.values(partRefs.current).forEach(p => { try { p?.dispose() } catch {} })
partRefs.current = {}
stems.forEach((_, si) => {
const synth = synthRefs.current[si]
if (synth) buildStemPart(Tone, si, synth)
})
}
resetPlayhead()
setCurrentTime(0)
setPlaying(false)
}
const noteCount = midi?.tracks.reduce((s: number, t: any) => s + t.notes.length, 0) ?? 0
const bpm = midi?.header?.tempos?.[0]?.bpm
const dur = midi?.duration ?? 0
const durStr = `${Math.floor(dur / 60)}:${String(Math.floor(dur % 60)).padStart(2, '0')}`
function handleSeek(t: number) {
const Tone = ToneRef.current
if (!Tone || sfState !== 'ready') return
const wasPlaying = playing
Tone.Transport.pause()
cancelRaf()
Tone.Transport.cancel()
Tone.Transport.seconds = t
setCurrentTime(t)
Object.values(partRefs.current).forEach(p => { try { p?.dispose() } catch {} })
partRefs.current = {}
stems.forEach((_, si) => {
const synth = synthRefs.current[si]
if (synth) buildStemPart(Tone, si, synth)
})
if (playheadRef.current) {
const pct = Math.min(1, t / (durationRef.current || 1))
playheadRef.current.style.display = 'block'
playheadRef.current.setAttribute('x1', String(pct * SVG_W))
playheadRef.current.setAttribute('x2', String(pct * SVG_W))
}
if (wasPlaying) {
Tone.Transport.start()
setPlaying(true)
startRaf(durationRef.current)
}
}
function toggleStemMute(si: number) {
setMutedStems(prev => {
const next = { ...prev, [si]: !(prev[si] ?? false) }
applyMutes({ mutedStems: next })
return next
})
}
function toggleStemSolo(si: number) {
setSoloedStems(prev => {
const next = { ...prev, [si]: !(prev[si] ?? false) }
applyMutes({ soloedStems: next })
return next
})
}
async function changeStemInstrument(si: number, program: number) {
const Tone = ToneRef.current
const wasPlaying = playing
if (wasPlaying) { Tone?.Transport.pause(); cancelRaf(); setPlaying(false) }
setStemInstruments(prev => ({ ...prev, [si]: program }))
if (!Tone || sfState !== 'ready') return
try { partRefs.current[si]?.dispose() } catch {}; delete partRefs.current[si]
try { synthRefs.current[si]?.dispose() } catch {}; delete synthRefs.current[si]
const vol = volRefs.current[si]
const newSynth = createSynthForProgram(Tone, program)
if (vol) newSynth.connect(vol)
synthRefs.current[si] = newSynth
if (program <= 7) {
setSfState('loading')
try { await newSynth.loaded } catch {}
setSfState('ready')
}
// Rebuild all parts to stay in sync
Tone.Transport.cancel()
stems.forEach((_, idx) => {
try { partRefs.current[idx]?.dispose() } catch {}
const synth = synthRefs.current[idx]
if (synth) buildStemPart(Tone, idx, synth)
})
}
const midiList = stems.map((_, si) => ({ midi: midis[si], stemIdx: si })).filter(x => x.midi)
const totalNotes = midiList.reduce((s, { midi }) =>
s + midi.tracks.reduce((s2: number, t: any) => s2 + t.notes.length, 0), 0)
const bpm = midiList[0]?.midi?.header?.tempos?.[0]?.bpm
const durStr = formatTime(duration)
const curStr = formatTime(currentTime)
return (
<div className="rounded-lg border border-purple-200/60 dark:border-purple-900/40 overflow-hidden">
{/* Toggle header */}
<button
onClick={toggle}
onClick={handleToggle}
className="w-full flex items-center justify-between px-3 py-2 bg-purple-50/60 dark:bg-purple-950/20 text-xs font-medium text-purple-700 dark:text-purple-300 hover:bg-purple-100/60 dark:hover:bg-purple-950/40 transition-colors"
>
<span className="flex items-center gap-1.5">
{loading ? <Loader2 className="w-3 h-3 animate-spin" /> : <Music2 className="w-3 h-3" />}
Piano roll
<Music2 className="w-3 h-3" />
Piano Roll · {stems.length} stem{stems.length !== 1 ? 's' : ''}
</span>
<span className="flex items-center gap-3">
{midi && (
<>
{bpm && <span className="text-purple-400/70">{Math.round(bpm)} BPM</span>}
{noteCount > 0 && <span className="text-purple-400/70">{noteCount} notes</span>}
<span className="text-purple-400/70">{durStr}</span>
</>
)}
<a
href={downloadUrl(jobId, file.name)}
download={file.name}
onClick={e => e.stopPropagation()}
title={`Download ${file.name}`}
className="text-purple-400 hover:text-purple-600 dark:hover:text-purple-300 transition-colors"
>
<Download className="w-3 h-3" />
</a>
{bpm && <span className="text-purple-400/70">{Math.round(bpm)} BPM</span>}
{duration > 0 && <span className="text-purple-400/70">{durStr}</span>}
{open ? <ChevronUp className="w-3 h-3" /> : <ChevronDown className="w-3 h-3" />}
</span>
</button>
{open && (
err
? <p className="px-3 py-4 text-xs text-red-400 italic text-center bg-gray-950">{err}</p>
: !midi
? <div className="flex items-center justify-center py-8 bg-gray-950"><Loader2 className="w-5 h-5 animate-spin text-purple-400" /></div>
: <>
<PianoRoll midi={midi} playheadRef={playheadRef} />
loading
? <div className="flex items-center justify-center py-8 bg-gray-950"><Loader2 className="w-5 h-5 animate-spin text-purple-400" /></div>
: err
? <p className="px-3 py-4 text-xs text-red-400 italic text-center bg-gray-950">{err}</p>
: midiList.length > 0
? <>
<PianoRoll midiList={midiList} hiddenStems={hiddenStems} playheadRef={playheadRef} />
{/* Playback transport bar */}
<div className="flex items-center gap-2 px-3 py-2 bg-gray-900 border-t border-gray-800">
<button
onClick={playing ? handlePause : handlePlay}
disabled={sfState === 'error'}
className="flex items-center gap-1.5 px-3 py-1.5 rounded-lg bg-purple-600 hover:bg-purple-500 disabled:bg-gray-700 disabled:text-gray-500 text-white text-xs font-medium transition-colors min-w-[72px] justify-center"
>
{sfState === 'loading'
? <><Loader2 className="w-3 h-3 animate-spin" /> Loading</>
: playing
? <><Pause className="w-3 h-3" /> Pause</>
: <><Play className="w-3 h-3" /> Play</>}
</button>
{sfState === 'ready' && (
<button
onClick={handleStop}
title="Stop and rewind"
className="p-1.5 rounded-lg text-gray-500 hover:text-white hover:bg-gray-700 transition-colors"
>
<Square className="w-3 h-3" />
</button>
)}
<span className="ml-auto text-[10px] text-gray-600 select-none">
{sfState === 'idle' && 'Grand Piano · click Play to load'}
{sfState === 'loading' && 'Fetching piano samples…'}
{sfState === 'ready' && 'Grand Piano'}
{sfState === 'error' && 'Soundfont load failed'}
</span>
</div>
</>
<div className="px-3 py-2 bg-gray-900 border-t border-gray-800 space-y-2">
{/* Seek bar */}
<div className="flex items-center gap-2">
<span className="text-[10px] tabular-nums text-gray-500 w-8 shrink-0">{curStr}</span>
<input
type="range" min={0} max={duration || 1} step={0.1} value={currentTime}
onChange={e => handleSeek(parseFloat(e.target.value))}
disabled={sfState !== 'ready'}
className="flex-1 h-1.5 bg-gray-700 rounded-lg appearance-none cursor-pointer accent-purple-500 disabled:opacity-40"
/>
<span className="text-[10px] tabular-nums text-gray-500 w-8 shrink-0 text-right">{durStr}</span>
</div>
{/* Play / Pause / Stop */}
<div className="flex items-center gap-2">
<button
onClick={playing ? handlePause : handlePlay}
disabled={sfState === 'error'}
className="flex items-center gap-1.5 px-3 py-1.5 rounded-lg bg-purple-600 hover:bg-purple-500 disabled:bg-gray-700 disabled:text-gray-500 text-white text-xs font-medium transition-colors min-w-[72px] justify-center"
>
{sfState === 'loading'
? <><Loader2 className="w-3 h-3 animate-spin" /> Loading</>
: playing
? <><Pause className="w-3 h-3" /> Pause</>
: <><Play className="w-3 h-3" /> Play</>}
</button>
{sfState === 'ready' && (
<button onClick={handleStop} title="Stop and rewind"
className="p-1.5 rounded-lg text-gray-500 hover:text-white hover:bg-gray-700 transition-colors">
<Square className="w-3 h-3" />
</button>
)}
<span className="ml-auto text-[10px] text-gray-600 select-none">
{sfState === 'idle' && `${totalNotes} notes · click Play`}
{sfState === 'loading' && 'Loading instruments…'}
{sfState === 'ready' && `${totalNotes} notes`}
{sfState === 'error' && 'Failed to load'}
</span>
</div>
{/* Per-stem rows: M / S / label / instrument / download */}
<div className="space-y-1.5 pt-1.5 border-t border-gray-800">
{stems.map((stem, si) => {
const program = stemInstruments[si] ?? 0
const isMuted = mutedStems[si] ?? false
const isSoloed = soloedStems[si] ?? false
const isHidden = hiddenStems[si] ?? false
return (
<div key={si} className="flex items-center gap-1.5">
<button
onClick={() => setHiddenStems(prev => ({ ...prev, [si]: !prev[si] }))}
title={isHidden ? 'Show in piano roll' : 'Hide from piano roll'}
className="flex-shrink-0 text-gray-500 hover:text-gray-300 transition-colors"
>
{isHidden ? <EyeOff className="w-3 h-3" /> : <Eye className="w-3 h-3" />}
</button>
<button
onClick={() => toggleStemMute(si)}
title={isMuted ? 'Unmute' : 'Mute'}
className={`w-5 h-5 text-[10px] font-bold rounded transition-colors flex-shrink-0 ${
isMuted ? 'bg-red-500 text-white' : 'bg-gray-700 text-gray-400 hover:bg-gray-600'
}`}
>M</button>
<button
onClick={() => toggleStemSolo(si)}
title={isSoloed ? 'Unsolo' : 'Solo'}
className={`w-5 h-5 text-[10px] font-bold rounded transition-colors flex-shrink-0 ${
isSoloed ? 'bg-yellow-400 text-black' : 'bg-gray-700 text-gray-400 hover:bg-gray-600'
}`}
>S</button>
<span
className="text-[10px] font-medium w-14 shrink-0 truncate capitalize"
style={{ color: TRACK_COLORS[si % TRACK_COLORS.length] }}
title={stem.label}
>{stem.label}</span>
<select
value={program}
onChange={e => changeStemInstrument(si, parseInt(e.target.value))}
className="flex-1 min-w-0 bg-gray-800 border border-gray-700 rounded px-1.5 py-0.5 text-[10px] text-gray-300 focus:outline-none focus:border-purple-500 cursor-pointer"
>
{Object.entries(GM_FAMILIES).map(([family, programs]) => (
<optgroup key={family} label={family}>
{programs.map(p => (
<option key={p} value={p}>{p}: {GM_NAMES[p]}</option>
))}
</optgroup>
))}
</select>
<a
href={downloadUrl(stem.jobId, stem.file.name)}
download={stem.file.name}
onClick={e => e.stopPropagation()}
title={`Download ${stem.file.name}`}
className="text-gray-600 hover:text-purple-400 transition-colors flex-shrink-0"
>
<Download className="w-3 h-3" />
</a>
</div>
)
})}
</div>
</div>
</>
: null
)}
</div>
)
@@ -857,13 +1129,6 @@ function StemMidiRow({
{transJob && isActive(transJob) && (
<p className="text-xs text-slate-400 dark:text-gray-500 truncate">{transJob.progress}</p>
)}
{transJob?.status === 'done' && (
<div className="space-y-1.5">
{transJob.files.filter((f) => f.name.endsWith('.mid')).map((f) => (
<MidiPreview key={f.name} file={f} jobId={transJob.id} />
))}
</div>
)}
{transJob?.status === 'error' && (
<p className="text-xs text-red-400 break-all">{transJob.error}</p>
)}
@@ -951,6 +1216,21 @@ function MidiSection({
})}
</div>
)}
{(() => {
const doneStems: MidiGroupStem[] = stems
.map(stem => {
const stemName = stem.stem ?? stem.name
const job = transJobs[stemName]
if (job?.status !== 'done') return null
const midFile = job.files.find(f => f.name.endsWith('.mid'))
if (!midFile) return null
const label = stemName.charAt(0).toUpperCase() + stemName.slice(1)
return { label, file: midFile, jobId: job.id }
})
.filter((x): x is MidiGroupStem => x !== null)
if (!doneStems.length) return null
return <MidiGroupPlayer key={doneStems.map(s => s.jobId).join(',')} stems={doneStems} />
})()}
</>)}
</section>
)
@@ -1141,6 +1421,8 @@ function AbcFileViewer({
}: { file: JobFile; jobId: string; expanded: boolean; onToggle: () => void }) {
const [text, setText] = useState<string | null>(null)
const [loading, setLoading] = useState(false)
const [view, setView] = useState<'sheet' | 'text'>('sheet')
const svgRef = useRef<HTMLDivElement | null>(null)
async function load() {
if (text !== null) { onToggle(); return }
@@ -1153,21 +1435,67 @@ function AbcFileViewer({
onToggle()
}
// Render / re-render sheet music whenever visible
useEffect(() => {
if (!expanded || view !== 'sheet' || !text || !svgRef.current) return
import('abcjs').then(({ renderAbc }) => {
renderAbc(svgRef.current!, text, {
responsive: 'resize',
add_classes: true,
paddingbottom: 20,
paddingtop: 16,
paddingright: 16,
paddingleft: 16,
wrap: { minSpacing: 1.8, maxSpacing: 2.7, preferredMeasuresPerLine: 4 },
})
}).catch(() => {})
}, [expanded, view, text])
return (
<div className="rounded-lg border border-slate-200 dark:border-gray-700 overflow-hidden">
{/* Header */}
<div className="flex items-center justify-between px-3 py-2 bg-white dark:bg-gray-900">
<button onClick={load} className="flex items-center gap-1.5 text-xs font-medium text-slate-700 dark:text-gray-300 hover:text-emerald-600 dark:hover:text-emerald-400 transition-colors">
{loading ? <Loader2 className="w-3 h-3 animate-spin" /> : expanded ? <ChevronUp className="w-3 h-3" /> : <ChevronDown className="w-3 h-3" />}
<button
onClick={load}
className="flex items-center gap-1.5 text-xs font-medium text-slate-700 dark:text-gray-300 hover:text-emerald-600 dark:hover:text-emerald-400 transition-colors"
>
{loading
? <Loader2 className="w-3 h-3 animate-spin" />
: expanded ? <ChevronUp className="w-3 h-3" /> : <ChevronDown className="w-3 h-3" />}
{file.name}
</button>
<a href={downloadUrl(jobId, file.name)} download={file.name} className="text-xs text-slate-400 hover:text-brand-500 transition-colors">
<Download className="w-3 h-3" />
</a>
<div className="flex items-center gap-2">
{expanded && text !== null && (
<div className="flex rounded overflow-hidden border border-slate-200 dark:border-gray-700 text-[10px] font-medium">
<button
onClick={() => setView('sheet')}
className={`px-2 py-0.5 transition-colors ${view === 'sheet' ? 'bg-emerald-500 text-white' : 'bg-white dark:bg-gray-900 text-slate-500 dark:text-gray-400 hover:bg-slate-50 dark:hover:bg-gray-800'}`}
>Sheet</button>
<button
onClick={() => setView('text')}
className={`px-2 py-0.5 transition-colors ${view === 'text' ? 'bg-emerald-500 text-white' : 'bg-white dark:bg-gray-900 text-slate-500 dark:text-gray-400 hover:bg-slate-50 dark:hover:bg-gray-800'}`}
>Text</button>
</div>
)}
<a
href={downloadUrl(jobId, file.name)}
download={file.name}
className="text-xs text-slate-400 hover:text-brand-500 transition-colors"
>
<Download className="w-3 h-3" />
</a>
</div>
</div>
{expanded && text !== null && (
<pre className="p-3 text-xs text-slate-600 dark:text-gray-400 bg-slate-50 dark:bg-gray-950 overflow-x-auto max-h-64 overflow-y-auto whitespace-pre-wrap">
{text}
</pre>
view === 'sheet'
? <div
ref={svgRef}
className="bg-white p-2 overflow-x-auto overflow-y-auto max-h-[480px] [&_svg]:max-w-full"
/>
: <pre className="p-3 text-xs text-slate-600 dark:text-gray-400 bg-slate-50 dark:bg-gray-950 overflow-x-auto max-h-64 overflow-y-auto whitespace-pre-wrap font-mono">
{text}
</pre>
)}
</div>
)
@@ -1348,6 +1676,7 @@ function StemLyricsRow({
onAction: () => void
}) {
const [mode, setMode] = useState<'align' | 'transcribe'>('transcribe')
const [twoPass, setTwoPass] = useState(false)
// activeJob is the job relevant to the current mode tab
const activeJob = mode === 'transcribe' ? whisperJob : lrcJob
// Track job IDs started in this session — error state is only shown for fresh jobs
@@ -1493,6 +1822,7 @@ function StemLyricsRow({
diarize,
lrcPowerPreset: powerPreset,
phonemeAlign,
twoPass,
})
sessionJobIds.current.add(res.job_id)
onAction()
@@ -1517,11 +1847,11 @@ function StemLyricsRow({
{label}
</span>
<div className="flex gap-1">
<button onClick={() => setMode('align')} className={tabBtn('align')}>
Align
</button>
<button onClick={() => setMode('transcribe')} className={tabBtn('transcribe')}>
AI Transcribe
Transcribe
</button>
<button onClick={() => setMode('align')} className={tabBtn('align')}>
Use Existing Lyrics
</button>
</div>
{activeJob && (activeJob.status !== 'error' || showError) && <JobBadge job={activeJob} />}
@@ -1627,6 +1957,15 @@ function StemLyricsRow({
/>
<span className="text-xs text-slate-600 dark:text-gray-400">Phonemes</span>
</label>
<label className="flex items-center gap-1.5 cursor-pointer select-none" title="Run a second VAD pass to transcribe gaps larger than 2 seconds">
<input
type="checkbox"
checked={twoPass}
onChange={(e) => setTwoPass(e.target.checked)}
className="w-3.5 h-3.5 rounded accent-violet-500"
/>
<span className="text-xs text-slate-600 dark:text-gray-400">Two-pass</span>
</label>
<button
onClick={handleTranscribe}
disabled={busy || !!isActive(whisperJob)}
+10 -1
View File
@@ -1145,7 +1145,7 @@ function AutoPipelineSettings() {
setAddOpen(false)
if (type === 'reprocess') saveSteps([...steps, { type: 'reprocess', stem: 'other', model: sepModel }])
if (type === 'midi') saveSteps([...steps, { type: 'midi', mode: 'polyphonic' }])
if (type === 'lyrics') saveSteps([...steps, { type: 'lyrics', whisperModel: 'large-v2', diarize: false, phonemeAlign: false }])
if (type === 'lyrics') saveSteps([...steps, { type: 'lyrics', whisperModel: 'large-v2', diarize: false, phonemeAlign: false, twoPass: false }])
if (type === 'lyricsAlign') saveSteps([...steps, { type: 'lyricsAlign', timingPrecision: 'word' }])
}
@@ -1423,6 +1423,15 @@ function AutoPipelineSettings() {
/>
<span className="text-xs text-slate-600 dark:text-gray-400">Phoneme alignment</span>
</label>
<label className="flex items-center gap-2 cursor-pointer select-none" title="Run a second VAD pass to fill transcription gaps larger than 2 seconds">
<input
type="checkbox"
checked={step.twoPass ?? false}
onChange={e => patchStep(idx, { twoPass: e.target.checked } as Partial<AutoStep>)}
className="w-3.5 h-3.5 rounded accent-violet-500"
/>
<span className="text-xs text-slate-600 dark:text-gray-400">Two-pass (fill gaps)</span>
</label>
</>
)}
+1
View File
@@ -23,6 +23,7 @@ export interface AutoStepLyrics {
whisperModel: string // e.g. 'large-v2'
diarize: boolean
phonemeAlign: boolean // use phonemizer + wav2vec2-xlsr for syllable-level timestamps
twoPass: boolean // run a second VAD pass to fill transcription gaps > 2 s
}
export interface AutoStepLyricsAlign {
+1 -1
View File
@@ -33,4 +33,4 @@ npx vite build --watch 2>&1 &
cd /app
echo "[backend] Starting uvicorn..."
exec python3.11 -m uvicorn backend.main:app \
--host 0.0.0.0 --port 8000 --log-level info --reload
--host 0.0.0.0 --port 8000 --log-level debug --reload