mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary Adds optional guitar chords to karaoke videos, first used for a Fiverr medley customer who wants chords on their video. - **Rendering (gen library):** `OutputGenerator.generate_outputs(chords=[ChordChange(start, end, label)])`. - Each chord change during a sung line is drawn above the word it lands on, in a row reserved above every line (the same slot mechanism as translated lyrics). - Chords light up when played, using the same `\k` highlight as lyrics: gold while upcoming (theme `chord_color`), sung-lyric colour once played. Quick changes on one word never overlap. - A top-right "now / next" chord box runs for the whole song, covering intros and instrumental breaks. - With no chords supplied, output is unchanged. A test asserts the ASS is byte-identical. - **Detection (`scripts/medley/detect_chords.py`):** - Ensembles both ChordMini checkpoints (BTC + 2E1D, MIT; about 80% majmin/MIREX, the best open weights we found) over the full mix, gen's clean instrumental, and an htdemucs_6s harmonic mix (guitar + piano + bass + other). - Weighted per-frame vote, then merges short blips, snaps changes to beats and spells chords for the key (Ab/Db/Eb7, F#m, borrowed bIII/bVI/bVII as flats). - **Medley script:** new `chords` step plus a `"chords": true` spec option. Chords are clipped per section and hand over mid-crossfade, then shifted for the countdown. - **Docs:** `docs/MEDLEYS.md` § Guitar chords (setup, method, accuracy caveats, theme keys) and a docs/README.md entry. There's no job or UI option yet. The medley pipeline uses this for now; a product version would need a chords toggle plus detection in a worker. ## Testing - New: `test_guitar_chords.py` (12: chord timeline helpers, word placement measured against the font, layout geometry, highlight timing, now/next box, theme colour, no-chords path byte-identical), `test_detect_chords.py` (7), and 2 medley chord-shifting tests. - `tests/unit`: 2087 passed, 65 skipped. - End to end on the 5 real medley songs: models/inputs agree on 91–99% of major/minor labels, and the output matches the expected I–IV–V progressions. Rendered 1080p and 4K medley videos with 146 chord changes; spot-checked frames (verse, instrumental screen, chorus). - Local CodeRabbit: 1 finding (doc section reference), fixed. @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01QV2TQwyC7pwVkGPCk98Bqg --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
216 lines
8.9 KiB
Python
216 lines
8.9 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Detect guitar chords for one song: an ensemble of ChordMini models over several stem mixes.
|
|
|
|
python scripts/medley/detect_chords.py --chordmini ~/ChordMini \\
|
|
--input mix=song.flac --input inst=instrumental_clean.flac --input harm=harmonic.flac \\
|
|
--out chords.json
|
|
|
|
Run it with ChordMini's own Python (it needs torch + librosa + mir_eval); see
|
|
docs/MEDLEYS.md § Guitar chords for setup and why this ensemble.
|
|
|
|
Pipeline:
|
|
1. Both ChordMini checkpoints (BTC and 2E1D, continual-learning versions; the best open
|
|
chord models we found, ~80% MIREX/majmin) on every input -> .lab files.
|
|
2. Each label simplified to a guitar chord (root + one of: major, m, 7, maj7, m7, sus2,
|
|
sus4, dim, aug, m7b5, dim7), then a weighted frame-by-frame vote across all runs.
|
|
3. Changes shorter than MIN_CHORD_SECONDS merged away; boundaries snapped to the
|
|
nearest beat when one is within BEAT_SNAP_SECONDS.
|
|
4. Roots spelled for the local key (Ab/Db/Eb7 in A-flat, not G#/C#/D#7).
|
|
|
|
Output: JSON list of {"start", "end", "label"} (seconds, no "no chord" entries).
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from collections import defaultdict
|
|
|
|
import numpy as np
|
|
|
|
FRAME = 0.05
|
|
MIN_CHORD_SECONDS = 0.6
|
|
BEAT_SNAP_SECONDS = 0.2
|
|
KEY_WINDOW_SECONDS = 20.0
|
|
MODELS = {"BTC": ("btc_model_best.pth", 1.1), "ChordNet": ("2e1d_model_best.pth", 1.0)}
|
|
INPUT_WEIGHTS = {"inst": 1.2} # vocals removed = cleaner harmony; everything else 1.0
|
|
|
|
PITCH = {"C": 0, "C#": 1, "Db": 1, "D": 2, "D#": 3, "Eb": 3, "E": 4, "Fb": 4, "E#": 5, "F": 5, "F#": 6,
|
|
"Gb": 6, "G": 7, "G#": 8, "Ab": 8, "A": 9, "A#": 10, "Bb": 10, "B": 11, "Cb": 11, "B#": 0}
|
|
SHARP_NAMES = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"]
|
|
FLAT_NAMES = ["C", "Db", "D", "Eb", "E", "F", "Gb", "G", "Ab", "A", "Bb", "B"]
|
|
FLAT_KEYS = {1, 3, 5, 6, 8, 10} # Db Eb F Gb Ab Bb major
|
|
QUALITY = {"maj": "", "": "", "min": "m", "7": "7", "maj7": "maj7", "min7": "m7", "sus4": "sus4",
|
|
"sus2": "sus2", "dim": "dim", "aug": "aug", "maj6": "", "min6": "m", "dim7": "dim7",
|
|
"hdim7": "m7b5", "minmaj7": "m", "9": "7", "maj9": "maj7", "min9": "m7", "11": "7", "13": "7"}
|
|
# Scale degree (semitones above the key) -> qualities that are diatonic there in a major key
|
|
DIATONIC = {0: {"", "maj7"}, 2: {"m", "m7"}, 4: {"m", "m7"}, 5: {"", "maj7"}, 7: {"", "7"}, 9: {"m", "m7"},
|
|
11: {"dim", "m7b5"}}
|
|
|
|
|
|
def simplify(label: str):
|
|
"""'G#:7' -> (8, '7'); 'N'/'X' -> None. Drops inversions ('C/E' -> C)."""
|
|
label = label.strip()
|
|
if label in ("N", "X", ""):
|
|
return None
|
|
root, _, qual = label.partition(":")
|
|
qual = qual.split("/")[0].split("(")[0]
|
|
root = root.split("/")[0]
|
|
if root not in PITCH:
|
|
return None
|
|
return PITCH[root], QUALITY.get(qual, "")
|
|
|
|
|
|
def spell(pc: int, qual: str, key: int) -> str:
|
|
names = FLAT_NAMES if key in FLAT_KEYS else SHARP_NAMES
|
|
name = names[pc]
|
|
# Borrowed bIII / bVI / bVII in sharp keys read better as flats (Eb, Ab, Bb in C)
|
|
if key not in FLAT_KEYS and pc in ((key + 3) % 12, (key + 8) % 12, (key + 10) % 12) and len(name) > 1:
|
|
name = FLAT_NAMES[pc]
|
|
return name + qual
|
|
|
|
|
|
def estimate_key(chords, t0: float, t1: float) -> int:
|
|
"""Major key whose diatonic chords cover the most time in [t0, t1]."""
|
|
scores = np.zeros(12)
|
|
for start, end, (pc, qual) in chords:
|
|
overlap = min(end, t1) - max(start, t0)
|
|
if overlap <= 0:
|
|
continue
|
|
for k in range(12):
|
|
if qual in DIATONIC.get((pc - k) % 12, ()):
|
|
scores[k] += overlap * (1.5 if (pc - k) % 12 in (0, 7) else 1.0)
|
|
return int(np.argmax(scores))
|
|
|
|
|
|
def read_lab(path):
|
|
rows = []
|
|
with open(path) as f:
|
|
for line in f:
|
|
parts = line.split()
|
|
if len(parts) >= 3:
|
|
rows.append((float(parts[0]), float(parts[1]), " ".join(parts[2:])))
|
|
return rows
|
|
|
|
|
|
def vote(labs, duration: float):
|
|
"""labs: list of (weight, rows). Returns per-frame winning (pc, qual) or None."""
|
|
n = int(np.ceil(duration / FRAME))
|
|
tallies = [defaultdict(float) for _ in range(n)]
|
|
for weight, rows in labs:
|
|
for start, end, label in rows:
|
|
chord = simplify(label)
|
|
for i in range(int(start / FRAME), min(n, int(np.ceil(end / FRAME)))):
|
|
tallies[i][chord] += weight
|
|
return [max(t.items(), key=lambda kv: kv[1])[0] if t else None for t in tallies]
|
|
|
|
|
|
def frames_to_segments(frames):
|
|
segs, start = [], 0
|
|
for i in range(1, len(frames) + 1):
|
|
if i == len(frames) or frames[i] != frames[start]:
|
|
segs.append([start * FRAME, i * FRAME, frames[start]])
|
|
start = i
|
|
return segs
|
|
|
|
|
|
def merge_short(segs, min_len: float = MIN_CHORD_SECONDS):
|
|
"""Fold segments shorter than min_len into the longer neighbour, until none remain."""
|
|
segs = [s[:] for s in segs]
|
|
while True:
|
|
short = [i for i, s in enumerate(segs) if s[1] - s[0] < min_len]
|
|
if not short or len(segs) == 1:
|
|
break
|
|
i = min(short, key=lambda j: segs[j][1] - segs[j][0])
|
|
prev_len = segs[i - 1][1] - segs[i - 1][0] if i > 0 else -1
|
|
next_len = segs[i + 1][1] - segs[i + 1][0] if i + 1 < len(segs) else -1
|
|
if prev_len >= next_len:
|
|
segs[i - 1][1] = segs[i][1]
|
|
else:
|
|
segs[i + 1][0] = segs[i][0]
|
|
del segs[i]
|
|
# Re-join neighbours that now carry the same chord
|
|
j = 0
|
|
while j + 1 < len(segs):
|
|
if segs[j][2] == segs[j + 1][2]:
|
|
segs[j][1] = segs[j + 1][1]
|
|
del segs[j + 1]
|
|
else:
|
|
j += 1
|
|
return segs
|
|
|
|
|
|
def snap_to_beats(segs, beats, tolerance: float = BEAT_SNAP_SECONDS):
|
|
if len(beats) == 0:
|
|
return segs
|
|
beats = np.asarray(beats)
|
|
for i in range(1, len(segs)):
|
|
b = beats[np.argmin(np.abs(beats - segs[i][0]))]
|
|
if abs(b - segs[i][0]) <= tolerance and segs[i - 1][0] < b < segs[i][1]:
|
|
segs[i - 1][1] = segs[i][0] = float(b)
|
|
return segs
|
|
|
|
|
|
def label_segments(segs):
|
|
"""Spell each chord for the key around it; drop no-chord segments."""
|
|
chords = [(s, e, c) for s, e, c in segs if c is not None]
|
|
out = []
|
|
for start, end, (pc, qual) in chords:
|
|
mid = (start + end) / 2
|
|
key = estimate_key(chords, mid - KEY_WINDOW_SECONDS / 2, mid + KEY_WINDOW_SECONDS / 2)
|
|
out.append({"start": round(start, 3), "end": round(end, 3), "label": spell(pc, qual, key)})
|
|
return out
|
|
|
|
|
|
def run_chordmini(chordmini: str, audio: str, model: str, out_dir: str) -> str:
|
|
ckpt, _ = MODELS[model]
|
|
in_dir = tempfile.mkdtemp(prefix="cm_in_")
|
|
link = os.path.join(in_dir, "song" + os.path.splitext(audio)[1])
|
|
os.symlink(os.path.abspath(audio), link)
|
|
args = [sys.executable, "src/evaluation/test.py", "--model_type", model, "--checkpoint", f"checkpoints/{ckpt}",
|
|
"--config", "config/ChordMini.yaml", "--audio_dir", in_dir, "--save_dir", out_dir, "--use_overlap",
|
|
"--vote_aggregation", "logit", "--min_segment_duration", "0.5", "--smooth_predictions",
|
|
"--use_gaussian", "--kernel_size", "9"]
|
|
if model == "BTC":
|
|
args.append("--smooth_logits")
|
|
subprocess.run(args, cwd=chordmini, check=True, capture_output=True)
|
|
return os.path.join(out_dir, "song.lab")
|
|
|
|
|
|
def main(argv=None):
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
p.add_argument("--chordmini", required=True, help="ChordMini checkout (github.com/ptnghia-j/ChordMini)")
|
|
p.add_argument("--input", action="append", required=True, help="name=path; e.g. mix=song.flac inst=inst.flac")
|
|
p.add_argument("--beats-from", help="audio to beat-track (default: the 'mix' input)")
|
|
p.add_argument("--out", required=True)
|
|
args = p.parse_args(argv)
|
|
import librosa
|
|
import scipy.signal
|
|
|
|
if not hasattr(scipy.signal, "hann"): # librosa 0.10.1 beat tracking on scipy >= 1.13
|
|
scipy.signal.hann = scipy.signal.windows.hann
|
|
|
|
inputs = dict(i.split("=", 1) for i in args.input)
|
|
labs, duration = [], 0.0
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
for name, path in inputs.items():
|
|
duration = max(duration, librosa.get_duration(path=path))
|
|
for model, (_, model_weight) in MODELS.items():
|
|
lab = run_chordmini(args.chordmini, path, model, os.path.join(tmp, f"{name}_{model}"))
|
|
labs.append((model_weight * INPUT_WEIGHTS.get(name, 1.0), read_lab(lab)))
|
|
segs = merge_short(frames_to_segments(vote(labs, duration)))
|
|
beat_src = args.beats_from or inputs.get("mix") or next(iter(inputs.values()))
|
|
y, sr = librosa.load(beat_src, sr=22050, mono=True)
|
|
_, beat_frames = librosa.beat.beat_track(y=y, sr=sr)
|
|
segs = snap_to_beats(segs, librosa.frames_to_time(beat_frames, sr=sr))
|
|
chords = label_segments(segs)
|
|
with open(args.out, "w") as f:
|
|
json.dump(chords, f, indent=1)
|
|
print(f"{len(chords)} chord changes -> {args.out}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|