576 lines
20 KiB
Python
576 lines
20 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import mimetypes
|
|
import re
|
|
import time
|
|
import unicodedata
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from zipfile import BadZipFile, ZipFile
|
|
|
|
from ..config import MEDIA_DIR, SETTINGS_PATH, STEMENGINE_KARAOKE_EXPORT_DIR
|
|
from .project_service_common import ProjectOperationError
|
|
|
|
|
|
_MANIFEST_CACHE_TTL_SECONDS = 300.0
|
|
_MANIFEST_CACHE_MAX_ENTRIES = 2048
|
|
_MANIFEST_CACHE: dict[str, tuple[float, dict[str, Any]]] = {}
|
|
|
|
_MATCH_CACHE_TTL_SECONDS = 30.0
|
|
_MATCH_CACHE_MAX_ENTRIES = 256
|
|
_MATCH_CACHE: dict[str, tuple[float, dict[str, Any]]] = {}
|
|
|
|
|
|
def _load_settings() -> dict[str, Any]:
|
|
if not SETTINGS_PATH.exists():
|
|
return {}
|
|
try:
|
|
raw = json.loads(SETTINGS_PATH.read_text(encoding="utf-8"))
|
|
return raw if isinstance(raw, dict) else {}
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def _default_sek_folder_path() -> Path:
|
|
configured = Path(str(STEMENGINE_KARAOKE_EXPORT_DIR).strip() or str(MEDIA_DIR / "sekfold"))
|
|
return configured if configured.is_absolute() else (MEDIA_DIR / configured)
|
|
|
|
|
|
def _coerce_folder_path(value: str) -> Path:
|
|
raw = str(value or "").strip()
|
|
if not raw:
|
|
raise ProjectOperationError(400, "No SEK folder configured")
|
|
|
|
# Friendly aliases used in UI/ops docs.
|
|
alias = raw.strip("/\\").lower()
|
|
if alias in {"sekfold", "default", "karaoke-sek", "karaoke_sek"}:
|
|
return _default_sek_folder_path()
|
|
|
|
path = Path(raw).expanduser()
|
|
if path.is_absolute():
|
|
return path
|
|
|
|
# Resolve relative paths against MEDIA_DIR to keep behavior container-stable.
|
|
return MEDIA_DIR / path
|
|
|
|
|
|
def _resolve_folder_path(folder: str | None = None) -> Path:
|
|
if folder and str(folder).strip():
|
|
return _coerce_folder_path(str(folder))
|
|
settings = _load_settings()
|
|
configured = str(settings.get("karaoke_sek_folder") or "").strip()
|
|
if not configured:
|
|
raise ProjectOperationError(400, "No SEK folder configured")
|
|
return _coerce_folder_path(configured)
|
|
|
|
|
|
def _ensure_folder_accessible(base: Path) -> None:
|
|
try:
|
|
if not base.exists() or not base.is_dir():
|
|
raise ProjectOperationError(404, f"SEK folder does not exist: {base}")
|
|
except ProjectOperationError:
|
|
raise
|
|
except OSError as exc:
|
|
raise ProjectOperationError(503, f"SEK folder is unavailable: {base} ({exc})") from exc
|
|
|
|
|
|
def _safe_resolved_str(path: Path) -> str:
|
|
try:
|
|
return str(path.resolve())
|
|
except OSError:
|
|
return str(path)
|
|
|
|
|
|
def _package_id(path: Path) -> str:
|
|
stable = _safe_resolved_str(path)
|
|
return hashlib.sha1(stable.encode("utf-8")).hexdigest()[:16]
|
|
|
|
|
|
def _prune_expired_cache(cache: dict[str, tuple[float, Any]], now: float, max_entries: int) -> None:
|
|
stale_keys = [k for k, (expires_at, _) in cache.items() if expires_at <= now]
|
|
for key in stale_keys:
|
|
cache.pop(key, None)
|
|
if len(cache) <= max_entries:
|
|
return
|
|
# Drop oldest-expiring first to keep bounded memory use.
|
|
overflow = len(cache) - max_entries
|
|
for key, _ in sorted(cache.items(), key=lambda item: item[1][0])[:overflow]:
|
|
cache.pop(key, None)
|
|
|
|
|
|
def _normalize_for_match(value: Any) -> str:
|
|
if value is None:
|
|
return ""
|
|
text = str(value).strip().lower()
|
|
if not text:
|
|
return ""
|
|
|
|
text = unicodedata.normalize("NFKD", text)
|
|
text = "".join(ch for ch in text if not unicodedata.combining(ch))
|
|
text = re.sub(r"\.[a-z0-9]{1,6}$", "", text)
|
|
text = re.sub(r"[^a-z0-9]+", " ", text)
|
|
text = re.sub(r"\s+", " ", text).strip()
|
|
return text
|
|
|
|
|
|
def _join_norm(*parts: Any) -> str:
|
|
values = [_normalize_for_match(p) for p in parts if _normalize_for_match(p)]
|
|
return " ".join(values).strip()
|
|
|
|
|
|
def _iter_sek_paths(base: Path) -> list[Path]:
|
|
# Recursive scan so exports organized as <sekfold>/<project>/<name>.sek are discovered.
|
|
try:
|
|
candidates = list(base.rglob("*.sek"))
|
|
except OSError as exc:
|
|
raise ProjectOperationError(503, f"Failed to scan SEK folder: {base} ({exc})") from exc
|
|
|
|
with_mtime: list[tuple[Path, float]] = []
|
|
for path in candidates:
|
|
try:
|
|
if path.is_file():
|
|
with_mtime.append((path, path.stat().st_mtime))
|
|
except OSError:
|
|
# Skip transient/unreadable entries instead of failing the whole listing.
|
|
continue
|
|
|
|
with_mtime.sort(key=lambda item: item[1], reverse=True)
|
|
return [p for p, _ in with_mtime]
|
|
|
|
|
|
def _extract_package_match_keys(
|
|
package_path: Path,
|
|
manifest: dict[str, Any] | None,
|
|
display_title: str,
|
|
) -> tuple[str, str, str, str, str, str, str, list[str]]:
|
|
metadata = (manifest or {}).get("metadata") if isinstance(manifest, dict) else {}
|
|
metadata = metadata if isinstance(metadata, dict) else {}
|
|
|
|
meta_title = str(metadata.get("title") or "").strip()
|
|
meta_artist = str(metadata.get("artist") or "").strip()
|
|
meta_albumartist = str(metadata.get("albumartist") or "").strip()
|
|
meta_album = str(metadata.get("album") or "").strip()
|
|
meta_source_file = str(metadata.get("source_file") or "").strip()
|
|
meta_source_url = str(metadata.get("source_url") or "").strip()
|
|
meta_navidrome_id = str(metadata.get("navidrome_id") or "").strip()
|
|
|
|
pkg_title = _normalize_for_match(meta_title or display_title)
|
|
pkg_artist = _normalize_for_match(meta_artist)
|
|
pkg_albumartist = _normalize_for_match(meta_albumartist)
|
|
pkg_filename = _normalize_for_match(package_path.stem)
|
|
|
|
keys: list[str] = []
|
|
for key in (
|
|
_join_norm(pkg_title, pkg_artist),
|
|
_join_norm(pkg_title, pkg_albumartist),
|
|
pkg_title,
|
|
pkg_filename,
|
|
):
|
|
if key and key not in keys:
|
|
keys.append(key)
|
|
|
|
return (
|
|
meta_title,
|
|
meta_artist,
|
|
meta_albumartist,
|
|
meta_album,
|
|
meta_source_file,
|
|
meta_source_url,
|
|
meta_navidrome_id,
|
|
keys,
|
|
)
|
|
|
|
|
|
def _safe_zip_member(member: str) -> bool:
|
|
if not member:
|
|
return False
|
|
p = Path(member)
|
|
if p.is_absolute():
|
|
return False
|
|
return ".." not in p.parts
|
|
|
|
|
|
def _read_manifest(package_path: Path) -> dict[str, Any]:
|
|
with ZipFile(package_path, "r") as zf:
|
|
try:
|
|
raw = zf.read("manifest.json")
|
|
except KeyError as exc:
|
|
raise ProjectOperationError(400, f"Package missing manifest.json: {package_path.name}") from exc
|
|
try:
|
|
parsed = json.loads(raw.decode("utf-8"))
|
|
if not isinstance(parsed, dict):
|
|
raise ValueError("manifest root must be object")
|
|
return parsed
|
|
except Exception as exc:
|
|
raise ProjectOperationError(400, f"Invalid manifest.json in {package_path.name}") from exc
|
|
|
|
|
|
def _read_manifest_cached(package_path: Path) -> dict[str, Any]:
|
|
try:
|
|
stat = package_path.stat()
|
|
except Exception as exc:
|
|
raise ProjectOperationError(400, f"Cannot stat package: {package_path.name}") from exc
|
|
|
|
key = "|".join(
|
|
[
|
|
str(package_path.resolve()),
|
|
str(getattr(stat, "st_mtime_ns", int(stat.st_mtime * 1_000_000_000))),
|
|
str(stat.st_size),
|
|
]
|
|
)
|
|
|
|
now = time.monotonic()
|
|
cached = _MANIFEST_CACHE.get(key)
|
|
if cached and cached[0] > now:
|
|
return cached[1]
|
|
|
|
manifest = _read_manifest(package_path)
|
|
_MANIFEST_CACHE[key] = (now + _MANIFEST_CACHE_TTL_SECONDS, manifest)
|
|
_prune_expired_cache(_MANIFEST_CACHE, now, _MANIFEST_CACHE_MAX_ENTRIES)
|
|
return manifest
|
|
|
|
|
|
def _extract_numeric(value: Any) -> float | None:
|
|
try:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, bool):
|
|
return None
|
|
num = float(value)
|
|
if num < 0:
|
|
return None
|
|
return num
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _extract_duration_seconds_from_manifest(manifest: dict[str, Any] | None) -> float | None:
|
|
if not isinstance(manifest, dict):
|
|
return None
|
|
|
|
metadata = manifest.get("metadata") if isinstance(manifest.get("metadata"), dict) else {}
|
|
meta = manifest.get("meta") if isinstance(manifest.get("meta"), dict) else {}
|
|
|
|
direct_candidates = [
|
|
metadata.get("duration_seconds"),
|
|
metadata.get("duration"),
|
|
metadata.get("length_seconds"),
|
|
meta.get("duration_seconds"),
|
|
meta.get("duration"),
|
|
]
|
|
for candidate in direct_candidates:
|
|
numeric = _extract_numeric(candidate)
|
|
if numeric is not None:
|
|
return numeric
|
|
|
|
source_media = metadata.get("source_media") if isinstance(metadata.get("source_media"), dict) else {}
|
|
for candidate in [source_media.get("duration_seconds"), source_media.get("duration")]:
|
|
numeric = _extract_numeric(candidate)
|
|
if numeric is not None:
|
|
return numeric
|
|
|
|
return None
|
|
|
|
|
|
def list_sek_packages(folder: str | None = None) -> dict[str, Any]:
|
|
started = time.perf_counter()
|
|
base = _resolve_folder_path(folder)
|
|
_ensure_folder_accessible(base)
|
|
|
|
packages: list[dict[str, Any]] = []
|
|
scanned = _iter_sek_paths(base)
|
|
for package_path in scanned:
|
|
warnings: list[str] = []
|
|
manifest: dict[str, Any] | None = None
|
|
valid = True
|
|
try:
|
|
mtime = package_path.stat().st_mtime
|
|
except OSError:
|
|
mtime = 0.0
|
|
warnings.append("Package mtime unavailable")
|
|
|
|
try:
|
|
manifest = _read_manifest_cached(package_path)
|
|
except (ProjectOperationError, BadZipFile):
|
|
valid = False
|
|
warnings.append("Unreadable package or missing manifest.json")
|
|
|
|
contents = (manifest or {}).get("contents") if isinstance(manifest, dict) else {}
|
|
metadata = (manifest or {}).get("metadata") if isinstance(manifest, dict) else {}
|
|
stems = contents.get("stems") if isinstance(contents, dict) else []
|
|
lyrics = contents.get("lyrics") if isinstance(contents, dict) else ""
|
|
lyrics_tracks = contents.get("lyrics_tracks") if isinstance(contents, dict) else {}
|
|
video = contents.get("video") if isinstance(contents, dict) else ""
|
|
dynamic_palette = metadata.get("dynamic_palette") if isinstance(metadata, dict) else None
|
|
if not isinstance(dynamic_palette, dict):
|
|
dynamic_palette = None
|
|
|
|
title = (
|
|
((manifest or {}).get("project") or {}).get("name")
|
|
if isinstance((manifest or {}).get("project"), dict)
|
|
else None
|
|
) or package_path.stem
|
|
|
|
(
|
|
meta_title,
|
|
meta_artist,
|
|
meta_albumartist,
|
|
meta_album,
|
|
meta_source_file,
|
|
meta_source_url,
|
|
meta_navidrome_id,
|
|
match_keys,
|
|
) = _extract_package_match_keys(package_path, manifest, str(title))
|
|
meta_duration_seconds = _extract_duration_seconds_from_manifest(manifest)
|
|
|
|
has_lyrics_tracks = isinstance(lyrics_tracks, dict) and any(isinstance(v, str) and v for v in lyrics_tracks.values())
|
|
if valid and not lyrics and not has_lyrics_tracks:
|
|
warnings.append("manifest contents.lyrics is missing")
|
|
if valid and not isinstance(stems, list):
|
|
warnings.append("manifest contents.stems is not a list")
|
|
|
|
packages.append(
|
|
{
|
|
"id": _package_id(package_path),
|
|
"filename": package_path.name,
|
|
"path": str(package_path.resolve()),
|
|
"title": title,
|
|
"valid": valid,
|
|
"stem_count": len(stems) if isinstance(stems, list) else 0,
|
|
"has_video": bool(video),
|
|
"lyrics": lyrics if isinstance(lyrics, str) else "",
|
|
"lyrics_track_count": len(lyrics_tracks) if isinstance(lyrics_tracks, dict) else 0,
|
|
"mtime": mtime,
|
|
"warnings": warnings,
|
|
"meta_title": meta_title,
|
|
"meta_artist": meta_artist,
|
|
"meta_albumartist": meta_albumartist,
|
|
"meta_album": meta_album,
|
|
"meta_source_file": meta_source_file,
|
|
"meta_source_url": meta_source_url,
|
|
"meta_navidrome_id": meta_navidrome_id,
|
|
"meta_duration_seconds": meta_duration_seconds,
|
|
"match_keys": match_keys,
|
|
"meta_dynamic_palette": dynamic_palette,
|
|
}
|
|
)
|
|
|
|
return {
|
|
"folder": _safe_resolved_str(base),
|
|
"scan_root": _safe_resolved_str(base),
|
|
"scan_count": len(scanned),
|
|
"scan_ms": round((time.perf_counter() - started) * 1000, 2),
|
|
"count": len(packages),
|
|
"packages": packages,
|
|
}
|
|
|
|
|
|
def _resolve_package_by_id(package_id: str, folder: str | None = None) -> Path:
|
|
base = _resolve_folder_path(folder)
|
|
_ensure_folder_accessible(base)
|
|
|
|
for package_path in _iter_sek_paths(base):
|
|
if _package_id(package_path) == package_id:
|
|
return package_path
|
|
raise ProjectOperationError(404, f"Package not found: {package_id}")
|
|
|
|
|
|
def get_sek_manifest(package_id: str, folder: str | None = None) -> dict[str, Any]:
|
|
package_path = _resolve_package_by_id(package_id, folder)
|
|
manifest = _read_manifest(package_path)
|
|
return {
|
|
"id": package_id,
|
|
"filename": package_path.name,
|
|
"path": str(package_path.resolve()),
|
|
"manifest": manifest,
|
|
}
|
|
|
|
|
|
def get_sek_asset(package_id: str, inner_path: str, folder: str | None = None) -> tuple[bytes, str, str]:
|
|
package_path = _resolve_package_by_id(package_id, folder)
|
|
normalized = str(inner_path or "").strip().replace("\\", "/")
|
|
if not _safe_zip_member(normalized):
|
|
raise ProjectOperationError(400, "Invalid package asset path")
|
|
|
|
with ZipFile(package_path, "r") as zf:
|
|
names = set(zf.namelist())
|
|
if normalized not in names:
|
|
raise ProjectOperationError(404, f"Asset not found in package: {normalized}")
|
|
data = zf.read(normalized)
|
|
|
|
mime, _ = mimetypes.guess_type(normalized)
|
|
return data, (mime or "application/octet-stream"), normalized.split("/")[-1]
|
|
|
|
|
|
def _score_song_vs_package(song: dict[str, Any], package: dict[str, Any]) -> tuple[float, list[str]]:
|
|
reasons: list[str] = []
|
|
score = 0.0
|
|
|
|
song_title = _normalize_for_match(song.get("title"))
|
|
song_artist_primary = _normalize_for_match(song.get("songArtist") or song.get("artist"))
|
|
song_artist_fallback = _normalize_for_match(song.get("artist"))
|
|
song_albumartist = _normalize_for_match(song.get("albumArtist"))
|
|
|
|
package_title = _normalize_for_match(package.get("meta_title") or package.get("title"))
|
|
package_keys = set(
|
|
k for k in (package.get("match_keys") or []) if isinstance(k, str) and k.strip()
|
|
)
|
|
|
|
song_primary = _join_norm(song_title, song_artist_primary)
|
|
song_artist_alt = _join_norm(song_title, song_artist_fallback)
|
|
song_albumartist_key = _join_norm(song_title, song_albumartist)
|
|
|
|
if song_title and song_artist_primary and song_primary and song_primary in package_keys:
|
|
score = max(score, 1.0)
|
|
reasons.append("exact:title+primary-artist")
|
|
|
|
if song_title and song_artist_fallback and song_artist_alt and song_artist_alt in package_keys:
|
|
score = max(score, 0.98)
|
|
reasons.append("exact:title+artist")
|
|
|
|
if song_title and song_albumartist and song_albumartist_key and song_albumartist_key in package_keys:
|
|
score = max(score, 0.95)
|
|
reasons.append("exact:title+albumartist")
|
|
|
|
if song_title and song_title in package_keys:
|
|
score = max(score, 0.85)
|
|
reasons.append("exact:title")
|
|
|
|
# Title containment fallback for older packages with sparse metadata.
|
|
if song_title and package_title and (song_title in package_title or package_title in song_title):
|
|
score = max(score, 0.70)
|
|
reasons.append("near:title-contains")
|
|
|
|
# Filename containment fallback (low confidence).
|
|
pkg_filename = _normalize_for_match(package.get("filename"))
|
|
if song_title and pkg_filename and (song_title in pkg_filename or pkg_filename in song_title):
|
|
score = max(score, 0.70)
|
|
reasons.append("near:filename-contains")
|
|
|
|
song_duration = _extract_numeric(song.get("duration") or song.get("durationSeconds") or song.get("duration_seconds"))
|
|
package_duration = _extract_numeric(package.get("meta_duration_seconds"))
|
|
|
|
if song_duration is not None and package_duration is not None:
|
|
delta = abs(song_duration - package_duration)
|
|
if delta <= 0.5:
|
|
score = min(1.0, score + 0.04)
|
|
reasons.append("duration:exact")
|
|
elif delta <= 2.0:
|
|
score = min(1.0, score + 0.03)
|
|
reasons.append("duration:within-2s")
|
|
elif delta <= 5.0:
|
|
score = min(1.0, score + 0.01)
|
|
reasons.append("duration:within-5s")
|
|
elif delta >= 20.0 and score >= 0.85:
|
|
# Mild penalty for otherwise-strong text matches with very different durations.
|
|
score = max(0.0, score - 0.03)
|
|
reasons.append("duration:far")
|
|
|
|
return score, reasons
|
|
|
|
|
|
def _build_match_cache_key(folder: str | None, songs: list[dict[str, Any]]) -> str:
|
|
base = _resolve_folder_path(folder)
|
|
songs_payload = [
|
|
{
|
|
"id": str((song or {}).get("id") or ""),
|
|
"title": str((song or {}).get("title") or ""),
|
|
"artist": str((song or {}).get("artist") or ""),
|
|
"songArtist": str((song or {}).get("songArtist") or ""),
|
|
"albumArtist": str((song or {}).get("albumArtist") or ""),
|
|
"album": str((song or {}).get("album") or ""),
|
|
"duration": (song or {}).get("duration"),
|
|
}
|
|
for song in songs
|
|
]
|
|
songs_json = json.dumps(songs_payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
songs_hash = hashlib.sha1(songs_json.encode("utf-8")).hexdigest()
|
|
return f"{_safe_resolved_str(base)}|{songs_hash}"
|
|
|
|
|
|
def match_subsonic_songs(
|
|
songs: list[dict[str, Any]],
|
|
folder: str | None = None,
|
|
) -> dict[str, Any]:
|
|
cache_key = _build_match_cache_key(folder, songs)
|
|
now = time.monotonic()
|
|
cached = _MATCH_CACHE.get(cache_key)
|
|
if cached and cached[0] > now:
|
|
cached_payload = dict(cached[1])
|
|
cached_payload["cache"] = "hit"
|
|
return cached_payload
|
|
|
|
inventory = list_sek_packages(folder)
|
|
packages = inventory.get("packages") or []
|
|
|
|
matches: dict[str, list[dict[str, Any]]] = {}
|
|
matched = 0
|
|
unmatched = 0
|
|
ambiguous = 0
|
|
|
|
for index, song in enumerate(songs):
|
|
song_id = str((song or {}).get("id") or f"row-{index}")
|
|
candidates: list[dict[str, Any]] = []
|
|
|
|
for package in packages:
|
|
score, reasons = _score_song_vs_package(song or {}, package)
|
|
if score < 0.70:
|
|
continue
|
|
|
|
song_duration = _extract_numeric((song or {}).get("duration") or (song or {}).get("durationSeconds") or (song or {}).get("duration_seconds"))
|
|
package_duration = _extract_numeric(package.get("meta_duration_seconds"))
|
|
duration_delta: float | None = None
|
|
if song_duration is not None and package_duration is not None:
|
|
duration_delta = abs(song_duration - package_duration)
|
|
|
|
candidates.append(
|
|
{
|
|
"package_id": package.get("id"),
|
|
"filename": package.get("filename"),
|
|
"title": package.get("title"),
|
|
"confidence": round(score, 4),
|
|
"reasons": reasons,
|
|
"duration_delta_seconds": round(duration_delta, 3) if duration_delta is not None else None,
|
|
}
|
|
)
|
|
|
|
candidates.sort(
|
|
key=lambda c: (
|
|
-float(c.get("confidence") or 0.0),
|
|
float(c.get("duration_delta_seconds")) if c.get("duration_delta_seconds") is not None else 999999.0,
|
|
str(c.get("title") or ""),
|
|
str(c.get("filename") or ""),
|
|
),
|
|
)
|
|
|
|
if candidates:
|
|
matched += 1
|
|
strongish = [c for c in candidates if float(c.get("confidence") or 0.0) >= 0.85]
|
|
if len(strongish) > 1:
|
|
ambiguous += 1
|
|
else:
|
|
unmatched += 1
|
|
|
|
matches[song_id] = candidates
|
|
|
|
response = {
|
|
"folder": inventory.get("folder"),
|
|
"scan_root": inventory.get("scan_root"),
|
|
"scan_count": inventory.get("scan_count"),
|
|
"scan_ms": inventory.get("scan_ms"),
|
|
"cache": "miss",
|
|
"summary": {
|
|
"songs": len(songs),
|
|
"matched": matched,
|
|
"unmatched": unmatched,
|
|
"ambiguous": ambiguous,
|
|
},
|
|
"matches": matches,
|
|
}
|
|
|
|
_MATCH_CACHE[cache_key] = (now + _MATCH_CACHE_TTL_SECONDS, response)
|
|
_prune_expired_cache(_MATCH_CACHE, now, _MATCH_CACHE_MAX_ENTRIES)
|
|
return response
|