Files
2026-04-23 14:59:57 +09:00

576 lines
20 KiB
Python

from __future__ import annotations
import hashlib
import json
import mimetypes
import re
import time
import unicodedata
from pathlib import Path
from typing import Any
from zipfile import BadZipFile, ZipFile
from ..config import MEDIA_DIR, SETTINGS_PATH, STEMENGINE_KARAOKE_EXPORT_DIR
from .project_service_common import ProjectOperationError
_MANIFEST_CACHE_TTL_SECONDS = 300.0
_MANIFEST_CACHE_MAX_ENTRIES = 2048
_MANIFEST_CACHE: dict[str, tuple[float, dict[str, Any]]] = {}
_MATCH_CACHE_TTL_SECONDS = 30.0
_MATCH_CACHE_MAX_ENTRIES = 256
_MATCH_CACHE: dict[str, tuple[float, dict[str, Any]]] = {}
def _load_settings() -> dict[str, Any]:
if not SETTINGS_PATH.exists():
return {}
try:
raw = json.loads(SETTINGS_PATH.read_text(encoding="utf-8"))
return raw if isinstance(raw, dict) else {}
except Exception:
return {}
def _default_sek_folder_path() -> Path:
configured = Path(str(STEMENGINE_KARAOKE_EXPORT_DIR).strip() or str(MEDIA_DIR / "sekfold"))
return configured if configured.is_absolute() else (MEDIA_DIR / configured)
def _coerce_folder_path(value: str) -> Path:
raw = str(value or "").strip()
if not raw:
raise ProjectOperationError(400, "No SEK folder configured")
# Friendly aliases used in UI/ops docs.
alias = raw.strip("/\\").lower()
if alias in {"sekfold", "default", "karaoke-sek", "karaoke_sek"}:
return _default_sek_folder_path()
path = Path(raw).expanduser()
if path.is_absolute():
return path
# Resolve relative paths against MEDIA_DIR to keep behavior container-stable.
return MEDIA_DIR / path
def _resolve_folder_path(folder: str | None = None) -> Path:
if folder and str(folder).strip():
return _coerce_folder_path(str(folder))
settings = _load_settings()
configured = str(settings.get("karaoke_sek_folder") or "").strip()
if not configured:
raise ProjectOperationError(400, "No SEK folder configured")
return _coerce_folder_path(configured)
def _ensure_folder_accessible(base: Path) -> None:
try:
if not base.exists() or not base.is_dir():
raise ProjectOperationError(404, f"SEK folder does not exist: {base}")
except ProjectOperationError:
raise
except OSError as exc:
raise ProjectOperationError(503, f"SEK folder is unavailable: {base} ({exc})") from exc
def _safe_resolved_str(path: Path) -> str:
try:
return str(path.resolve())
except OSError:
return str(path)
def _package_id(path: Path) -> str:
stable = _safe_resolved_str(path)
return hashlib.sha1(stable.encode("utf-8")).hexdigest()[:16]
def _prune_expired_cache(cache: dict[str, tuple[float, Any]], now: float, max_entries: int) -> None:
stale_keys = [k for k, (expires_at, _) in cache.items() if expires_at <= now]
for key in stale_keys:
cache.pop(key, None)
if len(cache) <= max_entries:
return
# Drop oldest-expiring first to keep bounded memory use.
overflow = len(cache) - max_entries
for key, _ in sorted(cache.items(), key=lambda item: item[1][0])[:overflow]:
cache.pop(key, None)
def _normalize_for_match(value: Any) -> str:
if value is None:
return ""
text = str(value).strip().lower()
if not text:
return ""
text = unicodedata.normalize("NFKD", text)
text = "".join(ch for ch in text if not unicodedata.combining(ch))
text = re.sub(r"\.[a-z0-9]{1,6}$", "", text)
text = re.sub(r"[^a-z0-9]+", " ", text)
text = re.sub(r"\s+", " ", text).strip()
return text
def _join_norm(*parts: Any) -> str:
values = [_normalize_for_match(p) for p in parts if _normalize_for_match(p)]
return " ".join(values).strip()
def _iter_sek_paths(base: Path) -> list[Path]:
# Recursive scan so exports organized as <sekfold>/<project>/<name>.sek are discovered.
try:
candidates = list(base.rglob("*.sek"))
except OSError as exc:
raise ProjectOperationError(503, f"Failed to scan SEK folder: {base} ({exc})") from exc
with_mtime: list[tuple[Path, float]] = []
for path in candidates:
try:
if path.is_file():
with_mtime.append((path, path.stat().st_mtime))
except OSError:
# Skip transient/unreadable entries instead of failing the whole listing.
continue
with_mtime.sort(key=lambda item: item[1], reverse=True)
return [p for p, _ in with_mtime]
def _extract_package_match_keys(
package_path: Path,
manifest: dict[str, Any] | None,
display_title: str,
) -> tuple[str, str, str, str, str, str, str, list[str]]:
metadata = (manifest or {}).get("metadata") if isinstance(manifest, dict) else {}
metadata = metadata if isinstance(metadata, dict) else {}
meta_title = str(metadata.get("title") or "").strip()
meta_artist = str(metadata.get("artist") or "").strip()
meta_albumartist = str(metadata.get("albumartist") or "").strip()
meta_album = str(metadata.get("album") or "").strip()
meta_source_file = str(metadata.get("source_file") or "").strip()
meta_source_url = str(metadata.get("source_url") or "").strip()
meta_navidrome_id = str(metadata.get("navidrome_id") or "").strip()
pkg_title = _normalize_for_match(meta_title or display_title)
pkg_artist = _normalize_for_match(meta_artist)
pkg_albumartist = _normalize_for_match(meta_albumartist)
pkg_filename = _normalize_for_match(package_path.stem)
keys: list[str] = []
for key in (
_join_norm(pkg_title, pkg_artist),
_join_norm(pkg_title, pkg_albumartist),
pkg_title,
pkg_filename,
):
if key and key not in keys:
keys.append(key)
return (
meta_title,
meta_artist,
meta_albumartist,
meta_album,
meta_source_file,
meta_source_url,
meta_navidrome_id,
keys,
)
def _safe_zip_member(member: str) -> bool:
if not member:
return False
p = Path(member)
if p.is_absolute():
return False
return ".." not in p.parts
def _read_manifest(package_path: Path) -> dict[str, Any]:
with ZipFile(package_path, "r") as zf:
try:
raw = zf.read("manifest.json")
except KeyError as exc:
raise ProjectOperationError(400, f"Package missing manifest.json: {package_path.name}") from exc
try:
parsed = json.loads(raw.decode("utf-8"))
if not isinstance(parsed, dict):
raise ValueError("manifest root must be object")
return parsed
except Exception as exc:
raise ProjectOperationError(400, f"Invalid manifest.json in {package_path.name}") from exc
def _read_manifest_cached(package_path: Path) -> dict[str, Any]:
try:
stat = package_path.stat()
except Exception as exc:
raise ProjectOperationError(400, f"Cannot stat package: {package_path.name}") from exc
key = "|".join(
[
str(package_path.resolve()),
str(getattr(stat, "st_mtime_ns", int(stat.st_mtime * 1_000_000_000))),
str(stat.st_size),
]
)
now = time.monotonic()
cached = _MANIFEST_CACHE.get(key)
if cached and cached[0] > now:
return cached[1]
manifest = _read_manifest(package_path)
_MANIFEST_CACHE[key] = (now + _MANIFEST_CACHE_TTL_SECONDS, manifest)
_prune_expired_cache(_MANIFEST_CACHE, now, _MANIFEST_CACHE_MAX_ENTRIES)
return manifest
def _extract_numeric(value: Any) -> float | None:
try:
if value is None:
return None
if isinstance(value, bool):
return None
num = float(value)
if num < 0:
return None
return num
except Exception:
return None
def _extract_duration_seconds_from_manifest(manifest: dict[str, Any] | None) -> float | None:
if not isinstance(manifest, dict):
return None
metadata = manifest.get("metadata") if isinstance(manifest.get("metadata"), dict) else {}
meta = manifest.get("meta") if isinstance(manifest.get("meta"), dict) else {}
direct_candidates = [
metadata.get("duration_seconds"),
metadata.get("duration"),
metadata.get("length_seconds"),
meta.get("duration_seconds"),
meta.get("duration"),
]
for candidate in direct_candidates:
numeric = _extract_numeric(candidate)
if numeric is not None:
return numeric
source_media = metadata.get("source_media") if isinstance(metadata.get("source_media"), dict) else {}
for candidate in [source_media.get("duration_seconds"), source_media.get("duration")]:
numeric = _extract_numeric(candidate)
if numeric is not None:
return numeric
return None
def list_sek_packages(folder: str | None = None) -> dict[str, Any]:
started = time.perf_counter()
base = _resolve_folder_path(folder)
_ensure_folder_accessible(base)
packages: list[dict[str, Any]] = []
scanned = _iter_sek_paths(base)
for package_path in scanned:
warnings: list[str] = []
manifest: dict[str, Any] | None = None
valid = True
try:
mtime = package_path.stat().st_mtime
except OSError:
mtime = 0.0
warnings.append("Package mtime unavailable")
try:
manifest = _read_manifest_cached(package_path)
except (ProjectOperationError, BadZipFile):
valid = False
warnings.append("Unreadable package or missing manifest.json")
contents = (manifest or {}).get("contents") if isinstance(manifest, dict) else {}
metadata = (manifest or {}).get("metadata") if isinstance(manifest, dict) else {}
stems = contents.get("stems") if isinstance(contents, dict) else []
lyrics = contents.get("lyrics") if isinstance(contents, dict) else ""
lyrics_tracks = contents.get("lyrics_tracks") if isinstance(contents, dict) else {}
video = contents.get("video") if isinstance(contents, dict) else ""
dynamic_palette = metadata.get("dynamic_palette") if isinstance(metadata, dict) else None
if not isinstance(dynamic_palette, dict):
dynamic_palette = None
title = (
((manifest or {}).get("project") or {}).get("name")
if isinstance((manifest or {}).get("project"), dict)
else None
) or package_path.stem
(
meta_title,
meta_artist,
meta_albumartist,
meta_album,
meta_source_file,
meta_source_url,
meta_navidrome_id,
match_keys,
) = _extract_package_match_keys(package_path, manifest, str(title))
meta_duration_seconds = _extract_duration_seconds_from_manifest(manifest)
has_lyrics_tracks = isinstance(lyrics_tracks, dict) and any(isinstance(v, str) and v for v in lyrics_tracks.values())
if valid and not lyrics and not has_lyrics_tracks:
warnings.append("manifest contents.lyrics is missing")
if valid and not isinstance(stems, list):
warnings.append("manifest contents.stems is not a list")
packages.append(
{
"id": _package_id(package_path),
"filename": package_path.name,
"path": str(package_path.resolve()),
"title": title,
"valid": valid,
"stem_count": len(stems) if isinstance(stems, list) else 0,
"has_video": bool(video),
"lyrics": lyrics if isinstance(lyrics, str) else "",
"lyrics_track_count": len(lyrics_tracks) if isinstance(lyrics_tracks, dict) else 0,
"mtime": mtime,
"warnings": warnings,
"meta_title": meta_title,
"meta_artist": meta_artist,
"meta_albumartist": meta_albumartist,
"meta_album": meta_album,
"meta_source_file": meta_source_file,
"meta_source_url": meta_source_url,
"meta_navidrome_id": meta_navidrome_id,
"meta_duration_seconds": meta_duration_seconds,
"match_keys": match_keys,
"meta_dynamic_palette": dynamic_palette,
}
)
return {
"folder": _safe_resolved_str(base),
"scan_root": _safe_resolved_str(base),
"scan_count": len(scanned),
"scan_ms": round((time.perf_counter() - started) * 1000, 2),
"count": len(packages),
"packages": packages,
}
def _resolve_package_by_id(package_id: str, folder: str | None = None) -> Path:
base = _resolve_folder_path(folder)
_ensure_folder_accessible(base)
for package_path in _iter_sek_paths(base):
if _package_id(package_path) == package_id:
return package_path
raise ProjectOperationError(404, f"Package not found: {package_id}")
def get_sek_manifest(package_id: str, folder: str | None = None) -> dict[str, Any]:
package_path = _resolve_package_by_id(package_id, folder)
manifest = _read_manifest(package_path)
return {
"id": package_id,
"filename": package_path.name,
"path": str(package_path.resolve()),
"manifest": manifest,
}
def get_sek_asset(package_id: str, inner_path: str, folder: str | None = None) -> tuple[bytes, str, str]:
package_path = _resolve_package_by_id(package_id, folder)
normalized = str(inner_path or "").strip().replace("\\", "/")
if not _safe_zip_member(normalized):
raise ProjectOperationError(400, "Invalid package asset path")
with ZipFile(package_path, "r") as zf:
names = set(zf.namelist())
if normalized not in names:
raise ProjectOperationError(404, f"Asset not found in package: {normalized}")
data = zf.read(normalized)
mime, _ = mimetypes.guess_type(normalized)
return data, (mime or "application/octet-stream"), normalized.split("/")[-1]
def _score_song_vs_package(song: dict[str, Any], package: dict[str, Any]) -> tuple[float, list[str]]:
reasons: list[str] = []
score = 0.0
song_title = _normalize_for_match(song.get("title"))
song_artist_primary = _normalize_for_match(song.get("songArtist") or song.get("artist"))
song_artist_fallback = _normalize_for_match(song.get("artist"))
song_albumartist = _normalize_for_match(song.get("albumArtist"))
package_title = _normalize_for_match(package.get("meta_title") or package.get("title"))
package_keys = set(
k for k in (package.get("match_keys") or []) if isinstance(k, str) and k.strip()
)
song_primary = _join_norm(song_title, song_artist_primary)
song_artist_alt = _join_norm(song_title, song_artist_fallback)
song_albumartist_key = _join_norm(song_title, song_albumartist)
if song_title and song_artist_primary and song_primary and song_primary in package_keys:
score = max(score, 1.0)
reasons.append("exact:title+primary-artist")
if song_title and song_artist_fallback and song_artist_alt and song_artist_alt in package_keys:
score = max(score, 0.98)
reasons.append("exact:title+artist")
if song_title and song_albumartist and song_albumartist_key and song_albumartist_key in package_keys:
score = max(score, 0.95)
reasons.append("exact:title+albumartist")
if song_title and song_title in package_keys:
score = max(score, 0.85)
reasons.append("exact:title")
# Title containment fallback for older packages with sparse metadata.
if song_title and package_title and (song_title in package_title or package_title in song_title):
score = max(score, 0.70)
reasons.append("near:title-contains")
# Filename containment fallback (low confidence).
pkg_filename = _normalize_for_match(package.get("filename"))
if song_title and pkg_filename and (song_title in pkg_filename or pkg_filename in song_title):
score = max(score, 0.70)
reasons.append("near:filename-contains")
song_duration = _extract_numeric(song.get("duration") or song.get("durationSeconds") or song.get("duration_seconds"))
package_duration = _extract_numeric(package.get("meta_duration_seconds"))
if song_duration is not None and package_duration is not None:
delta = abs(song_duration - package_duration)
if delta <= 0.5:
score = min(1.0, score + 0.04)
reasons.append("duration:exact")
elif delta <= 2.0:
score = min(1.0, score + 0.03)
reasons.append("duration:within-2s")
elif delta <= 5.0:
score = min(1.0, score + 0.01)
reasons.append("duration:within-5s")
elif delta >= 20.0 and score >= 0.85:
# Mild penalty for otherwise-strong text matches with very different durations.
score = max(0.0, score - 0.03)
reasons.append("duration:far")
return score, reasons
def _build_match_cache_key(folder: str | None, songs: list[dict[str, Any]]) -> str:
base = _resolve_folder_path(folder)
songs_payload = [
{
"id": str((song or {}).get("id") or ""),
"title": str((song or {}).get("title") or ""),
"artist": str((song or {}).get("artist") or ""),
"songArtist": str((song or {}).get("songArtist") or ""),
"albumArtist": str((song or {}).get("albumArtist") or ""),
"album": str((song or {}).get("album") or ""),
"duration": (song or {}).get("duration"),
}
for song in songs
]
songs_json = json.dumps(songs_payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
songs_hash = hashlib.sha1(songs_json.encode("utf-8")).hexdigest()
return f"{_safe_resolved_str(base)}|{songs_hash}"
def match_subsonic_songs(
songs: list[dict[str, Any]],
folder: str | None = None,
) -> dict[str, Any]:
cache_key = _build_match_cache_key(folder, songs)
now = time.monotonic()
cached = _MATCH_CACHE.get(cache_key)
if cached and cached[0] > now:
cached_payload = dict(cached[1])
cached_payload["cache"] = "hit"
return cached_payload
inventory = list_sek_packages(folder)
packages = inventory.get("packages") or []
matches: dict[str, list[dict[str, Any]]] = {}
matched = 0
unmatched = 0
ambiguous = 0
for index, song in enumerate(songs):
song_id = str((song or {}).get("id") or f"row-{index}")
candidates: list[dict[str, Any]] = []
for package in packages:
score, reasons = _score_song_vs_package(song or {}, package)
if score < 0.70:
continue
song_duration = _extract_numeric((song or {}).get("duration") or (song or {}).get("durationSeconds") or (song or {}).get("duration_seconds"))
package_duration = _extract_numeric(package.get("meta_duration_seconds"))
duration_delta: float | None = None
if song_duration is not None and package_duration is not None:
duration_delta = abs(song_duration - package_duration)
candidates.append(
{
"package_id": package.get("id"),
"filename": package.get("filename"),
"title": package.get("title"),
"confidence": round(score, 4),
"reasons": reasons,
"duration_delta_seconds": round(duration_delta, 3) if duration_delta is not None else None,
}
)
candidates.sort(
key=lambda c: (
-float(c.get("confidence") or 0.0),
float(c.get("duration_delta_seconds")) if c.get("duration_delta_seconds") is not None else 999999.0,
str(c.get("title") or ""),
str(c.get("filename") or ""),
),
)
if candidates:
matched += 1
strongish = [c for c in candidates if float(c.get("confidence") or 0.0) >= 0.85]
if len(strongish) > 1:
ambiguous += 1
else:
unmatched += 1
matches[song_id] = candidates
response = {
"folder": inventory.get("folder"),
"scan_root": inventory.get("scan_root"),
"scan_count": inventory.get("scan_count"),
"scan_ms": inventory.get("scan_ms"),
"cache": "miss",
"summary": {
"songs": len(songs),
"matched": matched,
"unmatched": unmatched,
"ambiguous": ambiguous,
},
"matches": matches,
}
_MATCH_CACHE[cache_key] = (now + _MATCH_CACHE_TTL_SECONDS, response)
_prune_expired_cache(_MATCH_CACHE, now, _MATCH_CACHE_MAX_ENTRIES)
return response