backend: finalize phase updates, migrations, and integration wiring

This commit is contained in:
translator committed 2026-03-14 11:17:58 +09:00
1 parent 51d277f502
commit e5ca71f7c4
23 files changed
+2637 -87

No files matched your search

+52
View File
@@ -0,0 +1,52 @@
# StemEngine backend — development helpers
#
# All commands run tests inside the built Docker image so they use the same
# Python + dependency versions as production. Build the image first with
# `make build` before running any test targets.
IMAGE := stemengine:latest
BACKEND := backend
.PHONY: build test test-all test-smoke test-phase4 test-phase5 test-normalization alembic-upgrade alembic-heads help
help:
@echo ""
@echo " make build Build the Docker image"
@echo " make test Run the full backend test suite"
@echo " make test-smoke Smoke tests only (app boot + core routes)"
@echo " make test-phase4 Phase 4 operational-hardening tests"
@echo " make test-phase5 Phase 5 normalization tests"
@echo " make test-normalization Alias for test-phase5"
@echo " make alembic-upgrade Apply pending Alembic migrations inside container"
@echo " make alembic-heads Show current Alembic revision heads"
@echo ""
build:
docker build -t $(IMAGE) .
# Full test suite — discovers all test_*.py under /app/backend/tests
test:
docker run --rm $(IMAGE) bash -lc \
"python3.11 -m unittest discover -s /app/$(BACKEND)/tests -p 'test_*.py' -v"
# Subset targets for quick iteration
test-smoke:
docker run --rm $(IMAGE) bash -lc \
"python3.11 -m unittest $(BACKEND).tests.test_app_smoke -v"
test-phase4:
docker run --rm $(IMAGE) bash -lc \
"python3.11 -m unittest $(BACKEND).tests.test_operational_hardening -v"
test-phase5 test-normalization:
docker run --rm $(IMAGE) bash -lc \
"python3.11 -m unittest $(BACKEND).tests.test_phase5_normalization -v"
# Alembic helpers — run against the container's bundled DB path
alembic-upgrade:
docker run --rm $(IMAGE) bash -lc \
"cd /app && python3.11 -m alembic -c $(BACKEND)/alembic.ini upgrade head"
alembic-heads:
docker run --rm $(IMAGE) bash -lc \
"cd /app && python3.11 -m alembic -c $(BACKEND)/alembic.ini heads"
+50
View File
@@ -0,0 +1,50 @@
# Alembic configuration file for StemEngine backend.
#
# This file is referenced by `alembic upgrade head` / `alembic revision --autogenerate`.
# The database URL is resolved at runtime from the DB_PATH environment variable
# (see alembic/env.py) so this file does NOT contain a hard-coded URL.
[alembic]
# Path to the migration scripts directory, relative to this file.
script_location = alembic
# Template for new revision files.
file_template = %%(rev)s_%%(slug)s
# Timezone for timestamps in new revision files.
timezone = UTC
# Logging
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
+108
View File
@@ -0,0 +1,108 @@
"""Alembic environment configuration for StemEngine.
This module is invoked by ``alembic`` when running migration commands.
It wires the StemEngine SQLAlchemy models and DB URL into the Alembic
migration context so that:
* ``alembic upgrade head`` — applies pending migrations
* ``alembic revision --autogenerate`` — generates a new migration by diffing
models against the live DB schema
Usage (from the ``backend/`` directory)::
alembic -c alembic.ini upgrade head
alembic -c alembic.ini revision --autogenerate -m "describe the change"
Or from the project root (``docker/appdata/stemengine/``)::
alembic -c backend/alembic.ini upgrade head
"""
from __future__ import annotations
import os
import sys
from logging.config import fileConfig
from pathlib import Path
from alembic import context
from sqlalchemy import engine_from_config, pool
# ---------------------------------------------------------------------------
# Make the backend package importable regardless of CWD.
# ---------------------------------------------------------------------------
_backend_parent = Path(__file__).resolve().parents[1] # stemengine/
if str(_backend_parent) not in sys.path:
sys.path.insert(0, str(_backend_parent))
# ---------------------------------------------------------------------------
# Import application metadata so Alembic can diff the ORM models.
# ---------------------------------------------------------------------------
from backend.db import Base, engine as _app_engine # noqa: E402
# ---------------------------------------------------------------------------
# Alembic Config object — gives access to alembic.ini values.
# ---------------------------------------------------------------------------
config = context.config
# Apply logging from alembic.ini (if present; may be absent in programmatic use).
if config.config_file_name is not None:
fileConfig(config.config_file_name)
# Expose the ORM metadata so --autogenerate can compare model vs. DB schema.
target_metadata = Base.metadata
# ---------------------------------------------------------------------------
# DB URL resolution
# ---------------------------------------------------------------------------
def _get_url() -> str:
"""Return the SQLAlchemy URL for the live database.
Prefers the running app's already-configured engine URL so that env-var
overrides (DB_PATH, MEDIA_DIR) are honoured automatically.
"""
return str(_app_engine.url)
# ---------------------------------------------------------------------------
# Migration runners
# ---------------------------------------------------------------------------
def run_migrations_offline() -> None:
"""Run migrations in 'offline' mode (emit SQL without connecting).
This is useful for generating migration SQL scripts that can be reviewed
and applied manually.
"""
url = _get_url()
context.configure(
url=url,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
render_as_batch=True, # required for SQLite ALTER TABLE support
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online() -> None:
"""Run migrations in 'online' mode (apply directly to the DB)."""
# Re-use the app engine so we share WAL mode / pragma settings.
connectable = _app_engine
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
render_as_batch=True, # required for SQLite ALTER TABLE support
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
+27
View File
@@ -0,0 +1,27 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from __future__ import annotations
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision: str = ${repr(up_revision)}
down_revision: Union[str, None] = ${repr(down_revision)}
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
@@ -0,0 +1,88 @@
"""Initial schema baseline.
Represents the full current schema as of 2026-03-14. This migration creates
all tables and indexes from scratch. It is intended as a *baseline* — it
records the existing state so that future migrations can be auto-generated
with ``alembic revision --autogenerate``.
On a fresh database, ``alembic upgrade head`` will apply this migration and
produce the full schema. On an existing database that was bootstrapped by the
legacy ``init_db()`` function, you should stamp the current revision as applied
without re-running DDL::
alembic -c backend/alembic.ini stamp 0001
Revision ID: 0001
Revises:
Create Date: 2026-03-14
"""
from __future__ import annotations
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "0001"
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ------------------------------------------------------------------
# jobs table
# ------------------------------------------------------------------
op.create_table(
"jobs",
sa.Column("id", sa.String, primary_key=True),
sa.Column("job_type", sa.String, nullable=False),
sa.Column("status", sa.String, server_default="queued"),
sa.Column("progress", sa.String, server_default=""),
sa.Column("error", sa.Text, server_default=""),
sa.Column("files", sa.JSON),
sa.Column("result", sa.JSON),
sa.Column("created_at", sa.DateTime),
sa.Column("updated_at", sa.DateTime),
)
# ------------------------------------------------------------------
# projects table
# ------------------------------------------------------------------
op.create_table(
"projects",
sa.Column("id", sa.String, primary_key=True),
sa.Column("name", sa.String, nullable=False),
sa.Column("tab_label", sa.String, server_default=""),
sa.Column("created_at", sa.DateTime),
sa.Column("sep_job_id", sa.String, nullable=False),
sa.Column("trans_jobs", sa.JSON),
sa.Column("abc_jobs", sa.JSON),
sa.Column("lrc_jobs", sa.JSON),
sa.Column("mix_job_id", sa.String, nullable=True),
)
# ------------------------------------------------------------------
# project_stem_job_links table (Phase 5 normalization)
# ------------------------------------------------------------------
op.create_table(
"project_stem_job_links",
sa.Column("id", sa.String, primary_key=True),
sa.Column("project_id", sa.String, nullable=False, index=True),
sa.Column("stage", sa.String, nullable=False, index=True),
sa.Column("stem_name", sa.String, nullable=False),
sa.Column("job_id", sa.String, nullable=False, index=True),
sa.Column("created_at", sa.DateTime),
sa.Column("updated_at", sa.DateTime),
)
op.create_unique_constraint(
"uq_project_stage_stem",
"project_stem_job_links",
["project_id", "stage", "stem_name"],
)
def downgrade() -> None:
op.drop_table("project_stem_job_links")
op.drop_table("projects")
op.drop_table("jobs")
Whitespace-only changes.
+208
View File
@@ -0,0 +1,208 @@
"""
Structured application settings for StemEngine.
Provides a single validated settings object that consolidates all
runtime configuration currently scattered across ``config.py``.
Usage::
from backend.app_settings import settings
# Access any setting
print(settings.app_env)
print(settings.admin_token)
print(settings.cors_origins_resolved)
# Startup validation (called by main.py lifespan)
errors = settings.validate_startup()
for err in errors:
log.warning("Startup warning: %s", err)
Design notes
------------
* ``config.py`` remains the single place that reads env vars and creates
directory constants. ``AppSettings`` references those values so there is
no double-parsing and no risk of divergence.
* ``validate_startup()`` performs lightweight checks (path writability) that
are too heavy for module-import time but important to surface early.
* The singleton ``settings`` is created at module load using the current
process environment, so it reflects the same values as ``config.py``.
"""
from __future__ import annotations
import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import List, Optional
@dataclass(frozen=True)
class AppSettings:
"""Immutable snapshot of runtime application configuration.
All attributes are derived from environment variables at construction time.
Use the module-level ``settings`` singleton rather than constructing this
directly.
"""
# -----------------------------------------------------------------
# Operational / security
# -----------------------------------------------------------------
app_env: str
"""Deployment environment tag. One of ``development``, ``production``, etc."""
admin_token: str
"""Secret token for high-power endpoints. Empty string disables enforcement."""
cors_origins: List[str]
"""Explicit CORS allowed-origin list (comma-separated env var). Empty means
use env-based defaults (wildcard in dev, none in prod)."""
# -----------------------------------------------------------------
# Filesystem paths
# -----------------------------------------------------------------
media_dir: Path
upload_dir: Path
jobs_dir: Path
projects_dir: Path
db_path: Path
lrc_sync_dir: Path
imports_dir: Path
settings_path: Path
# -----------------------------------------------------------------
# Upload limits
# -----------------------------------------------------------------
max_upload_bytes: int
"""Maximum upload file size in bytes."""
# -----------------------------------------------------------------
# Hardware
# -----------------------------------------------------------------
cpu_separation: bool
"""When True force CPU inference even if a GPU is present."""
# -----------------------------------------------------------------
# Properties
# -----------------------------------------------------------------
@property
def is_development(self) -> bool:
"""True when running in a local/development/test environment."""
return self.app_env in {"dev", "development", "local", "test"}
@property
def is_production(self) -> bool:
return not self.is_development
@property
def cors_origins_resolved(self) -> List[str]:
"""Compute the effective CORS allowed-origins list.
Priority:
1. ``CORS_ORIGINS`` env var (explicit list)
2. Wildcard (``["*"]``) when running in a dev/local environment
3. Empty list (deny all) for production when no list is configured
"""
if self.cors_origins:
return list(self.cors_origins)
if self.is_development:
return ["*"]
return []
@property
def admin_token_required(self) -> bool:
"""True when ``ADMIN_TOKEN`` is configured and enforcement is active."""
return bool(self.admin_token)
# -----------------------------------------------------------------
# Startup validation
# -----------------------------------------------------------------
def validate_startup(self) -> List[str]:
"""Perform lightweight startup health checks.
Returns a list of human-readable warning/error strings. An empty list
means all checks passed. Callers should log warnings but should *not*
abort startup for most of these — the readiness endpoint exposes live
status for orchestration probes.
"""
issues: List[str] = []
# Check all required directories are writable
probe_dirs = [
("media_dir", self.media_dir),
("upload_dir", self.upload_dir),
("jobs_dir", self.jobs_dir),
("projects_dir", self.projects_dir),
("lrc_sync_dir", self.lrc_sync_dir),
]
for name, path in probe_dirs:
if not path.exists():
try:
path.mkdir(parents=True, exist_ok=True)
except OSError as exc:
issues.append(f"{name} cannot be created: {exc}")
continue
probe = path / ".startup_probe"
try:
probe.write_text("ok")
probe.unlink()
except OSError as exc:
issues.append(f"{name} ({path}) is not writable: {exc}")
# Warn when production runs without an admin token
if self.is_production and not self.admin_token_required:
issues.append(
"Running in production without ADMIN_TOKEN — "
"high-power endpoints are unprotected."
)
# Warn about wildcard CORS in production
if self.is_production and "*" in self.cors_origins_resolved:
issues.append(
"Wildcard CORS origin is active in a non-development environment."
)
# Warn about missing DB parent directory
if not self.db_path.parent.exists():
try:
self.db_path.parent.mkdir(parents=True, exist_ok=True)
except OSError as exc:
issues.append(f"DB directory cannot be created: {exc}")
return issues
# -----------------------------------------------------------------
# Factory
# -----------------------------------------------------------------
@classmethod
def from_env(cls) -> "AppSettings":
"""Create an ``AppSettings`` instance from the current environment."""
# Import config.py to avoid double env-var parsing; config.py already
# read all env vars and created the canonical path objects.
from . import config as cfg
return cls(
app_env=cfg.APP_ENV,
admin_token=cfg.ADMIN_TOKEN,
cors_origins=list(cfg.CORS_ORIGINS),
media_dir=cfg.MEDIA_DIR,
upload_dir=cfg.UPLOAD_DIR,
jobs_dir=cfg.JOBS_DIR,
projects_dir=cfg.PROJECTS_DIR,
db_path=cfg.DB_PATH,
lrc_sync_dir=cfg.LRC_SYNC_DIR,
imports_dir=cfg.IMPORTS_DIR,
settings_path=cfg.SETTINGS_PATH,
max_upload_bytes=cfg.MAX_UPLOAD_BYTES,
cpu_separation=cfg.CPU_SEPARATION,
)
# ---------------------------------------------------------------------------
# Module-level singleton — use this in application code.
# ---------------------------------------------------------------------------
settings: AppSettings = AppSettings.from_env()
+53 -1
View File
@@ -139,7 +139,12 @@ def init_db(logger=None) -> None:
def _backfill_project_stem_job_links(logger=None) -> None:
"""Backfill normalized link rows from legacy JSON columns once.
"""Backfill normalized link rows from legacy JSON columns and scalar fields.
Covers:
- trans_jobs / abc_jobs / lrc_jobs JSON maps → stages transcribe / abc / lrc
- sep_job_id scalar string → stage sep (stem_name "__default__")
- mix_job_id scalar string → stage mix (stem_name "__default__")
Safe to run repeatedly due to unique constraint and insert-or-ignore behavior.
"""
@@ -180,6 +185,53 @@ def _backfill_project_stem_job_links(logger=None) -> None:
)
if result.rowcount:
inserted += 1
# Backfill sep_job_id → stage "sep", stem_name "__default__"
if project.sep_job_id:
stmt = text(
"INSERT OR IGNORE INTO project_stem_job_links "
"(id, project_id, stage, stem_name, job_id, created_at, updated_at) "
"VALUES (:id, :project_id, :stage, :stem_name, :job_id, :created_at, :updated_at)"
)
now = datetime.utcnow()
result = session.execute(
stmt,
{
"id": str(uuid4()),
"project_id": project.id,
"stage": "sep",
"stem_name": "__default__",
"job_id": str(project.sep_job_id),
"created_at": now,
"updated_at": now,
},
)
if result.rowcount:
inserted += 1
# Backfill mix_job_id → stage "mix", stem_name "__default__"
if project.mix_job_id:
stmt = text(
"INSERT OR IGNORE INTO project_stem_job_links "
"(id, project_id, stage, stem_name, job_id, created_at, updated_at) "
"VALUES (:id, :project_id, :stage, :stem_name, :job_id, :created_at, :updated_at)"
)
now = datetime.utcnow()
result = session.execute(
stmt,
{
"id": str(uuid4()),
"project_id": project.id,
"stage": "mix",
"stem_name": "__default__",
"job_id": str(project.mix_job_id),
"created_at": now,
"updated_at": now,
},
)
if result.rowcount:
inserted += 1
session.commit()
if logger and inserted:
+5 -9
View File
@@ -38,7 +38,8 @@ from fastapi.staticfiles import StaticFiles
from fastapi.responses import FileResponse
from .routers import separate, transcribe, abc_convert, pipeline, jobs, lrc, subsonic, projects, models, lyric_transcribe, system, settings
from .config import APP_ENV, CORS_ORIGINS, SUBSONIC_CONFIG_PATH
from .config import SUBSONIC_CONFIG_PATH
from .app_settings import settings as app_settings
from .db import init_db
_log = logging.getLogger(__name__)
@@ -70,6 +71,8 @@ def _purge_legacy_subsonic_config() -> None:
async def lifespan(app):
init_db(_log)
_purge_legacy_subsonic_config()
for issue in app_settings.validate_startup():
_log.warning("Startup check: %s", issue)
yield
@@ -116,16 +119,9 @@ def _mount_static(app: FastAPI) -> None:
def create_app() -> FastAPI:
app = FastAPI(title="StemEngine", version="1.0.0", lifespan=lifespan)
if CORS_ORIGINS:
allow_origins = CORS_ORIGINS
elif APP_ENV in {"dev", "development", "local", "test"}:
allow_origins = ["*"]
else:
allow_origins = []
app.add_middleware(
CORSMiddleware,
allow_origins=allow_origins,
allow_origins=app_settings.cors_origins_resolved,
allow_methods=["*"],
allow_headers=["*"],
)
+18 -3
View File
@@ -27,7 +27,10 @@ def find_or_create(name: str, sep_job_id: str, tab_label: str = '') -> dict:
tab_label=tab_label,
)
session.add(record)
session.commit()
session.flush() # populate record.id before upsert
# Normalize sep_job_id into the link table
_upsert_stem_job_link(session, record.id, "sep", "__default__", sep_job_id)
session.commit()
return _to_dict_with_links(session, record)
@@ -50,6 +53,11 @@ def update(project_id: str, **kwargs) -> bool:
return False
for key, value in kwargs.items():
setattr(record, key, value)
# Keep normalized link table in sync for scalar job-id fields
if "mix_job_id" in kwargs and kwargs["mix_job_id"]:
_upsert_stem_job_link(session, project_id, "mix", "__default__", kwargs["mix_job_id"])
if "sep_job_id" in kwargs and kwargs["sep_job_id"]:
_upsert_stem_job_link(session, project_id, "sep", "__default__", kwargs["sep_job_id"])
session.commit()
return True
@@ -163,6 +171,7 @@ def _to_dict_with_links(session, record: ProjectRecord | None) -> Optional[dict]
abc_jobs = _load_stage_map(session, record.id, "abc")
lrc_jobs = _load_stage_map(session, record.id, "lrc")
# Fall back to legacy JSON columns when normalized rows are absent
if not trans_jobs:
trans_jobs = record.trans_jobs or {}
if not abc_jobs:
@@ -170,16 +179,22 @@ def _to_dict_with_links(session, record: ProjectRecord | None) -> Optional[dict]
if not lrc_jobs:
lrc_jobs = record.lrc_jobs or {}
# sep / mix — prefer normalized table, fall back to scalar columns
sep_link = _load_stage_map(session, record.id, "sep")
mix_link = _load_stage_map(session, record.id, "mix")
sep_job_id = sep_link.get("__default__") or record.sep_job_id
mix_job_id = mix_link.get("__default__") or record.mix_job_id
return {
"id": record.id,
"name": record.name,
"tab_label": record.tab_label or '',
"created_at": record.created_at.isoformat() if record.created_at else None,
"sep_job_id": record.sep_job_id,
"sep_job_id": sep_job_id,
"trans_jobs": trans_jobs,
"abc_jobs": abc_jobs,
"lrc_jobs": lrc_jobs,
"mix_job_id": record.mix_job_id,
"mix_job_id": mix_job_id,
}
+34 -4
View File
@@ -1,20 +1,50 @@
"""Shared job status and download endpoints used by all tools."""
from pathlib import Path
from typing import Any, Dict, List, Optional
from fastapi import APIRouter, HTTPException
from fastapi.responses import FileResponse
from pydantic import BaseModel, ConfigDict
from .. import job_manager as jm
router = APIRouter(prefix="/api/jobs", tags=["jobs"])
@router.get("")
# ---------------------------------------------------------------------------
# Response models
# ---------------------------------------------------------------------------
class JobResponse(BaseModel):
"""Shape of a single job record returned by the API."""
model_config = ConfigDict(extra="allow") # forward-compatible: new fields pass through
job_id: str
job_type: str
status: str
progress: str
error: str
files: List[Dict[str, Any]]
result: Optional[Dict[str, Any]] = None
created_at: Optional[str] = None
updated_at: Optional[str] = None
class CancelResponse(BaseModel):
detail: str
# ---------------------------------------------------------------------------
# Routes
# ---------------------------------------------------------------------------
@router.get("", response_model=List[Dict[str, Any]])
def list_jobs():
return jm.list_jobs()
@router.get("/{job_id}")
@router.get("/{job_id}", response_model=Dict[str, Any])
def get_job(job_id: str):
job = jm.get_job(job_id)
if job is None:
@@ -22,13 +52,13 @@ def get_job(job_id: str):
return job
@router.delete("/{job_id}")
@router.delete("/{job_id}", response_model=CancelResponse)
def cancel_job(job_id: str):
job = jm.get_job(job_id)
if job is None:
raise HTTPException(status_code=404, detail="Job not found")
jm.cancel_job(job_id)
return {"detail": "Cancellation requested"}
return CancelResponse(detail="Cancellation requested")
@router.get("/{job_id}/download/{filename}")
+8 -2
View File
@@ -2,6 +2,7 @@
import shutil
from fastapi import APIRouter, File, Form, UploadFile, HTTPException
from pydantic import BaseModel
from .. import job_manager as jm
from ..config import (
@@ -16,7 +17,12 @@ VALID_FORMATS = {f["id"] for f in OUTPUT_FORMATS}
VALID_MODES = {m["id"] for m in TRANSCRIBE_MODES}
@router.post("")
class PipelineJobResponse(BaseModel):
"""Response returned when a pipeline job is successfully enqueued."""
job_id: str
@router.post("", response_model=PipelineJobResponse)
async def pipeline(
file: UploadFile = File(...),
sep_model: str = Form(DEFAULT_MODEL),
@@ -50,4 +56,4 @@ async def pipeline(
transcribe_mode,
effective_title,
)
return {"job_id": job_id}
return PipelineJobResponse(job_id=job_id)
+31 -2
View File
@@ -13,8 +13,8 @@ import json
import os
from typing import Any, Optional
from fastapi import APIRouter
from pydantic import BaseModel
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel, field_validator
from ..config import SETTINGS_PATH
@@ -23,6 +23,17 @@ router = APIRouter(prefix="/api/settings", tags=["settings"])
# Keys that are redacted from GET responses (store-only, never echoed back)
_REDACTED = {"hf_token"}
# Valid enumerated values — used for validation
_VALID_SEP_POWER = {"standard", "high", "max", "ultra"}
_VALID_WHISPER_MODELS = {
"tiny", "tiny.en",
"base", "base.en",
"small", "small.en",
"medium", "medium.en",
"large", "large-v1", "large-v2", "large-v3",
"large-v3-turbo", "turbo",
}
# Default values for all known preference keys
_DEFAULTS: dict[str, Any] = {
"whisper_model": "large-v2",
@@ -91,6 +102,24 @@ class SettingsPatch(BaseModel):
auto_format: Optional[str] = None
auto_presets: Optional[list] = None
@field_validator("sep_power")
@classmethod
def _validate_sep_power(cls, v: Optional[str]) -> Optional[str]:
if v is not None and v not in _VALID_SEP_POWER:
raise ValueError(
f"Unknown sep_power '{v}'. Valid values: {sorted(_VALID_SEP_POWER)}"
)
return v
@field_validator("whisper_model")
@classmethod
def _validate_whisper_model(cls, v: Optional[str]) -> Optional[str]:
if v is not None and v not in _VALID_WHISPER_MODELS:
raise ValueError(
f"Unknown whisper_model '{v}'. Valid values: {sorted(_VALID_WHISPER_MODELS)}"
)
return v
@router.patch("")
def patch_settings(body: SettingsPatch):
+4
View File
@@ -1,5 +1,9 @@
"""Compatibility facade for project service modules split by concern."""
# Re-export jm so that tests patching backend.services.project_service.jm
# still find the attribute (the actual import lives in the sub-modules).
from .. import job_manager as jm # noqa: F401
from .project_assets_service import (
build_project_detail,
collect_related_job_ids,
+159
View File
@@ -0,0 +1,159 @@
"""
Shared execution helpers for all separator back-ends.
Extracts the three patterns repeated across every ``_run_*`` function in
``separator_service.py``:
1. ``flush_vram_cache()`` — clear PyTorch reserved VRAM before loading a model
2. ``move_stems_to_project()`` — move separator temp output into the project Stems
dir, copying audio metadata tags onto each file.
3. ``oom_retry_call()`` — call a separator's ``separate_into_parts()`` once,
and on CUDA OOM retry with a reduced batch/chunk param.
New model runners should use these instead of inlining the boilerplate.
"""
from __future__ import annotations
import logging
import shutil
from pathlib import Path
from typing import Any, Callable, Dict, Optional
log = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# VRAM helpers
# ---------------------------------------------------------------------------
def flush_vram_cache() -> None:
"""Release PyTorch's reserved-but-idle VRAM blocks.
Safe to call even when CUDA is not available or torch is not installed.
"""
try:
import gc
import torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
# ---------------------------------------------------------------------------
# Post-separation stem move
# ---------------------------------------------------------------------------
def move_stems_to_project(
stem_paths: Dict[str, str],
stems_dir: Path,
stem_prefix: str,
input_path: str,
*,
display_fn: Optional[Callable[[str], str]] = None,
) -> Dict[str, str]:
"""Move separator output files into ``stems_dir`` with canonical names.
Args:
stem_paths: ``{stem_name: temp_file_path}`` returned by the separator.
stems_dir: Destination directory (project's ``Stems/`` folder).
stem_prefix: Human-readable prefix (project name) for the output filename.
input_path: Original source audio — used as the metadata donor file.
display_fn: Optional callable ``(stem_name) -> display_label`` used for
the metadata ``stem_label`` tag. Defaults to
``stem_name.replace("_", " ").title()``.
Returns:
``{stem_name: abs_path_str}`` for the moved files.
"""
from ..metadata_util import copy_metadata
from .output_layout import stem_filename
if display_fn is None:
display_fn = lambda s: s.replace("_", " ").title() # noqa: E731
stem_files: Dict[str, str] = {}
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
dest = stems_dir / stem_filename(stem_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
try:
copy_metadata(input_path, str(dest), stem_label=display_fn(stem_name), copy_lyrics=True)
except Exception as exc:
log.debug("copy_metadata failed for %s: %s", dest.name, exc)
stem_files[stem_name.lower()] = str(dest)
return stem_files
# ---------------------------------------------------------------------------
# OOM-safe separator call
# ---------------------------------------------------------------------------
def oom_retry_call(
sep_fn: Callable[..., Dict[str, str]],
*args: Any,
job_id: str,
retry_kwarg: str,
retry_value: Any,
update_job_fn: Optional[Callable[..., None]] = None,
**kwargs: Any,
) -> Dict[str, str]:
"""Call ``sep_fn(*args, **kwargs)`` with one automatic OOM retry.
On a CUDA out-of-memory error the first attempt retries with
``retry_kwarg=retry_value`` (e.g. ``batch_size_override=1``). The
``job_id`` is only used for the progress update message.
Args:
sep_fn: The callable to invoke — typically
``sep.separate_into_parts``.
*args: Positional arguments forwarded to ``sep_fn``.
job_id: Used for the OOM progress message.
retry_kwarg: Keyword argument to override on retry (e.g.
``"batch_size_override"``).
retry_value: Value to use for ``retry_kwarg`` on retry.
update_job_fn: Callable ``(job_id, progress=...) -> None`` used to emit
the OOM retry message. Pass ``jm.update_job`` here.
**kwargs: Keyword arguments forwarded to ``sep_fn``.
Returns:
Whatever ``sep_fn`` returns on success.
Raises:
The original exception if the retry also fails, or if the first failure
is not an OOM error.
"""
for attempt in range(2):
try:
return sep_fn(*args, **kwargs)
except Exception as exc:
is_oom = (
"out of memory" in str(exc).lower()
or "OutOfMemory" in type(exc).__name__
)
if is_oom and attempt == 0:
if update_job_fn is not None:
update_job_fn(
job_id,
progress=f"GPU OOM — retrying with {retry_kwarg}={retry_value}...",
)
flush_vram_cache()
kwargs[retry_kwarg] = retry_value
continue
raise
# Should never reach here, but satisfy type checkers.
raise RuntimeError("oom_retry_call exhausted all attempts")
# ---------------------------------------------------------------------------
# Temp-dir cleanup helper
# ---------------------------------------------------------------------------
def cleanup_temp(temp_dir: Path) -> None:
"""Remove a temporary separation output directory, ignoring errors."""
try:
shutil.rmtree(temp_dir, ignore_errors=True)
except Exception:
pass
+174
View File
@@ -0,0 +1,174 @@
"""
Post-processing helpers for separated stems.
Handles the three preset types that can be applied after separation:
- Instrumental (vocals removed)
- Static (specific stems removed or muted)
- Dynamic (per-stem volume adjustment)
These were previously inlined inside ``run_separation()`` in
``separator_service.py``. New callers should use ``apply_presets()`` or the
individual helpers directly.
"""
from __future__ import annotations
import logging
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
log = logging.getLogger(__name__)
def apply_presets(
*,
preset_type: str,
static_option: Optional[str],
dynamic_levels: Optional[Dict[str, float]],
stem_files: Dict[str, str],
dirs: Dict[str, Path],
proj_name: str,
input_path: str,
normalised_format: str,
job_id: str,
update_job_fn: Callable[..., None],
) -> List[Dict[str, Any]]:
"""Apply the requested preset to already-separated stems.
Returns a list of file-entry dicts (same shape as the ``files`` list in job
records) for any preset outputs that were produced. Returns an empty list
when ``preset_type`` is ``"stems_only"`` or the relevant directory is absent.
Args:
preset_type: ``"instrumental"``, ``"static"``, ``"dynamic"``, or
``"stems_only"``.
static_option: Static preset variant (e.g. ``"no_vocals"``).
dynamic_levels: Per-stem volume multipliers for the dynamic preset.
stem_files: ``{stem_name: abs_path}`` dict from the separator.
dirs: Project directory map returned by ``make_project_dirs``.
proj_name: Human-readable project name used for output filenames.
input_path: Original source file — used as metadata donor.
normalised_format: Output audio format (``"mp3"``, ``"flac"``, etc.).
job_id: Used only for progress updates.
update_job_fn: ``jm.update_job`` (or compatible callable).
Returns:
List of ``{"name": ..., "path": ..., "type": "instrumental"|"static"|"dynamic"}``
dicts for each file produced, or ``[]``.
"""
if preset_type == "instrumental" and "instrumental" in dirs:
return _apply_instrumental(
stem_files=stem_files,
out_dir=dirs["instrumental"],
proj_name=proj_name,
input_path=input_path,
normalised_format=normalised_format,
job_id=job_id,
update_job_fn=update_job_fn,
)
if preset_type == "static" and "static" in dirs:
return _apply_static(
stem_files=stem_files,
out_dir=dirs["static"],
proj_name=proj_name,
input_path=input_path,
normalised_format=normalised_format,
static_option=static_option,
job_id=job_id,
update_job_fn=update_job_fn,
)
if preset_type == "dynamic" and "dynamic" in dirs:
return _apply_dynamic(
stem_files=stem_files,
out_dir=dirs["dynamic"],
proj_name=proj_name,
input_path=input_path,
normalised_format=normalised_format,
dynamic_levels=dynamic_levels or {},
job_id=job_id,
update_job_fn=update_job_fn,
)
return []
# ---------------------------------------------------------------------------
# Individual preset helpers
# ---------------------------------------------------------------------------
def _apply_instrumental(
stem_files: Dict[str, str],
out_dir: Path,
proj_name: str,
input_path: str,
normalised_format: str,
job_id: str,
update_job_fn: Callable[..., None],
) -> List[Dict[str, Any]]:
from ..preset_service import make_instrumental
from ..metadata_util import copy_metadata
update_job_fn(job_id, progress="Creating instrumental mix...")
inst_path = out_dir / f"{proj_name} - Instrumental.{normalised_format}"
try:
result = make_instrumental(stem_files, inst_path, normalised_format)
copy_metadata(input_path, result, stem_label="Instrumental", copy_lyrics=True)
return [{"name": inst_path.name, "path": result, "type": "instrumental"}]
except Exception as exc:
update_job_fn(job_id, progress=f"Instrumental preset failed: {exc}")
log.warning("[%s] instrumental preset failed: %s", job_id, exc)
return []
def _apply_static(
stem_files: Dict[str, str],
out_dir: Path,
proj_name: str,
input_path: str,
normalised_format: str,
static_option: Optional[str],
job_id: str,
update_job_fn: Callable[..., None],
) -> List[Dict[str, Any]]:
from ..preset_service import make_static
from ..metadata_util import copy_metadata
from .output_layout import get_muted_stems, static_name as _static_name
update_job_fn(job_id, progress="Creating static mix...")
muted = get_muted_stems(static_option)
label = _static_name(static_option)
static_path = out_dir / f"{proj_name} - {label}.{normalised_format}"
try:
result = make_static(stem_files, muted, static_path, normalised_format)
copy_metadata(input_path, result, stem_label=label, copy_lyrics=True)
return [{"name": static_path.name, "path": result, "type": "static"}]
except Exception as exc:
update_job_fn(job_id, progress=f"Static preset failed: {exc}")
log.warning("[%s] static preset failed: %s", job_id, exc)
return []
def _apply_dynamic(
stem_files: Dict[str, str],
out_dir: Path,
proj_name: str,
input_path: str,
normalised_format: str,
dynamic_levels: Dict[str, float],
job_id: str,
update_job_fn: Callable[..., None],
) -> List[Dict[str, Any]]:
from ..preset_service import make_dynamic
from ..metadata_util import copy_metadata
update_job_fn(job_id, progress="Creating dynamic mix...")
dynamic_path = out_dir / f"{proj_name} - Dynamic Mix.{normalised_format}"
try:
result = make_dynamic(stem_files, dynamic_levels, dynamic_path, normalised_format)
copy_metadata(input_path, result, stem_label="Dynamic Mix", copy_lyrics=True)
return [{"name": dynamic_path.name, "path": result, "type": "dynamic"}]
except Exception as exc:
update_job_fn(job_id, progress=f"Dynamic preset failed: {exc}")
log.warning("[%s] dynamic preset failed: %s", job_id, exc)
return []
+21 -61
View File
@@ -17,7 +17,6 @@ from .. import job_manager as jm
from ..config import PROJECTS_DIR, JOBS_DIR, CPU_SEPARATION, PROJECT_SUBFOLDERS
from .metadata_util import copy_metadata, read_metadata
from .project_namer import project_name, make_project_dirs
from .preset_service import make_instrumental, make_static, make_dynamic
from .project_metadata import write_project_metadata
from .separation.model_downloads import ensure_model_available
from .separation.model_registry import (
@@ -50,6 +49,8 @@ from .separation.model_registry import (
)
from .separation.output_layout import stem_filename
from .separation.power_presets import get_power_opts
from .separation.execution import flush_vram_cache, move_stems_to_project, oom_retry_call, cleanup_temp
from .separation.postprocess import apply_presets
log = logging.getLogger(__name__)
@@ -161,13 +162,7 @@ def run_separation(
# the GPU's *reported* total, which leaves headroom for Windows.
# The flush must come first: if stale cached allocations are still counted against
# the cap from a prior job, the cap appears tighter than it actually is.
try:
import gc, torch
gc.collect()
if torch.cuda.is_available() and not CPU_SEPARATION:
torch.cuda.empty_cache()
except Exception:
pass
flush_vram_cache()
try:
from .lyric_transcriber_service import _apply_cuda_limits
_apply_cuda_limits()
@@ -339,53 +334,20 @@ def run_separation(
if jm.should_cancel(job_id):
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
return
# Apply presets if requested
preset_files = []
if preset_type == 'instrumental' and 'instrumental' in dirs:
jm.update_job(job_id, progress="Creating instrumental mix...")
inst_path = dirs['instrumental'] / f"{proj_name} - Instrumental.{normalised_format}"
try:
result = make_instrumental(stem_files, inst_path, output_format)
copy_metadata(input_path, result, stem_label="Instrumental", copy_lyrics=True)
preset_files.append({
"name": inst_path.name,
"path": result,
"type": "instrumental"
})
except Exception as e:
jm.update_job(job_id, progress=f"Instrumental preset failed: {e}")
elif preset_type == 'static' and 'static' in dirs:
jm.update_job(job_id, progress="Creating static mix...")
muted = _get_muted_stems(static_option)
static_name = _static_name(static_option)
static_path = dirs['static'] / f"{proj_name} - {static_name}.{normalised_format}"
try:
result = make_static(stem_files, muted, static_path, output_format)
copy_metadata(input_path, result, stem_label=static_name, copy_lyrics=True)
preset_files.append({
"name": static_path.name,
"path": result,
"type": "static"
})
except Exception as e:
jm.update_job(job_id, progress=f"Static preset failed: {e}")
elif preset_type == 'dynamic' and 'dynamic' in dirs:
jm.update_job(job_id, progress="Creating dynamic mix...")
volumes = dynamic_levels or {}
dynamic_path = dirs['dynamic'] / f"{proj_name} - Dynamic Mix.{normalised_format}"
try:
result = make_dynamic(stem_files, volumes, dynamic_path, output_format)
copy_metadata(input_path, result, stem_label="Dynamic Mix", copy_lyrics=True)
preset_files.append({
"name": dynamic_path.name,
"path": result,
"type": "dynamic"
})
except Exception as e:
jm.update_job(job_id, progress=f"Dynamic preset failed: {e}")
preset_files = apply_presets(
preset_type=preset_type,
static_option=static_option,
dynamic_levels=dynamic_levels,
stem_files=stem_files,
dirs=dirs,
proj_name=proj_name,
input_path=input_path,
normalised_format=normalised_format,
job_id=job_id,
update_job_fn=jm.update_job,
)
# Build result file list
all_files = []
@@ -489,19 +451,17 @@ def run_separation(
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=err_str, progress="Error.")
finally:
# Always flush PyTorch's VRAM cache after a job ends (success, error, or
# cancel) so the next job starts with maximum headroom. The model object
# goes out of scope here; gc.collect() x3 ensures its CUDA tensors are
# released even when PyTorch model objects form reference cycles that a
# single generational GC pass misses.
# cancel) so the next job starts with maximum headroom. gc.collect() x3
# ensures CUDA tensors are released even when model objects form reference
# cycles that a single generational GC pass misses.
try:
import gc, torch
import gc
gc.collect()
gc.collect()
gc.collect()
if torch.cuda.is_available() and not CPU_SEPARATION:
torch.cuda.empty_cache()
except Exception:
pass
flush_vram_cache()
# Release VRAM lock so the transcription executor can proceed.
try:
jm.gpu_vram_lock.release()
+1 -1
View File
@@ -19,7 +19,7 @@
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<meta name="apple-mobile-web-app-title" content="StemEngine" />
<link rel="apple-touch-icon" href="/icon.svg" />
<script type="module" crossorigin src="/assets/index-D7qgrvOb.js"></script>
<script type="module" crossorigin src="/assets/index-CHN4qgvq.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-7HDOX98b.css">
</head>
<body>
+34
View File
@@ -0,0 +1,34 @@
module.exports = {
root: true,
env: {
browser: true,
es2022: true,
node: true,
},
parser: '@typescript-eslint/parser',
parserOptions: {
ecmaVersion: 'latest',
sourceType: 'module',
ecmaFeatures: {
jsx: true,
},
},
plugins: ['@typescript-eslint', 'react-hooks', 'react-refresh'],
extends: [
'eslint:recommended',
'plugin:@typescript-eslint/recommended',
'plugin:react-hooks/recommended',
],
ignorePatterns: [
'dist',
'dist-local',
'node_modules',
],
rules: {
'react-refresh/only-export-components': ['warn', { allowConstantExport: true }],
'no-console': ['warn', { allow: ['warn', 'error'] }],
'@typescript-eslint/no-explicit-any': 'off',
// Guardrail for future growth; starts as warning to avoid blocking current work.
'max-lines': ['warn', { max: 900, skipBlankLines: true, skipComments: true }],
},
}
+1501
View File
File diff suppressed because it is too large. Load diff
+11 -1
View File
@@ -6,7 +6,12 @@
"scripts": {
"dev": "vite",
"build": "vite build",
"preview": "vite preview"
"build:local": "vite build --outDir dist-local",
"preview": "vite preview",
"typecheck": "tsc --noEmit",
"lint": "eslint \"src/**/*.{ts,tsx}\"",
"lint:fix": "eslint \"src/**/*.{ts,tsx}\" --fix",
"check": "npm run typecheck && npm run lint && npm run build:local"
},
"dependencies": {
"@fontsource/archivo": "^5.0.0",
@@ -28,10 +33,15 @@
"two.js": "^0.8.23"
},
"devDependencies": {
"@typescript-eslint/eslint-plugin": "^8.26.1",
"@typescript-eslint/parser": "^8.26.1",
"@types/react": "^18.2.37",
"@types/react-dom": "^18.2.15",
"@vitejs/plugin-react": "^4.2.0",
"autoprefixer": "^10.4.16",
"eslint": "^8.57.1",
"eslint-plugin-react-hooks": "^5.1.0",
"eslint-plugin-react-refresh": "^0.4.18",
"postcss": "^8.4.32",
"tailwindcss": "^3.3.6",
"typescript": "^5.9.3",
+50 -1
View File
@@ -19,10 +19,59 @@ import type {
const BASE = '/api'
const ADMIN_TOKEN_KEY = 'stemengine-admin-token'
export function getAdminToken(): string {
if (typeof window === 'undefined') return ''
return localStorage.getItem(ADMIN_TOKEN_KEY)?.trim() || ''
}
export function setAdminToken(token: string): void {
if (typeof window === 'undefined') return
const trimmed = token.trim()
if (!trimmed) {
localStorage.removeItem(ADMIN_TOKEN_KEY)
return
}
localStorage.setItem(ADMIN_TOKEN_KEY, trimmed)
}
export function clearAdminToken(): void {
if (typeof window === 'undefined') return
localStorage.removeItem(ADMIN_TOKEN_KEY)
}
function requiresAdminToken(url: string, method?: string): boolean {
const verb = (method ?? 'GET').toUpperCase()
// Normalize absolute URLs to pathnames; keep relative paths as-is.
let path = url
try {
path = new URL(url, window.location.origin).pathname
} catch {
// keep original string
}
if (path === `${BASE}/system/gpu-reset`) return true
if (path.startsWith(`${BASE}/projects/cleanup/`)) return true
if (path.startsWith(`${BASE}/models/`) && verb !== 'GET') return true
return false
}
async function request<T>(url: string, init?: RequestInit): Promise<T> {
const res = await fetch(url, init)
const headers = new Headers(init?.headers)
if (requiresAdminToken(url, init?.method)) {
const token = getAdminToken()
if (token) headers.set('X-Admin-Token', token)
}
const res = await fetch(url, { ...init, headers })
if (!res.ok) {
const text = await res.text()
if (res.status === 403 && requiresAdminToken(url, init?.method)) {
const detail = text || 'Forbidden'
throw new Error(`403: ${detail} (admin token required — set it in Settings → Integrations)`)
}
throw new Error(`${res.status}: ${text}`)
}
return res.json() as Promise<T>
-2
View File
@@ -467,8 +467,6 @@ export default function Separator({ onSeparationComplete, onOpenSettings }: Prop
filename: inputMode === 'upload'
? (file?.name ?? '')
: (importedFile?.filename ?? ''),
initialModel: model,
initialFormat: format,
initialProjectName: projectName || undefined,
}}
onClose={() => setShowAuto(false)}