backend: finalize phase updates, migrations, and integration wiring
This commit is contained in:
1 parent
51d277f502
commit
e5ca71f7c4
23 files changed
+2637
-87
No files matched your search
@@ -0,0 +1,52 @@
|
||||
# StemEngine backend — development helpers
|
||||
#
|
||||
# All commands run tests inside the built Docker image so they use the same
|
||||
# Python + dependency versions as production. Build the image first with
|
||||
# `make build` before running any test targets.
|
||||
|
||||
IMAGE := stemengine:latest
|
||||
BACKEND := backend
|
||||
|
||||
.PHONY: build test test-all test-smoke test-phase4 test-phase5 test-normalization alembic-upgrade alembic-heads help
|
||||
|
||||
help:
|
||||
@echo ""
|
||||
@echo " make build Build the Docker image"
|
||||
@echo " make test Run the full backend test suite"
|
||||
@echo " make test-smoke Smoke tests only (app boot + core routes)"
|
||||
@echo " make test-phase4 Phase 4 operational-hardening tests"
|
||||
@echo " make test-phase5 Phase 5 normalization tests"
|
||||
@echo " make test-normalization Alias for test-phase5"
|
||||
@echo " make alembic-upgrade Apply pending Alembic migrations inside container"
|
||||
@echo " make alembic-heads Show current Alembic revision heads"
|
||||
@echo ""
|
||||
|
||||
build:
|
||||
docker build -t $(IMAGE) .
|
||||
|
||||
# Full test suite — discovers all test_*.py under /app/backend/tests
|
||||
test:
|
||||
docker run --rm $(IMAGE) bash -lc \
|
||||
"python3.11 -m unittest discover -s /app/$(BACKEND)/tests -p 'test_*.py' -v"
|
||||
|
||||
# Subset targets for quick iteration
|
||||
test-smoke:
|
||||
docker run --rm $(IMAGE) bash -lc \
|
||||
"python3.11 -m unittest $(BACKEND).tests.test_app_smoke -v"
|
||||
|
||||
test-phase4:
|
||||
docker run --rm $(IMAGE) bash -lc \
|
||||
"python3.11 -m unittest $(BACKEND).tests.test_operational_hardening -v"
|
||||
|
||||
test-phase5 test-normalization:
|
||||
docker run --rm $(IMAGE) bash -lc \
|
||||
"python3.11 -m unittest $(BACKEND).tests.test_phase5_normalization -v"
|
||||
|
||||
# Alembic helpers — run against the container's bundled DB path
|
||||
alembic-upgrade:
|
||||
docker run --rm $(IMAGE) bash -lc \
|
||||
"cd /app && python3.11 -m alembic -c $(BACKEND)/alembic.ini upgrade head"
|
||||
|
||||
alembic-heads:
|
||||
docker run --rm $(IMAGE) bash -lc \
|
||||
"cd /app && python3.11 -m alembic -c $(BACKEND)/alembic.ini heads"
|
||||
@@ -0,0 +1,50 @@
|
||||
# Alembic configuration file for StemEngine backend.
|
||||
#
|
||||
# This file is referenced by `alembic upgrade head` / `alembic revision --autogenerate`.
|
||||
# The database URL is resolved at runtime from the DB_PATH environment variable
|
||||
# (see alembic/env.py) so this file does NOT contain a hard-coded URL.
|
||||
|
||||
[alembic]
|
||||
# Path to the migration scripts directory, relative to this file.
|
||||
script_location = alembic
|
||||
|
||||
# Template for new revision files.
|
||||
file_template = %%(rev)s_%%(slug)s
|
||||
|
||||
# Timezone for timestamps in new revision files.
|
||||
timezone = UTC
|
||||
|
||||
# Logging
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Alembic environment configuration for StemEngine.
|
||||
|
||||
This module is invoked by ``alembic`` when running migration commands.
|
||||
It wires the StemEngine SQLAlchemy models and DB URL into the Alembic
|
||||
migration context so that:
|
||||
|
||||
* ``alembic upgrade head`` — applies pending migrations
|
||||
* ``alembic revision --autogenerate`` — generates a new migration by diffing
|
||||
models against the live DB schema
|
||||
|
||||
Usage (from the ``backend/`` directory)::
|
||||
|
||||
alembic -c alembic.ini upgrade head
|
||||
alembic -c alembic.ini revision --autogenerate -m "describe the change"
|
||||
|
||||
Or from the project root (``docker/appdata/stemengine/``)::
|
||||
|
||||
alembic -c backend/alembic.ini upgrade head
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from logging.config import fileConfig
|
||||
from pathlib import Path
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Make the backend package importable regardless of CWD.
|
||||
# ---------------------------------------------------------------------------
|
||||
_backend_parent = Path(__file__).resolve().parents[1] # stemengine/
|
||||
if str(_backend_parent) not in sys.path:
|
||||
sys.path.insert(0, str(_backend_parent))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Import application metadata so Alembic can diff the ORM models.
|
||||
# ---------------------------------------------------------------------------
|
||||
from backend.db import Base, engine as _app_engine # noqa: E402
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Alembic Config object — gives access to alembic.ini values.
|
||||
# ---------------------------------------------------------------------------
|
||||
config = context.config
|
||||
|
||||
# Apply logging from alembic.ini (if present; may be absent in programmatic use).
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
# Expose the ORM metadata so --autogenerate can compare model vs. DB schema.
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DB URL resolution
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _get_url() -> str:
|
||||
"""Return the SQLAlchemy URL for the live database.
|
||||
|
||||
Prefers the running app's already-configured engine URL so that env-var
|
||||
overrides (DB_PATH, MEDIA_DIR) are honoured automatically.
|
||||
"""
|
||||
return str(_app_engine.url)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Migration runners
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
"""Run migrations in 'offline' mode (emit SQL without connecting).
|
||||
|
||||
This is useful for generating migration SQL scripts that can be reviewed
|
||||
and applied manually.
|
||||
"""
|
||||
url = _get_url()
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
render_as_batch=True, # required for SQLite ALTER TABLE support
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations in 'online' mode (apply directly to the DB)."""
|
||||
# Re-use the app engine so we share WAL mode / pragma settings.
|
||||
connectable = _app_engine
|
||||
|
||||
with connectable.connect() as connection:
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
render_as_batch=True, # required for SQLite ALTER TABLE support
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -0,0 +1,27 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Initial schema baseline.
|
||||
|
||||
Represents the full current schema as of 2026-03-14. This migration creates
|
||||
all tables and indexes from scratch. It is intended as a *baseline* — it
|
||||
records the existing state so that future migrations can be auto-generated
|
||||
with ``alembic revision --autogenerate``.
|
||||
|
||||
On a fresh database, ``alembic upgrade head`` will apply this migration and
|
||||
produce the full schema. On an existing database that was bootstrapped by the
|
||||
legacy ``init_db()`` function, you should stamp the current revision as applied
|
||||
without re-running DDL::
|
||||
|
||||
alembic -c backend/alembic.ini stamp 0001
|
||||
|
||||
Revision ID: 0001
|
||||
Revises:
|
||||
Create Date: 2026-03-14
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "0001"
|
||||
down_revision: Union[str, None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ------------------------------------------------------------------
|
||||
# jobs table
|
||||
# ------------------------------------------------------------------
|
||||
op.create_table(
|
||||
"jobs",
|
||||
sa.Column("id", sa.String, primary_key=True),
|
||||
sa.Column("job_type", sa.String, nullable=False),
|
||||
sa.Column("status", sa.String, server_default="queued"),
|
||||
sa.Column("progress", sa.String, server_default=""),
|
||||
sa.Column("error", sa.Text, server_default=""),
|
||||
sa.Column("files", sa.JSON),
|
||||
sa.Column("result", sa.JSON),
|
||||
sa.Column("created_at", sa.DateTime),
|
||||
sa.Column("updated_at", sa.DateTime),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# projects table
|
||||
# ------------------------------------------------------------------
|
||||
op.create_table(
|
||||
"projects",
|
||||
sa.Column("id", sa.String, primary_key=True),
|
||||
sa.Column("name", sa.String, nullable=False),
|
||||
sa.Column("tab_label", sa.String, server_default=""),
|
||||
sa.Column("created_at", sa.DateTime),
|
||||
sa.Column("sep_job_id", sa.String, nullable=False),
|
||||
sa.Column("trans_jobs", sa.JSON),
|
||||
sa.Column("abc_jobs", sa.JSON),
|
||||
sa.Column("lrc_jobs", sa.JSON),
|
||||
sa.Column("mix_job_id", sa.String, nullable=True),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# project_stem_job_links table (Phase 5 normalization)
|
||||
# ------------------------------------------------------------------
|
||||
op.create_table(
|
||||
"project_stem_job_links",
|
||||
sa.Column("id", sa.String, primary_key=True),
|
||||
sa.Column("project_id", sa.String, nullable=False, index=True),
|
||||
sa.Column("stage", sa.String, nullable=False, index=True),
|
||||
sa.Column("stem_name", sa.String, nullable=False),
|
||||
sa.Column("job_id", sa.String, nullable=False, index=True),
|
||||
sa.Column("created_at", sa.DateTime),
|
||||
sa.Column("updated_at", sa.DateTime),
|
||||
)
|
||||
op.create_unique_constraint(
|
||||
"uq_project_stage_stem",
|
||||
"project_stem_job_links",
|
||||
["project_id", "stage", "stem_name"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("project_stem_job_links")
|
||||
op.drop_table("projects")
|
||||
op.drop_table("jobs")
|
||||
Whitespace-only changes.
@@ -0,0 +1,208 @@
|
||||
"""
|
||||
Structured application settings for StemEngine.
|
||||
|
||||
Provides a single validated settings object that consolidates all
|
||||
runtime configuration currently scattered across ``config.py``.
|
||||
|
||||
Usage::
|
||||
|
||||
from backend.app_settings import settings
|
||||
|
||||
# Access any setting
|
||||
print(settings.app_env)
|
||||
print(settings.admin_token)
|
||||
print(settings.cors_origins_resolved)
|
||||
|
||||
# Startup validation (called by main.py lifespan)
|
||||
errors = settings.validate_startup()
|
||||
for err in errors:
|
||||
log.warning("Startup warning: %s", err)
|
||||
|
||||
Design notes
|
||||
------------
|
||||
* ``config.py`` remains the single place that reads env vars and creates
|
||||
directory constants. ``AppSettings`` references those values so there is
|
||||
no double-parsing and no risk of divergence.
|
||||
* ``validate_startup()`` performs lightweight checks (path writability) that
|
||||
are too heavy for module-import time but important to surface early.
|
||||
* The singleton ``settings`` is created at module load using the current
|
||||
process environment, so it reflects the same values as ``config.py``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AppSettings:
|
||||
"""Immutable snapshot of runtime application configuration.
|
||||
|
||||
All attributes are derived from environment variables at construction time.
|
||||
Use the module-level ``settings`` singleton rather than constructing this
|
||||
directly.
|
||||
"""
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Operational / security
|
||||
# -----------------------------------------------------------------
|
||||
app_env: str
|
||||
"""Deployment environment tag. One of ``development``, ``production``, etc."""
|
||||
|
||||
admin_token: str
|
||||
"""Secret token for high-power endpoints. Empty string disables enforcement."""
|
||||
|
||||
cors_origins: List[str]
|
||||
"""Explicit CORS allowed-origin list (comma-separated env var). Empty means
|
||||
use env-based defaults (wildcard in dev, none in prod)."""
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Filesystem paths
|
||||
# -----------------------------------------------------------------
|
||||
media_dir: Path
|
||||
upload_dir: Path
|
||||
jobs_dir: Path
|
||||
projects_dir: Path
|
||||
db_path: Path
|
||||
lrc_sync_dir: Path
|
||||
imports_dir: Path
|
||||
settings_path: Path
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Upload limits
|
||||
# -----------------------------------------------------------------
|
||||
max_upload_bytes: int
|
||||
"""Maximum upload file size in bytes."""
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Hardware
|
||||
# -----------------------------------------------------------------
|
||||
cpu_separation: bool
|
||||
"""When True force CPU inference even if a GPU is present."""
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Properties
|
||||
# -----------------------------------------------------------------
|
||||
|
||||
@property
|
||||
def is_development(self) -> bool:
|
||||
"""True when running in a local/development/test environment."""
|
||||
return self.app_env in {"dev", "development", "local", "test"}
|
||||
|
||||
@property
|
||||
def is_production(self) -> bool:
|
||||
return not self.is_development
|
||||
|
||||
@property
|
||||
def cors_origins_resolved(self) -> List[str]:
|
||||
"""Compute the effective CORS allowed-origins list.
|
||||
|
||||
Priority:
|
||||
1. ``CORS_ORIGINS`` env var (explicit list)
|
||||
2. Wildcard (``["*"]``) when running in a dev/local environment
|
||||
3. Empty list (deny all) for production when no list is configured
|
||||
"""
|
||||
if self.cors_origins:
|
||||
return list(self.cors_origins)
|
||||
if self.is_development:
|
||||
return ["*"]
|
||||
return []
|
||||
|
||||
@property
|
||||
def admin_token_required(self) -> bool:
|
||||
"""True when ``ADMIN_TOKEN`` is configured and enforcement is active."""
|
||||
return bool(self.admin_token)
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Startup validation
|
||||
# -----------------------------------------------------------------
|
||||
|
||||
def validate_startup(self) -> List[str]:
|
||||
"""Perform lightweight startup health checks.
|
||||
|
||||
Returns a list of human-readable warning/error strings. An empty list
|
||||
means all checks passed. Callers should log warnings but should *not*
|
||||
abort startup for most of these — the readiness endpoint exposes live
|
||||
status for orchestration probes.
|
||||
"""
|
||||
issues: List[str] = []
|
||||
|
||||
# Check all required directories are writable
|
||||
probe_dirs = [
|
||||
("media_dir", self.media_dir),
|
||||
("upload_dir", self.upload_dir),
|
||||
("jobs_dir", self.jobs_dir),
|
||||
("projects_dir", self.projects_dir),
|
||||
("lrc_sync_dir", self.lrc_sync_dir),
|
||||
]
|
||||
for name, path in probe_dirs:
|
||||
if not path.exists():
|
||||
try:
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as exc:
|
||||
issues.append(f"{name} cannot be created: {exc}")
|
||||
continue
|
||||
probe = path / ".startup_probe"
|
||||
try:
|
||||
probe.write_text("ok")
|
||||
probe.unlink()
|
||||
except OSError as exc:
|
||||
issues.append(f"{name} ({path}) is not writable: {exc}")
|
||||
|
||||
# Warn when production runs without an admin token
|
||||
if self.is_production and not self.admin_token_required:
|
||||
issues.append(
|
||||
"Running in production without ADMIN_TOKEN — "
|
||||
"high-power endpoints are unprotected."
|
||||
)
|
||||
|
||||
# Warn about wildcard CORS in production
|
||||
if self.is_production and "*" in self.cors_origins_resolved:
|
||||
issues.append(
|
||||
"Wildcard CORS origin is active in a non-development environment."
|
||||
)
|
||||
|
||||
# Warn about missing DB parent directory
|
||||
if not self.db_path.parent.exists():
|
||||
try:
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as exc:
|
||||
issues.append(f"DB directory cannot be created: {exc}")
|
||||
|
||||
return issues
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Factory
|
||||
# -----------------------------------------------------------------
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "AppSettings":
|
||||
"""Create an ``AppSettings`` instance from the current environment."""
|
||||
# Import config.py to avoid double env-var parsing; config.py already
|
||||
# read all env vars and created the canonical path objects.
|
||||
from . import config as cfg
|
||||
|
||||
return cls(
|
||||
app_env=cfg.APP_ENV,
|
||||
admin_token=cfg.ADMIN_TOKEN,
|
||||
cors_origins=list(cfg.CORS_ORIGINS),
|
||||
media_dir=cfg.MEDIA_DIR,
|
||||
upload_dir=cfg.UPLOAD_DIR,
|
||||
jobs_dir=cfg.JOBS_DIR,
|
||||
projects_dir=cfg.PROJECTS_DIR,
|
||||
db_path=cfg.DB_PATH,
|
||||
lrc_sync_dir=cfg.LRC_SYNC_DIR,
|
||||
imports_dir=cfg.IMPORTS_DIR,
|
||||
settings_path=cfg.SETTINGS_PATH,
|
||||
max_upload_bytes=cfg.MAX_UPLOAD_BYTES,
|
||||
cpu_separation=cfg.CPU_SEPARATION,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module-level singleton — use this in application code.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
settings: AppSettings = AppSettings.from_env()
|
||||
+53
-1
@@ -139,7 +139,12 @@ def init_db(logger=None) -> None:
|
||||
|
||||
|
||||
def _backfill_project_stem_job_links(logger=None) -> None:
|
||||
"""Backfill normalized link rows from legacy JSON columns once.
|
||||
"""Backfill normalized link rows from legacy JSON columns and scalar fields.
|
||||
|
||||
Covers:
|
||||
- trans_jobs / abc_jobs / lrc_jobs JSON maps → stages transcribe / abc / lrc
|
||||
- sep_job_id scalar string → stage sep (stem_name "__default__")
|
||||
- mix_job_id scalar string → stage mix (stem_name "__default__")
|
||||
|
||||
Safe to run repeatedly due to unique constraint and insert-or-ignore behavior.
|
||||
"""
|
||||
@@ -180,6 +185,53 @@ def _backfill_project_stem_job_links(logger=None) -> None:
|
||||
)
|
||||
if result.rowcount:
|
||||
inserted += 1
|
||||
|
||||
# Backfill sep_job_id → stage "sep", stem_name "__default__"
|
||||
if project.sep_job_id:
|
||||
stmt = text(
|
||||
"INSERT OR IGNORE INTO project_stem_job_links "
|
||||
"(id, project_id, stage, stem_name, job_id, created_at, updated_at) "
|
||||
"VALUES (:id, :project_id, :stage, :stem_name, :job_id, :created_at, :updated_at)"
|
||||
)
|
||||
now = datetime.utcnow()
|
||||
result = session.execute(
|
||||
stmt,
|
||||
{
|
||||
"id": str(uuid4()),
|
||||
"project_id": project.id,
|
||||
"stage": "sep",
|
||||
"stem_name": "__default__",
|
||||
"job_id": str(project.sep_job_id),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
},
|
||||
)
|
||||
if result.rowcount:
|
||||
inserted += 1
|
||||
|
||||
# Backfill mix_job_id → stage "mix", stem_name "__default__"
|
||||
if project.mix_job_id:
|
||||
stmt = text(
|
||||
"INSERT OR IGNORE INTO project_stem_job_links "
|
||||
"(id, project_id, stage, stem_name, job_id, created_at, updated_at) "
|
||||
"VALUES (:id, :project_id, :stage, :stem_name, :job_id, :created_at, :updated_at)"
|
||||
)
|
||||
now = datetime.utcnow()
|
||||
result = session.execute(
|
||||
stmt,
|
||||
{
|
||||
"id": str(uuid4()),
|
||||
"project_id": project.id,
|
||||
"stage": "mix",
|
||||
"stem_name": "__default__",
|
||||
"job_id": str(project.mix_job_id),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
},
|
||||
)
|
||||
if result.rowcount:
|
||||
inserted += 1
|
||||
|
||||
session.commit()
|
||||
|
||||
if logger and inserted:
|
||||
|
||||
+5
-9
@@ -38,7 +38,8 @@ from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from .routers import separate, transcribe, abc_convert, pipeline, jobs, lrc, subsonic, projects, models, lyric_transcribe, system, settings
|
||||
from .config import APP_ENV, CORS_ORIGINS, SUBSONIC_CONFIG_PATH
|
||||
from .config import SUBSONIC_CONFIG_PATH
|
||||
from .app_settings import settings as app_settings
|
||||
from .db import init_db
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
@@ -70,6 +71,8 @@ def _purge_legacy_subsonic_config() -> None:
|
||||
async def lifespan(app):
|
||||
init_db(_log)
|
||||
_purge_legacy_subsonic_config()
|
||||
for issue in app_settings.validate_startup():
|
||||
_log.warning("Startup check: %s", issue)
|
||||
yield
|
||||
|
||||
|
||||
@@ -116,16 +119,9 @@ def _mount_static(app: FastAPI) -> None:
|
||||
def create_app() -> FastAPI:
|
||||
app = FastAPI(title="StemEngine", version="1.0.0", lifespan=lifespan)
|
||||
|
||||
if CORS_ORIGINS:
|
||||
allow_origins = CORS_ORIGINS
|
||||
elif APP_ENV in {"dev", "development", "local", "test"}:
|
||||
allow_origins = ["*"]
|
||||
else:
|
||||
allow_origins = []
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=allow_origins,
|
||||
allow_origins=app_settings.cors_origins_resolved,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
@@ -27,7 +27,10 @@ def find_or_create(name: str, sep_job_id: str, tab_label: str = '') -> dict:
|
||||
tab_label=tab_label,
|
||||
)
|
||||
session.add(record)
|
||||
session.commit()
|
||||
session.flush() # populate record.id before upsert
|
||||
# Normalize sep_job_id into the link table
|
||||
_upsert_stem_job_link(session, record.id, "sep", "__default__", sep_job_id)
|
||||
session.commit()
|
||||
return _to_dict_with_links(session, record)
|
||||
|
||||
|
||||
@@ -50,6 +53,11 @@ def update(project_id: str, **kwargs) -> bool:
|
||||
return False
|
||||
for key, value in kwargs.items():
|
||||
setattr(record, key, value)
|
||||
# Keep normalized link table in sync for scalar job-id fields
|
||||
if "mix_job_id" in kwargs and kwargs["mix_job_id"]:
|
||||
_upsert_stem_job_link(session, project_id, "mix", "__default__", kwargs["mix_job_id"])
|
||||
if "sep_job_id" in kwargs and kwargs["sep_job_id"]:
|
||||
_upsert_stem_job_link(session, project_id, "sep", "__default__", kwargs["sep_job_id"])
|
||||
session.commit()
|
||||
return True
|
||||
|
||||
@@ -163,6 +171,7 @@ def _to_dict_with_links(session, record: ProjectRecord | None) -> Optional[dict]
|
||||
abc_jobs = _load_stage_map(session, record.id, "abc")
|
||||
lrc_jobs = _load_stage_map(session, record.id, "lrc")
|
||||
|
||||
# Fall back to legacy JSON columns when normalized rows are absent
|
||||
if not trans_jobs:
|
||||
trans_jobs = record.trans_jobs or {}
|
||||
if not abc_jobs:
|
||||
@@ -170,16 +179,22 @@ def _to_dict_with_links(session, record: ProjectRecord | None) -> Optional[dict]
|
||||
if not lrc_jobs:
|
||||
lrc_jobs = record.lrc_jobs or {}
|
||||
|
||||
# sep / mix — prefer normalized table, fall back to scalar columns
|
||||
sep_link = _load_stage_map(session, record.id, "sep")
|
||||
mix_link = _load_stage_map(session, record.id, "mix")
|
||||
sep_job_id = sep_link.get("__default__") or record.sep_job_id
|
||||
mix_job_id = mix_link.get("__default__") or record.mix_job_id
|
||||
|
||||
return {
|
||||
"id": record.id,
|
||||
"name": record.name,
|
||||
"tab_label": record.tab_label or '',
|
||||
"created_at": record.created_at.isoformat() if record.created_at else None,
|
||||
"sep_job_id": record.sep_job_id,
|
||||
"sep_job_id": sep_job_id,
|
||||
"trans_jobs": trans_jobs,
|
||||
"abc_jobs": abc_jobs,
|
||||
"lrc_jobs": lrc_jobs,
|
||||
"mix_job_id": record.mix_job_id,
|
||||
"mix_job_id": mix_job_id,
|
||||
}
|
||||
|
||||
|
||||
|
||||
+34
-4
@@ -1,20 +1,50 @@
|
||||
"""Shared job status and download endpoints used by all tools."""
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from .. import job_manager as jm
|
||||
|
||||
router = APIRouter(prefix="/api/jobs", tags=["jobs"])
|
||||
|
||||
|
||||
@router.get("")
|
||||
# ---------------------------------------------------------------------------
|
||||
# Response models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class JobResponse(BaseModel):
|
||||
"""Shape of a single job record returned by the API."""
|
||||
|
||||
model_config = ConfigDict(extra="allow") # forward-compatible: new fields pass through
|
||||
|
||||
job_id: str
|
||||
job_type: str
|
||||
status: str
|
||||
progress: str
|
||||
error: str
|
||||
files: List[Dict[str, Any]]
|
||||
result: Optional[Dict[str, Any]] = None
|
||||
created_at: Optional[str] = None
|
||||
updated_at: Optional[str] = None
|
||||
|
||||
|
||||
class CancelResponse(BaseModel):
|
||||
detail: str
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Routes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("", response_model=List[Dict[str, Any]])
|
||||
def list_jobs():
|
||||
return jm.list_jobs()
|
||||
|
||||
|
||||
@router.get("/{job_id}")
|
||||
@router.get("/{job_id}", response_model=Dict[str, Any])
|
||||
def get_job(job_id: str):
|
||||
job = jm.get_job(job_id)
|
||||
if job is None:
|
||||
@@ -22,13 +52,13 @@ def get_job(job_id: str):
|
||||
return job
|
||||
|
||||
|
||||
@router.delete("/{job_id}")
|
||||
@router.delete("/{job_id}", response_model=CancelResponse)
|
||||
def cancel_job(job_id: str):
|
||||
job = jm.get_job(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
jm.cancel_job(job_id)
|
||||
return {"detail": "Cancellation requested"}
|
||||
return CancelResponse(detail="Cancellation requested")
|
||||
|
||||
|
||||
@router.get("/{job_id}/download/{filename}")
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import shutil
|
||||
|
||||
from fastapi import APIRouter, File, Form, UploadFile, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .. import job_manager as jm
|
||||
from ..config import (
|
||||
@@ -16,7 +17,12 @@ VALID_FORMATS = {f["id"] for f in OUTPUT_FORMATS}
|
||||
VALID_MODES = {m["id"] for m in TRANSCRIBE_MODES}
|
||||
|
||||
|
||||
@router.post("")
|
||||
class PipelineJobResponse(BaseModel):
|
||||
"""Response returned when a pipeline job is successfully enqueued."""
|
||||
job_id: str
|
||||
|
||||
|
||||
@router.post("", response_model=PipelineJobResponse)
|
||||
async def pipeline(
|
||||
file: UploadFile = File(...),
|
||||
sep_model: str = Form(DEFAULT_MODEL),
|
||||
@@ -50,4 +56,4 @@ async def pipeline(
|
||||
transcribe_mode,
|
||||
effective_title,
|
||||
)
|
||||
return {"job_id": job_id}
|
||||
return PipelineJobResponse(job_id=job_id)
|
||||
@@ -13,8 +13,8 @@ import json
|
||||
import os
|
||||
from typing import Any, Optional
|
||||
|
||||
from fastapi import APIRouter
|
||||
from pydantic import BaseModel
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel, field_validator
|
||||
|
||||
from ..config import SETTINGS_PATH
|
||||
|
||||
@@ -23,6 +23,17 @@ router = APIRouter(prefix="/api/settings", tags=["settings"])
|
||||
# Keys that are redacted from GET responses (store-only, never echoed back)
|
||||
_REDACTED = {"hf_token"}
|
||||
|
||||
# Valid enumerated values — used for validation
|
||||
_VALID_SEP_POWER = {"standard", "high", "max", "ultra"}
|
||||
_VALID_WHISPER_MODELS = {
|
||||
"tiny", "tiny.en",
|
||||
"base", "base.en",
|
||||
"small", "small.en",
|
||||
"medium", "medium.en",
|
||||
"large", "large-v1", "large-v2", "large-v3",
|
||||
"large-v3-turbo", "turbo",
|
||||
}
|
||||
|
||||
# Default values for all known preference keys
|
||||
_DEFAULTS: dict[str, Any] = {
|
||||
"whisper_model": "large-v2",
|
||||
@@ -91,6 +102,24 @@ class SettingsPatch(BaseModel):
|
||||
auto_format: Optional[str] = None
|
||||
auto_presets: Optional[list] = None
|
||||
|
||||
@field_validator("sep_power")
|
||||
@classmethod
|
||||
def _validate_sep_power(cls, v: Optional[str]) -> Optional[str]:
|
||||
if v is not None and v not in _VALID_SEP_POWER:
|
||||
raise ValueError(
|
||||
f"Unknown sep_power '{v}'. Valid values: {sorted(_VALID_SEP_POWER)}"
|
||||
)
|
||||
return v
|
||||
|
||||
@field_validator("whisper_model")
|
||||
@classmethod
|
||||
def _validate_whisper_model(cls, v: Optional[str]) -> Optional[str]:
|
||||
if v is not None and v not in _VALID_WHISPER_MODELS:
|
||||
raise ValueError(
|
||||
f"Unknown whisper_model '{v}'. Valid values: {sorted(_VALID_WHISPER_MODELS)}"
|
||||
)
|
||||
return v
|
||||
|
||||
|
||||
@router.patch("")
|
||||
def patch_settings(body: SettingsPatch):
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
"""Compatibility facade for project service modules split by concern."""
|
||||
|
||||
# Re-export jm so that tests patching backend.services.project_service.jm
|
||||
# still find the attribute (the actual import lives in the sub-modules).
|
||||
from .. import job_manager as jm # noqa: F401
|
||||
|
||||
from .project_assets_service import (
|
||||
build_project_detail,
|
||||
collect_related_job_ids,
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
"""
|
||||
Shared execution helpers for all separator back-ends.
|
||||
|
||||
Extracts the three patterns repeated across every ``_run_*`` function in
|
||||
``separator_service.py``:
|
||||
|
||||
1. ``flush_vram_cache()`` — clear PyTorch reserved VRAM before loading a model
|
||||
2. ``move_stems_to_project()`` — move separator temp output into the project Stems
|
||||
dir, copying audio metadata tags onto each file.
|
||||
3. ``oom_retry_call()`` — call a separator's ``separate_into_parts()`` once,
|
||||
and on CUDA OOM retry with a reduced batch/chunk param.
|
||||
|
||||
New model runners should use these instead of inlining the boilerplate.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VRAM helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def flush_vram_cache() -> None:
|
||||
"""Release PyTorch's reserved-but-idle VRAM blocks.
|
||||
|
||||
Safe to call even when CUDA is not available or torch is not installed.
|
||||
"""
|
||||
try:
|
||||
import gc
|
||||
import torch
|
||||
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Post-separation stem move
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def move_stems_to_project(
|
||||
stem_paths: Dict[str, str],
|
||||
stems_dir: Path,
|
||||
stem_prefix: str,
|
||||
input_path: str,
|
||||
*,
|
||||
display_fn: Optional[Callable[[str], str]] = None,
|
||||
) -> Dict[str, str]:
|
||||
"""Move separator output files into ``stems_dir`` with canonical names.
|
||||
|
||||
Args:
|
||||
stem_paths: ``{stem_name: temp_file_path}`` returned by the separator.
|
||||
stems_dir: Destination directory (project's ``Stems/`` folder).
|
||||
stem_prefix: Human-readable prefix (project name) for the output filename.
|
||||
input_path: Original source audio — used as the metadata donor file.
|
||||
display_fn: Optional callable ``(stem_name) -> display_label`` used for
|
||||
the metadata ``stem_label`` tag. Defaults to
|
||||
``stem_name.replace("_", " ").title()``.
|
||||
|
||||
Returns:
|
||||
``{stem_name: abs_path_str}`` for the moved files.
|
||||
"""
|
||||
from ..metadata_util import copy_metadata
|
||||
from .output_layout import stem_filename
|
||||
|
||||
if display_fn is None:
|
||||
display_fn = lambda s: s.replace("_", " ").title() # noqa: E731
|
||||
|
||||
stem_files: Dict[str, str] = {}
|
||||
for stem_name, path in sorted(stem_paths.items()):
|
||||
src = Path(path)
|
||||
dest = stems_dir / stem_filename(stem_prefix, stem_name, src.suffix)
|
||||
shutil.move(str(src), str(dest))
|
||||
try:
|
||||
copy_metadata(input_path, str(dest), stem_label=display_fn(stem_name), copy_lyrics=True)
|
||||
except Exception as exc:
|
||||
log.debug("copy_metadata failed for %s: %s", dest.name, exc)
|
||||
stem_files[stem_name.lower()] = str(dest)
|
||||
return stem_files
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OOM-safe separator call
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def oom_retry_call(
|
||||
sep_fn: Callable[..., Dict[str, str]],
|
||||
*args: Any,
|
||||
job_id: str,
|
||||
retry_kwarg: str,
|
||||
retry_value: Any,
|
||||
update_job_fn: Optional[Callable[..., None]] = None,
|
||||
**kwargs: Any,
|
||||
) -> Dict[str, str]:
|
||||
"""Call ``sep_fn(*args, **kwargs)`` with one automatic OOM retry.
|
||||
|
||||
On a CUDA out-of-memory error the first attempt retries with
|
||||
``retry_kwarg=retry_value`` (e.g. ``batch_size_override=1``). The
|
||||
``job_id`` is only used for the progress update message.
|
||||
|
||||
Args:
|
||||
sep_fn: The callable to invoke — typically
|
||||
``sep.separate_into_parts``.
|
||||
*args: Positional arguments forwarded to ``sep_fn``.
|
||||
job_id: Used for the OOM progress message.
|
||||
retry_kwarg: Keyword argument to override on retry (e.g.
|
||||
``"batch_size_override"``).
|
||||
retry_value: Value to use for ``retry_kwarg`` on retry.
|
||||
update_job_fn: Callable ``(job_id, progress=...) -> None`` used to emit
|
||||
the OOM retry message. Pass ``jm.update_job`` here.
|
||||
**kwargs: Keyword arguments forwarded to ``sep_fn``.
|
||||
|
||||
Returns:
|
||||
Whatever ``sep_fn`` returns on success.
|
||||
|
||||
Raises:
|
||||
The original exception if the retry also fails, or if the first failure
|
||||
is not an OOM error.
|
||||
"""
|
||||
for attempt in range(2):
|
||||
try:
|
||||
return sep_fn(*args, **kwargs)
|
||||
except Exception as exc:
|
||||
is_oom = (
|
||||
"out of memory" in str(exc).lower()
|
||||
or "OutOfMemory" in type(exc).__name__
|
||||
)
|
||||
if is_oom and attempt == 0:
|
||||
if update_job_fn is not None:
|
||||
update_job_fn(
|
||||
job_id,
|
||||
progress=f"GPU OOM — retrying with {retry_kwarg}={retry_value}...",
|
||||
)
|
||||
flush_vram_cache()
|
||||
kwargs[retry_kwarg] = retry_value
|
||||
continue
|
||||
raise
|
||||
# Should never reach here, but satisfy type checkers.
|
||||
raise RuntimeError("oom_retry_call exhausted all attempts")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Temp-dir cleanup helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def cleanup_temp(temp_dir: Path) -> None:
|
||||
"""Remove a temporary separation output directory, ignoring errors."""
|
||||
try:
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,174 @@
|
||||
"""
|
||||
Post-processing helpers for separated stems.
|
||||
|
||||
Handles the three preset types that can be applied after separation:
|
||||
- Instrumental (vocals removed)
|
||||
- Static (specific stems removed or muted)
|
||||
- Dynamic (per-stem volume adjustment)
|
||||
|
||||
These were previously inlined inside ``run_separation()`` in
|
||||
``separator_service.py``. New callers should use ``apply_presets()`` or the
|
||||
individual helpers directly.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def apply_presets(
|
||||
*,
|
||||
preset_type: str,
|
||||
static_option: Optional[str],
|
||||
dynamic_levels: Optional[Dict[str, float]],
|
||||
stem_files: Dict[str, str],
|
||||
dirs: Dict[str, Path],
|
||||
proj_name: str,
|
||||
input_path: str,
|
||||
normalised_format: str,
|
||||
job_id: str,
|
||||
update_job_fn: Callable[..., None],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Apply the requested preset to already-separated stems.
|
||||
|
||||
Returns a list of file-entry dicts (same shape as the ``files`` list in job
|
||||
records) for any preset outputs that were produced. Returns an empty list
|
||||
when ``preset_type`` is ``"stems_only"`` or the relevant directory is absent.
|
||||
|
||||
Args:
|
||||
preset_type: ``"instrumental"``, ``"static"``, ``"dynamic"``, or
|
||||
``"stems_only"``.
|
||||
static_option: Static preset variant (e.g. ``"no_vocals"``).
|
||||
dynamic_levels: Per-stem volume multipliers for the dynamic preset.
|
||||
stem_files: ``{stem_name: abs_path}`` dict from the separator.
|
||||
dirs: Project directory map returned by ``make_project_dirs``.
|
||||
proj_name: Human-readable project name used for output filenames.
|
||||
input_path: Original source file — used as metadata donor.
|
||||
normalised_format: Output audio format (``"mp3"``, ``"flac"``, etc.).
|
||||
job_id: Used only for progress updates.
|
||||
update_job_fn: ``jm.update_job`` (or compatible callable).
|
||||
|
||||
Returns:
|
||||
List of ``{"name": ..., "path": ..., "type": "instrumental"|"static"|"dynamic"}``
|
||||
dicts for each file produced, or ``[]``.
|
||||
"""
|
||||
if preset_type == "instrumental" and "instrumental" in dirs:
|
||||
return _apply_instrumental(
|
||||
stem_files=stem_files,
|
||||
out_dir=dirs["instrumental"],
|
||||
proj_name=proj_name,
|
||||
input_path=input_path,
|
||||
normalised_format=normalised_format,
|
||||
job_id=job_id,
|
||||
update_job_fn=update_job_fn,
|
||||
)
|
||||
|
||||
if preset_type == "static" and "static" in dirs:
|
||||
return _apply_static(
|
||||
stem_files=stem_files,
|
||||
out_dir=dirs["static"],
|
||||
proj_name=proj_name,
|
||||
input_path=input_path,
|
||||
normalised_format=normalised_format,
|
||||
static_option=static_option,
|
||||
job_id=job_id,
|
||||
update_job_fn=update_job_fn,
|
||||
)
|
||||
|
||||
if preset_type == "dynamic" and "dynamic" in dirs:
|
||||
return _apply_dynamic(
|
||||
stem_files=stem_files,
|
||||
out_dir=dirs["dynamic"],
|
||||
proj_name=proj_name,
|
||||
input_path=input_path,
|
||||
normalised_format=normalised_format,
|
||||
dynamic_levels=dynamic_levels or {},
|
||||
job_id=job_id,
|
||||
update_job_fn=update_job_fn,
|
||||
)
|
||||
|
||||
return []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Individual preset helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _apply_instrumental(
|
||||
stem_files: Dict[str, str],
|
||||
out_dir: Path,
|
||||
proj_name: str,
|
||||
input_path: str,
|
||||
normalised_format: str,
|
||||
job_id: str,
|
||||
update_job_fn: Callable[..., None],
|
||||
) -> List[Dict[str, Any]]:
|
||||
from ..preset_service import make_instrumental
|
||||
from ..metadata_util import copy_metadata
|
||||
|
||||
update_job_fn(job_id, progress="Creating instrumental mix...")
|
||||
inst_path = out_dir / f"{proj_name} - Instrumental.{normalised_format}"
|
||||
try:
|
||||
result = make_instrumental(stem_files, inst_path, normalised_format)
|
||||
copy_metadata(input_path, result, stem_label="Instrumental", copy_lyrics=True)
|
||||
return [{"name": inst_path.name, "path": result, "type": "instrumental"}]
|
||||
except Exception as exc:
|
||||
update_job_fn(job_id, progress=f"Instrumental preset failed: {exc}")
|
||||
log.warning("[%s] instrumental preset failed: %s", job_id, exc)
|
||||
return []
|
||||
|
||||
|
||||
def _apply_static(
|
||||
stem_files: Dict[str, str],
|
||||
out_dir: Path,
|
||||
proj_name: str,
|
||||
input_path: str,
|
||||
normalised_format: str,
|
||||
static_option: Optional[str],
|
||||
job_id: str,
|
||||
update_job_fn: Callable[..., None],
|
||||
) -> List[Dict[str, Any]]:
|
||||
from ..preset_service import make_static
|
||||
from ..metadata_util import copy_metadata
|
||||
from .output_layout import get_muted_stems, static_name as _static_name
|
||||
|
||||
update_job_fn(job_id, progress="Creating static mix...")
|
||||
muted = get_muted_stems(static_option)
|
||||
label = _static_name(static_option)
|
||||
static_path = out_dir / f"{proj_name} - {label}.{normalised_format}"
|
||||
try:
|
||||
result = make_static(stem_files, muted, static_path, normalised_format)
|
||||
copy_metadata(input_path, result, stem_label=label, copy_lyrics=True)
|
||||
return [{"name": static_path.name, "path": result, "type": "static"}]
|
||||
except Exception as exc:
|
||||
update_job_fn(job_id, progress=f"Static preset failed: {exc}")
|
||||
log.warning("[%s] static preset failed: %s", job_id, exc)
|
||||
return []
|
||||
|
||||
|
||||
def _apply_dynamic(
|
||||
stem_files: Dict[str, str],
|
||||
out_dir: Path,
|
||||
proj_name: str,
|
||||
input_path: str,
|
||||
normalised_format: str,
|
||||
dynamic_levels: Dict[str, float],
|
||||
job_id: str,
|
||||
update_job_fn: Callable[..., None],
|
||||
) -> List[Dict[str, Any]]:
|
||||
from ..preset_service import make_dynamic
|
||||
from ..metadata_util import copy_metadata
|
||||
|
||||
update_job_fn(job_id, progress="Creating dynamic mix...")
|
||||
dynamic_path = out_dir / f"{proj_name} - Dynamic Mix.{normalised_format}"
|
||||
try:
|
||||
result = make_dynamic(stem_files, dynamic_levels, dynamic_path, normalised_format)
|
||||
copy_metadata(input_path, result, stem_label="Dynamic Mix", copy_lyrics=True)
|
||||
return [{"name": dynamic_path.name, "path": result, "type": "dynamic"}]
|
||||
except Exception as exc:
|
||||
update_job_fn(job_id, progress=f"Dynamic preset failed: {exc}")
|
||||
log.warning("[%s] dynamic preset failed: %s", job_id, exc)
|
||||
return []
|
||||
@@ -17,7 +17,6 @@ from .. import job_manager as jm
|
||||
from ..config import PROJECTS_DIR, JOBS_DIR, CPU_SEPARATION, PROJECT_SUBFOLDERS
|
||||
from .metadata_util import copy_metadata, read_metadata
|
||||
from .project_namer import project_name, make_project_dirs
|
||||
from .preset_service import make_instrumental, make_static, make_dynamic
|
||||
from .project_metadata import write_project_metadata
|
||||
from .separation.model_downloads import ensure_model_available
|
||||
from .separation.model_registry import (
|
||||
@@ -50,6 +49,8 @@ from .separation.model_registry import (
|
||||
)
|
||||
from .separation.output_layout import stem_filename
|
||||
from .separation.power_presets import get_power_opts
|
||||
from .separation.execution import flush_vram_cache, move_stems_to_project, oom_retry_call, cleanup_temp
|
||||
from .separation.postprocess import apply_presets
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -161,13 +162,7 @@ def run_separation(
|
||||
# the GPU's *reported* total, which leaves headroom for Windows.
|
||||
# The flush must come first: if stale cached allocations are still counted against
|
||||
# the cap from a prior job, the cap appears tighter than it actually is.
|
||||
try:
|
||||
import gc, torch
|
||||
gc.collect()
|
||||
if torch.cuda.is_available() and not CPU_SEPARATION:
|
||||
torch.cuda.empty_cache()
|
||||
except Exception:
|
||||
pass
|
||||
flush_vram_cache()
|
||||
try:
|
||||
from .lyric_transcriber_service import _apply_cuda_limits
|
||||
_apply_cuda_limits()
|
||||
@@ -339,53 +334,20 @@ def run_separation(
|
||||
if jm.should_cancel(job_id):
|
||||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||||
return
|
||||
|
||||
|
||||
# Apply presets if requested
|
||||
preset_files = []
|
||||
if preset_type == 'instrumental' and 'instrumental' in dirs:
|
||||
jm.update_job(job_id, progress="Creating instrumental mix...")
|
||||
inst_path = dirs['instrumental'] / f"{proj_name} - Instrumental.{normalised_format}"
|
||||
try:
|
||||
result = make_instrumental(stem_files, inst_path, output_format)
|
||||
copy_metadata(input_path, result, stem_label="Instrumental", copy_lyrics=True)
|
||||
preset_files.append({
|
||||
"name": inst_path.name,
|
||||
"path": result,
|
||||
"type": "instrumental"
|
||||
})
|
||||
except Exception as e:
|
||||
jm.update_job(job_id, progress=f"Instrumental preset failed: {e}")
|
||||
|
||||
elif preset_type == 'static' and 'static' in dirs:
|
||||
jm.update_job(job_id, progress="Creating static mix...")
|
||||
muted = _get_muted_stems(static_option)
|
||||
static_name = _static_name(static_option)
|
||||
static_path = dirs['static'] / f"{proj_name} - {static_name}.{normalised_format}"
|
||||
try:
|
||||
result = make_static(stem_files, muted, static_path, output_format)
|
||||
copy_metadata(input_path, result, stem_label=static_name, copy_lyrics=True)
|
||||
preset_files.append({
|
||||
"name": static_path.name,
|
||||
"path": result,
|
||||
"type": "static"
|
||||
})
|
||||
except Exception as e:
|
||||
jm.update_job(job_id, progress=f"Static preset failed: {e}")
|
||||
|
||||
elif preset_type == 'dynamic' and 'dynamic' in dirs:
|
||||
jm.update_job(job_id, progress="Creating dynamic mix...")
|
||||
volumes = dynamic_levels or {}
|
||||
dynamic_path = dirs['dynamic'] / f"{proj_name} - Dynamic Mix.{normalised_format}"
|
||||
try:
|
||||
result = make_dynamic(stem_files, volumes, dynamic_path, output_format)
|
||||
copy_metadata(input_path, result, stem_label="Dynamic Mix", copy_lyrics=True)
|
||||
preset_files.append({
|
||||
"name": dynamic_path.name,
|
||||
"path": result,
|
||||
"type": "dynamic"
|
||||
})
|
||||
except Exception as e:
|
||||
jm.update_job(job_id, progress=f"Dynamic preset failed: {e}")
|
||||
preset_files = apply_presets(
|
||||
preset_type=preset_type,
|
||||
static_option=static_option,
|
||||
dynamic_levels=dynamic_levels,
|
||||
stem_files=stem_files,
|
||||
dirs=dirs,
|
||||
proj_name=proj_name,
|
||||
input_path=input_path,
|
||||
normalised_format=normalised_format,
|
||||
job_id=job_id,
|
||||
update_job_fn=jm.update_job,
|
||||
)
|
||||
|
||||
# Build result file list
|
||||
all_files = []
|
||||
@@ -489,19 +451,17 @@ def run_separation(
|
||||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=err_str, progress="Error.")
|
||||
finally:
|
||||
# Always flush PyTorch's VRAM cache after a job ends (success, error, or
|
||||
# cancel) so the next job starts with maximum headroom. The model object
|
||||
# goes out of scope here; gc.collect() x3 ensures its CUDA tensors are
|
||||
# released even when PyTorch model objects form reference cycles that a
|
||||
# single generational GC pass misses.
|
||||
# cancel) so the next job starts with maximum headroom. gc.collect() x3
|
||||
# ensures CUDA tensors are released even when model objects form reference
|
||||
# cycles that a single generational GC pass misses.
|
||||
try:
|
||||
import gc, torch
|
||||
import gc
|
||||
gc.collect()
|
||||
gc.collect()
|
||||
gc.collect()
|
||||
if torch.cuda.is_available() and not CPU_SEPARATION:
|
||||
torch.cuda.empty_cache()
|
||||
except Exception:
|
||||
pass
|
||||
flush_vram_cache()
|
||||
# Release VRAM lock so the transcription executor can proceed.
|
||||
try:
|
||||
jm.gpu_vram_lock.release()
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
|
||||
<meta name="apple-mobile-web-app-title" content="StemEngine" />
|
||||
<link rel="apple-touch-icon" href="/icon.svg" />
|
||||
<script type="module" crossorigin src="/assets/index-D7qgrvOb.js"></script>
|
||||
<script type="module" crossorigin src="/assets/index-CHN4qgvq.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-7HDOX98b.css">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
module.exports = {
|
||||
root: true,
|
||||
env: {
|
||||
browser: true,
|
||||
es2022: true,
|
||||
node: true,
|
||||
},
|
||||
parser: '@typescript-eslint/parser',
|
||||
parserOptions: {
|
||||
ecmaVersion: 'latest',
|
||||
sourceType: 'module',
|
||||
ecmaFeatures: {
|
||||
jsx: true,
|
||||
},
|
||||
},
|
||||
plugins: ['@typescript-eslint', 'react-hooks', 'react-refresh'],
|
||||
extends: [
|
||||
'eslint:recommended',
|
||||
'plugin:@typescript-eslint/recommended',
|
||||
'plugin:react-hooks/recommended',
|
||||
],
|
||||
ignorePatterns: [
|
||||
'dist',
|
||||
'dist-local',
|
||||
'node_modules',
|
||||
],
|
||||
rules: {
|
||||
'react-refresh/only-export-components': ['warn', { allowConstantExport: true }],
|
||||
'no-console': ['warn', { allow: ['warn', 'error'] }],
|
||||
'@typescript-eslint/no-explicit-any': 'off',
|
||||
// Guardrail for future growth; starts as warning to avoid blocking current work.
|
||||
'max-lines': ['warn', { max: 900, skipBlankLines: true, skipComments: true }],
|
||||
},
|
||||
}
|
||||
Generated
+1501
File diff suppressed because it is too large.
Load diff
+11
-1
@@ -6,7 +6,12 @@
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
"preview": "vite preview"
|
||||
"build:local": "vite build --outDir dist-local",
|
||||
"preview": "vite preview",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"lint": "eslint \"src/**/*.{ts,tsx}\"",
|
||||
"lint:fix": "eslint \"src/**/*.{ts,tsx}\" --fix",
|
||||
"check": "npm run typecheck && npm run lint && npm run build:local"
|
||||
},
|
||||
"dependencies": {
|
||||
"@fontsource/archivo": "^5.0.0",
|
||||
@@ -28,10 +33,15 @@
|
||||
"two.js": "^0.8.23"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@typescript-eslint/eslint-plugin": "^8.26.1",
|
||||
"@typescript-eslint/parser": "^8.26.1",
|
||||
"@types/react": "^18.2.37",
|
||||
"@types/react-dom": "^18.2.15",
|
||||
"@vitejs/plugin-react": "^4.2.0",
|
||||
"autoprefixer": "^10.4.16",
|
||||
"eslint": "^8.57.1",
|
||||
"eslint-plugin-react-hooks": "^5.1.0",
|
||||
"eslint-plugin-react-refresh": "^0.4.18",
|
||||
"postcss": "^8.4.32",
|
||||
"tailwindcss": "^3.3.6",
|
||||
"typescript": "^5.9.3",
|
||||
|
||||
@@ -19,10 +19,59 @@ import type {
|
||||
|
||||
const BASE = '/api'
|
||||
|
||||
const ADMIN_TOKEN_KEY = 'stemengine-admin-token'
|
||||
|
||||
export function getAdminToken(): string {
|
||||
if (typeof window === 'undefined') return ''
|
||||
return localStorage.getItem(ADMIN_TOKEN_KEY)?.trim() || ''
|
||||
}
|
||||
|
||||
export function setAdminToken(token: string): void {
|
||||
if (typeof window === 'undefined') return
|
||||
const trimmed = token.trim()
|
||||
if (!trimmed) {
|
||||
localStorage.removeItem(ADMIN_TOKEN_KEY)
|
||||
return
|
||||
}
|
||||
localStorage.setItem(ADMIN_TOKEN_KEY, trimmed)
|
||||
}
|
||||
|
||||
export function clearAdminToken(): void {
|
||||
if (typeof window === 'undefined') return
|
||||
localStorage.removeItem(ADMIN_TOKEN_KEY)
|
||||
}
|
||||
|
||||
function requiresAdminToken(url: string, method?: string): boolean {
|
||||
const verb = (method ?? 'GET').toUpperCase()
|
||||
|
||||
// Normalize absolute URLs to pathnames; keep relative paths as-is.
|
||||
let path = url
|
||||
try {
|
||||
path = new URL(url, window.location.origin).pathname
|
||||
} catch {
|
||||
// keep original string
|
||||
}
|
||||
|
||||
if (path === `${BASE}/system/gpu-reset`) return true
|
||||
if (path.startsWith(`${BASE}/projects/cleanup/`)) return true
|
||||
if (path.startsWith(`${BASE}/models/`) && verb !== 'GET') return true
|
||||
return false
|
||||
}
|
||||
|
||||
async function request<T>(url: string, init?: RequestInit): Promise<T> {
|
||||
const res = await fetch(url, init)
|
||||
const headers = new Headers(init?.headers)
|
||||
if (requiresAdminToken(url, init?.method)) {
|
||||
const token = getAdminToken()
|
||||
if (token) headers.set('X-Admin-Token', token)
|
||||
}
|
||||
|
||||
const res = await fetch(url, { ...init, headers })
|
||||
if (!res.ok) {
|
||||
const text = await res.text()
|
||||
if (res.status === 403 && requiresAdminToken(url, init?.method)) {
|
||||
const detail = text || 'Forbidden'
|
||||
throw new Error(`403: ${detail} (admin token required — set it in Settings → Integrations)`)
|
||||
}
|
||||
throw new Error(`${res.status}: ${text}`)
|
||||
}
|
||||
return res.json() as Promise<T>
|
||||
|
||||
@@ -467,8 +467,6 @@ export default function Separator({ onSeparationComplete, onOpenSettings }: Prop
|
||||
filename: inputMode === 'upload'
|
||||
? (file?.name ?? '')
|
||||
: (importedFile?.filename ?? ''),
|
||||
initialModel: model,
|
||||
initialFormat: format,
|
||||
initialProjectName: projectName || undefined,
|
||||
}}
|
||||
onClose={() => setShowAuto(false)}
|
||||
|
||||
Reference in new issue
Block a user