From e11f46527b5c1dc6d99358769d8b811540ce5774 Mon Sep 17 00:00:00 2001 From: translator Date: Thu, 9 Jul 2026 10:58:09 +0900 Subject: [PATCH] Myriad changes. --- .env.example | 13 + 20260313_Fixes (Copilot).md | 686 ++++++++++++++++ README.md | 28 + SUMMARIES (Claude).md | 407 ++++++++++ backend/main.py | 15 +- backend/requirements.txt | 3 +- backend/routers/midi_editor.py | 60 ++ backend/routers/projects.py | 3 +- backend/routers/sloppak_converter.py | 87 ++ backend/routers/transcribe.py | 12 +- backend/services/midi_editor_service.py | 311 ++++++++ backend/services/omnizart_service.py | 149 +++- backend/services/project_reprocess_service.py | 4 +- backend/services/separator_service.py | 18 +- backend/services/sloppak_converter_service.py | 751 ++++++++++++++++++ backend/services/transcriber_service.py | 141 +++- ...Windows_x64_Installable_Executable_Plan.md | 279 +++++++ ..._Image_Size_Analysis_And_Reduction_Plan.md | 154 ++++ ..._SlopPak_And_PitchBend_Integration_Plan.md | 488 ++++++++++++ frontend/src/App.tsx | 3 + frontend/src/api/client.ts | 153 +++- frontend/src/components/MidiLightEditor.tsx | 282 +++++++ frontend/src/components/Sidebar.tsx | 20 +- .../src/components/SloppakConverterPanel.tsx | 241 ++++++ frontend/src/pages/Conversions.tsx | 16 + frontend/src/pages/ProjectDetail.tsx | 152 +++- frontend/src/pages/Settings.tsx | 91 +++ frontend/src/routes.ts | 1 + frontend/src/types.ts | 12 +- stemengine.yml | 6 + 30 files changed, 4535 insertions(+), 51 deletions(-) create mode 100644 backend/routers/midi_editor.py create mode 100644 backend/routers/sloppak_converter.py create mode 100644 backend/services/midi_editor_service.py create mode 100644 backend/services/sloppak_converter_service.py create mode 100644 docs/plans/2026-07-08_StemEngine_Windows_x64_Installable_Executable_Plan.md create mode 100644 docs/plans/2026-07-09_StemEngine_Image_Size_Analysis_And_Reduction_Plan.md create mode 100644 docs/plans/2026-07-09_StemEngine_PSARC_SlopPak_And_PitchBend_Integration_Plan.md create mode 100644 frontend/src/components/MidiLightEditor.tsx create mode 100644 frontend/src/components/SloppakConverterPanel.tsx create mode 100644 frontend/src/pages/Conversions.tsx diff --git a/.env.example b/.env.example index 5c31058..054b8c3 100644 --- a/.env.example +++ b/.env.example @@ -8,3 +8,16 @@ MUON_IFRAME_URL=http://work:3011 # Pinned Muon fork commit/tag/branch used by muon-iframe build context. # Examples: main, v0.1.0, 1a2b3c4d MUON_GIT_REF=main + +# Optional: PSARC -> sloppak converter paths inside container. +# Defaults are usually fine. +SLOPPAK_SOURCE_ROOT=/app/media/uploads +SLOPPAK_OUTPUT_ROOT=/app/media/sloppak + +# Optional: command template used to unpack PSARC when python patcher module +# is not present. Must include both placeholders: +# {input} -> absolute source .psarc path +# {output} -> absolute extraction directory path +# Example (replace with your actual unpacker executable): +# SLOPPAK_PSARC_UNPACK_CMD=/usr/local/bin/psarc unpack {input} {output} +SLOPPAK_PSARC_UNPACK_CMD= diff --git a/20260313_Fixes (Copilot).md b/20260313_Fixes (Copilot).md index 91a8a67..dace3d1 100644 --- a/20260313_Fixes (Copilot).md +++ b/20260313_Fixes (Copilot).md @@ -1,5 +1,691 @@ # 2026-03-13 Backend Fixes / Improvement Plan (Copilot) +## 2026-07-09 Follow-up — implemented Slices F, A, and B (MIDI editor + sloppak converter queue) + +Status: implemented and validated (backend compile + frontend typecheck). + +Completed: + +- Implemented Slice F backend API and service: + - `backend/routers/midi_editor.py` + - `backend/services/midi_editor_service.py` + - Session lifecycle: create/apply/save/discard. + - Save behavior writes edited sibling MIDI artifact and preserves original source MIDI. + - Save emits a downloadable done job (`job_type: midi-edit-save`). + +- Implemented Slice F frontend editor: + - `frontend/src/components/MidiLightEditor.tsx` + - Integrated into `frontend/src/pages/ProjectDetail.tsx` per MIDI stem row. + - Supports note edits, bend edits, quantize, save, and download of saved artifact. + +- Implemented Slice A/B converter backend and controls: + - `backend/routers/sloppak_converter.py` + - `backend/services/sloppak_converter_service.py` + - `backend/main.py` router registration + - Endpoints for enqueue/enqueue-bulk/list/pause/resume/cancel-queued/retry-failed/clear-finished. + - Skip reasons and dedupe behavior included. + +- Implemented Slice A/B frontend trigger and queue panel: + - `frontend/src/components/SloppakConverterPanel.tsx` + - Integrated into `frontend/src/pages/ProjectDetail.tsx` + - UI supports discrete and batch enqueue plus queue controls and download links. + +- Added frontend client/type surface for new APIs: + - `frontend/src/api/client.ts` + - `frontend/src/types.ts` (`sloppak_convert`, `midi-edit-save` job types) + +Implementation note: + +- Current sloppak conversion path is an operational packaging baseline that produces `.sloppak` zip artifacts containing `manifest.json` + source `.psarc` payload. Advanced Rocksmith-specific extraction/normalization remains a follow-on enhancement. + +Validation evidence: + +- Backend: + - `python3 -m compileall backend/main.py backend/routers/sloppak_converter.py backend/routers/midi_editor.py backend/services/sloppak_converter_service.py backend/services/midi_editor_service.py` (pass) +- Frontend: + - `cd frontend && npm run -s typecheck` (pass) + +## 2026-07-09 Follow-up — deeper Rocksmith-aware extraction/normalization added to sloppak converter + +Status: implemented and validated. + +Completed: + +- Enhanced converter implementation in: + - `backend/services/sloppak_converter_service.py` +- Added optional PSARC unpack support: + - primary: `patcher.unpack_psarc` (when module exists) + - fallback: env-provided unpack command template (`SLOPPAK_PSARC_UNPACK_CMD`) +- Added normalization pass over unpacked payload: + - song metadata extraction (`title`, `artist`, `album`, `year`) + - arrangement extraction/normalization (`id`, `name`, `tuning`, `capo`) + - WEM file inventory + - vocals XML lyrics extraction to normalized token records +- Enriched sloppak package contents: + - `manifest.json` now includes rocksmith unpack status + normalized metadata + - `metadata/rocksmith-summary.json` included + - `lyrics.json` included when extractable + +Compatibility behavior: + +- If no unpacker is available, conversion still succeeds with baseline source packaging and explicit unpack-failure metadata in manifest. + +Validation evidence: + +- Backend compile: + - `python3 -m compileall backend/services/sloppak_converter_service.py` (pass) + +## 2026-07-09 Follow-up — unpacker capability endpoint + deployment docs + +Status: implemented and validated. + +Completed: + +- Added converter capability endpoint: + - `GET /api/sloppak/capabilities` + - implemented via: + - `backend/routers/sloppak_converter.py` + - `backend/services/sloppak_converter_service.py` (`get_converter_capabilities`) +- Endpoint reports active unpacker path availability and root path readiness. + +- Added deployment/env wiring for converter: + - `stemengine.yml` + - `SLOPPAK_SOURCE_ROOT` + - `SLOPPAK_OUTPUT_ROOT` + - `SLOPPAK_PSARC_UNPACK_CMD` + - `.env.example` entries with concrete command-template example. + +- Added operator docs in `README.md`: + - deep extraction unpacker resolution order + - command template requirements for `SLOPPAK_PSARC_UNPACK_CMD` + - `/api/sloppak/capabilities` verification guidance + +Validation evidence: + +- `python3 -m compileall backend/routers/sloppak_converter.py backend/services/sloppak_converter_service.py` (pass) + +## 2026-07-09 Follow-up — implemented Slices C, D, and E (bend policy/derivation/playback) + +Status: implemented and validated. + +Completed: + +- Slice C (basic-pitch auto bend policy + metadata): + - `backend/services/transcriber_service.py` + - `backend/routers/transcribe.py` + - `backend/routers/projects.py` + - `backend/services/project_reprocess_service.py` + - Added `bend_policy` support (`auto|force_on|force_off`) and adaptive auto behavior. + - Added `result.bend_stats` metadata including confidence summary, stability, and bend counts. + +- Slice D (Omnizart bend derivation + safeguards): + - `backend/services/omnizart_service.py` + - Added post-process bend derivation when native bends are absent. + - Auto mode derives for `omnizart-vocal-contour`; other modes remain opt-in via `force_on`. + - Added dense-polyphony guardrails and bounded bend range. + - Added transparent derivation metadata in `result.bend_stats`. + +- Slice E (MIDI player bend playback + UI controls): + - `frontend/src/pages/ProjectDetail.tsx` + - Added pitch-bend event parsing/scheduling during MIDI playback. + - Added playback controls: bend on/off toggle (default on) and intensity slider (0-200%). + - Added bend count visibility in playback status text. + +Validation evidence: + +- Backend compile: + - `python3 -m compileall backend/services/transcriber_service.py backend/services/omnizart_service.py backend/routers/transcribe.py backend/routers/projects.py backend/services/project_reprocess_service.py` (pass) +- Frontend typecheck: + - `cd frontend && npm run -s typecheck` (pass) + +## 2026-07-09 Follow-up — dedicated conversions page added + plan remainder checklist + +Status: implemented and documented. + +Completed: + +- Added dedicated conversions page (requested): + - `frontend/src/pages/Conversions.tsx` + - route constant: `frontend/src/routes.ts` (`/conversions`) + - route wiring: `frontend/src/App.tsx` + - sidebar navigation entry: `frontend/src/components/Sidebar.tsx` + +- Updated implementation plan with explicit remaining-work checklist: + - `docs/plans/2026-07-09_StemEngine_PSARC_SlopPak_And_PitchBend_Integration_Plan.md` + +Validation evidence: + +- Frontend typecheck: + - `cd frontend && npm run -s typecheck` (pass) + +## 2026-07-09 Follow-up — completed remaining plan work (Phase 3 + Slice G closure) + +Status: implemented and validated. + +Completed: + +- Phase 3 efficiency features implemented for sloppak converter: + - queue hydration/recovery across restart: + - `backend/services/sloppak_converter_service.py` (`ensure_queue_hydrated`) + - startup hook in `backend/main.py` + - threaded bulk preflight checks for large enqueue lists: + - `backend/services/sloppak_converter_service.py` (`ThreadPoolExecutor` path in `enqueue_bulk`) + - snapshot/update stream for low-latency UI updates: + - `backend/routers/sloppak_converter.py` + - `GET /api/sloppak/snapshot` + - `GET /api/sloppak/updates` (SSE) + - `POST /api/sloppak/hydrate` + - frontend wiring: + - `frontend/src/api/client.ts` + - `frontend/src/components/SloppakConverterPanel.tsx` (EventSource subscription) + +- Slice G evidence captured with runtime regression sample: + - executed in running `stemengine` container using `python3.11` + - sample artifact paths: + - source: `/app/media/jobs/midi_regression_evidence/regression_source.mid` + - edited: `/app/media/jobs/midi_regression_evidence/regression_source.edited.mid` + - verified: + - `source_unchanged_after_save=true` + - `source_unchanged_after_discard=true` + +Validation evidence: + +- Backend compile: + - `python3 -m compileall backend/services/sloppak_converter_service.py backend/routers/sloppak_converter.py backend/main.py` (pass) +- Frontend typecheck: + - `cd frontend && npm run -s typecheck` (pass) +- Runtime container smoke checks: + - `docker exec stemengine python3.11 -c "... ensure_queue_hydrated/get_snapshot ..."` (pass) + - `docker exec stemengine python3.11 -c "... stream_updates ..."` emitted `retry: 1500` and `event: snapshot` + +Cross-worktree sync (`/home/translator/docker/stemengine_master`): + +- Mirrored all touched backend/frontend source files from `appdata/stemengine` into `stemengine_master/src`. +- Validation in mirror tree: + - `python3 -m compileall backend/services/sloppak_converter_service.py backend/routers/sloppak_converter.py backend/main.py` (pass) + - `cd frontend && npm ci && npm run -s typecheck` (pass) + +## 2026-07-09 Follow-up — PSARC/sloppak + pitch bend integration plan + +Status: analysis and implementation plan documented; implementation pending. + +Completed: + +- Compared StemEngine current capabilities against Slopsmith engineup plugin features for: + - PSARC to sloppak conversion workflows (single + batch) + - converter queue orchestration and efficiency patterns + - note detection/pitch bend strategy touchpoints +- Confirmed StemEngine baseline gaps: + - no native PSARC/sloppak conversion endpoints yet + - no explicit frontend MIDI pitch bend playback path yet +- Authored new phased plan document: + - `docs/plans/2026-07-09_StemEngine_PSARC_SlopPak_And_PitchBend_Integration_Plan.md` + +Plan scope summary: + +- New converter backend/router endpoints for discrete and batch conversion. +- Queue controls (pause/resume/cancel/retry/clear) and dedupe/skip-reason semantics. +- Auto pitch bend policy for basic-pitch and contour-derived bend support for Omnizart. +- Frontend MIDI player pitch bend scheduling and playback toggle. +- Lightweight in-app MIDI editor with edit sessions and save/discard persistence flow. + +Scope delta (added after initial draft): + +- Added explicit planning for a lightweight MIDI editor inside StemEngine. +- Defined v1 edit operations (note and bend edits, velocity, coarse quantize). +- Added session-based backend API contract for safe apply/save/discard semantics. +- Added artifact persistence rule: keep source MIDI immutable, write edited sibling output. + +Validation evidence: + +- Source references reviewed: + - `/mnt/e/Docker/Slopsmith/engineup/slopsmith-plugin-sloppak-converter/*` + - `/mnt/e/Docker/Slopsmith/engineup/slopsmith-plugin-midi/*` + - `/mnt/e/Docker/Slopsmith/engineup/slopsmith-plugin-notedetect/*` +- StemEngine touchpoints reviewed: + - `backend/config.py` + - `backend/routers/transcribe.py` + - `backend/services/transcriber_service.py` + - `backend/services/omnizart_service.py` + - `frontend/src/pages/ProjectDetail.tsx` + +## 2026-07-09 Follow-up — stemengine image size analysis and reduction plan + +Status: analysis documented; implementation pending. + +Completed: + +- Investigated Docker size reports for `stemengine:latest`. +- Captured evidence from: + - `docker image ls` + - `docker system df -v` + - `docker image inspect` + - `docker history` +- Identified dominant contributors: + - heavy Python ML dependency layers (`requirements.txt` and CUDA torch wheels) + - large CUDA runtime/system packages + - dev tooling (Node + build toolchain) in main runtime image + - significant local build cache footprint +- Added plan document: + - `docs/plans/2026-07-09_StemEngine_Image_Size_Analysis_And_Reduction_Plan.md` + +Validation evidence: + +- `docker system df -v` reported `stemengine:latest` around `30.3GB` and build cache around `18GB`. +- `docker history stemengine:latest` showed largest install layers (`8.61GB`, `5.2GB`, `1.96GB`). + +## 2026-07-08 Follow-up — finalized location-independent pathing to ./src (no ./src/stemengine) + +Status: implemented and validated. + +Completed in `/home/translator/docker/stemengine`: + +- Normalized all workspace-level script/runtime defaults from `./src/stemengine` to `./src`: + - `scripts/build-webui.ps1` + - `scripts/build-backend.ps1` + - `scripts/assemble-runtime.ps1` + - `scripts/build-all.ps1` + - `scripts/build-release.ps1` + - `packaging/pyinstaller/launch_backend.py` + - `packaging/pyinstaller/stemengine_backend.spec` + - `.github/workflows/windows-packaging.yml` (`STEMENGINE_SOURCE_ROOT=${{ github.workspace }}/src`) + +- Normalized dev container stack paths from `./src/stemengine` to `./src`: + - `docker-compose.yml` + - `docker/Dockerfile.backend.dev` + - `docker/Dockerfile.webui.dev` + +- Improved bootstrap scripts for layman-friendly behavior: + - `scripts/bootstrap-source.sh` + - `scripts/bootstrap-source.ps1` + - now clone/update directly into `./src` + - no hard requirement for absolute machine path + - if requested ref (default `main`) does not exist, scripts keep current branch instead of failing. + +- Updated docs: + - `README.md` now documents `./src` as canonical local source root and includes simplified compose/bootstrap guidance. + +Validation: + +- `scripts/bootstrap-source.sh` successfully updated existing `./src` checkout and handled missing `main` ref gracefully. +- Verified `./src/frontend/package.json` and `./src/backend/main.py` exist (source ready). + +Notes: + +- This closes the pathing requirement: workspace scripts are now independent of absolute `/home/translator/...` locations and operate relative to workspace `./src`. + +## 2026-07-08 Follow-up — moved source-root defaults to ./src and added dev docker-compose with git.imibimi.xyz clone + Vite hot reload + +Status: implemented. + +Completed in `/home/translator/docker/stemengine`: + +- Updated build script defaults to local source root: + - `scripts/build-webui.ps1` + - `scripts/build-backend.ps1` + - `scripts/assemble-runtime.ps1` + - `scripts/build-all.ps1` + - `scripts/build-release.ps1` + - New default source path: `/home/translator/docker/stemengine/src/stemengine` + +- Added source bootstrap scripts for local checkout from git.imibimi.xyz: + - `scripts/bootstrap-source.sh` + - `scripts/bootstrap-source.ps1` + +- Added dev compose stack in workspace root: + - `docker-compose.yml` + - backend build: `docker/Dockerfile.backend.dev` + - webui build: `docker/Dockerfile.webui.dev` + +Compose/dev behavior implemented: + +- Image builds clone source from `https://git.imibimi.xyz/stemengine` (override via `STEMENGINE_GIT_URL` + `STEMENGINE_GIT_REF`). +- Runtime bind-mount uses `./src/stemengine` so local edits are live in containers. +- Backend open dev access with reload: + - `uvicorn ... --reload --host 0.0.0.0 --port 8000` +- WebUI open dev access + Vite hot reload: + - `npm run dev -- --host 0.0.0.0 --port 5173` +- Polling-based file watch env enabled for containerized hot reload reliability. + +- Updated docs and ignore rules: + - `README.md` source-root and compose quickstart sections updated + - `.gitignore` now ignores `src/`, local node_modules, and `.env` + +Validation: + +- Verified script/file set exists under workspace. +- Verified README contains updated source-root and compose/hot-reload guidance. + +## 2026-07-08 Follow-up — true one-command release build wrapper added + +Status: implemented. + +Completed in `/home/translator/docker/stemengine`: + +- Added one-command release wrapper script: + - `scripts/build-release.ps1` + - Runs: + - `scripts/build-all.ps1` + - `scripts/build-installer.ps1` + - Passes through version/product/manufacturer/upgrade-code parameters for MSI build. + +- Updated workspace docs: + - `README.md` + - Replaced ambiguous "One-shot build" wording with: + - `One-command release build: scripts/build-release.ps1` + - separate "Two-step equivalent" section. + +Validation: + +- Verified script exists via scripts directory listing. + +Notes: + +- This resolves the wording/UX mismatch where "one-shot" previously required two manual commands. + +## 2026-07-08 Follow-up — strict release-tag validation + layperson release docs + +Status: implemented. + +Completed in this pass under `/home/translator/docker/stemengine`: + +- Tightened CI version policy in `.github/workflows/windows-packaging.yml`: + - tag-triggered builds are now strict and fail on malformed tags, + - accepted format remains numeric semver-like tags (`vX.Y.Z` or `vX.Y.Z.W`), + - non-tag runs retain fallback version `1.0.`. + +- Updated workspace docs: + - `README.md` now reflects strict tag behavior. + - Added layperson guide: + - `docs/LAYPERSON_RELEASE_TAGS.md` + - Guide explains in plain language what tags are valid, what happens on invalid tags, and why strict failure protects release quality. + +Validation: + +- Verified workflow includes strict `throw` path for malformed tag refs. +- Verified README points to layperson doc and describes fallback only for non-tag runs. + +Notes: + +- This fulfills the requested "go ahead" (strict policy) and "document for laypeople" follow-up. + +## 2026-07-08 Follow-up — release-tag-driven installer versioning enabled + +Status: implemented. + +Completed in this pass under `/home/translator/docker/stemengine`: + +- Updated Windows packaging CI to resolve product version from git tags: + - `.github/workflows/windows-packaging.yml` + - Added trigger on push tags `v*`. + - Added `Resolve product version` step: + - tag `vX.Y.Z` or `X.Y.Z` -> MSI/ProductVersion `X.Y.Z` + - invalid/non-numeric tag fallback -> `1.0.` + - Installer build now always uses resolved `$env:PRODUCT_VERSION`. + - Uploaded artifact names now include resolved version. + +- Updated local workspace documentation: + - `README.md` + - Added release tag versioning behavior notes. + +Validation: + +- Verified workflow contains: + - tag trigger `v*` + - resolve-version step + - build-installer using `$env:PRODUCT_VERSION` + - version-suffixed artifact names. + +Notes: + +- This implements the requested next increment after CI packaging scaffold. + +## 2026-07-08 Follow-up — "next steps" implemented for Windows packaging workspace + +Status: implemented (WiX harvest + ops scripts + CI workflow). + +Completed in this pass under `/home/translator/docker/stemengine`: + +- Wired real WiX runtime harvest and installer build path: + - `packaging/wix/Product.wxs` now takes build-time variables (`ProductName`, `ProductManufacturer`, `ProductVersion`, `UpgradeCode`). + - Added `packaging/wix/RuntimeFiles.wxs` with WiX v4 harvest rule: + - `Files Include="!(bindpath.runtime)\\**"` + - Added `scripts/build-installer.ps1` to build MSI with bindpath `runtime=build/runtime/StemEngine`. + +- Added Windows service and firewall operation scripts: + - `scripts/windows/install-service.ps1` + - `scripts/windows/uninstall-service.ps1` + - `scripts/windows/configure-firewall.ps1` + - `scripts/windows/remove-firewall.ps1` + - Firewall rule is private-profile scoped for LAN mode. + +- Added CI packaging workflow: + - `.github/workflows/windows-packaging.yml` + - Workflow provisions Node/Python/.NET+WiX, runs build scripts, validates runtime, builds MSI, uploads runtime + installer artifacts. + +- Updated local workspace documentation: + - `README.md` + - `docs/IMPLEMENTATION_TRACKER.md` + - `docs/VALIDATION_CHECKLIST.md` + +Validation: + +- Verified expanded workspace file set via `find` listing. + +Notes: + +- This pass completes the three previously listed next steps. +- End-to-end MSI build execution is defined in CI and intended for Windows runners. + +## 2026-07-08 Work Log — Windows x64 executable workspace scaffold implemented under ~/docker/stemengine + +Status: implemented (scaffold + scripts + templates + docs). + +Completed in this pass: + +- Implemented a new standalone packaging workspace under: + - `/home/translator/docker/stemengine` + +- Added scope-locked implementation scaffold for local Windows distribution: + - backend + standard WebUI only + - SingUI explicitly out of scope for this local installer path + +- Added build/runtime orchestration scripts: + - `/home/translator/docker/stemengine/scripts/build-webui.ps1` + - `/home/translator/docker/stemengine/scripts/build-backend.ps1` + - `/home/translator/docker/stemengine/scripts/assemble-runtime.ps1` + - `/home/translator/docker/stemengine/scripts/build-all.ps1` + - `/home/translator/docker/stemengine/scripts/validate-runtime.ps1` + +- Added packaging templates: + - PyInstaller launcher/spec: + - `/home/translator/docker/stemengine/packaging/pyinstaller/launch_backend.py` + - `/home/translator/docker/stemengine/packaging/pyinstaller/stemengine_backend.spec` + - WinSW service template: + - `/home/translator/docker/stemengine/packaging/winsw/stemengine-service.xml` + - WiX installer template: + - `/home/translator/docker/stemengine/packaging/wix/Product.wxs` + +- Added runtime/docs/config support files: + - `/home/translator/docker/stemengine/README.md` + - `/home/translator/docker/stemengine/config/settings.template.json` + - `/home/translator/docker/stemengine/docs/IMPLEMENTATION_TRACKER.md` + - `/home/translator/docker/stemengine/docs/VALIDATION_CHECKLIST.md` + +Validation: + +- Structural verification: + - `find /home/translator/docker/stemengine -maxdepth 4 -type f | sort` +- Python syntax check: + - `python3 -m compileall /home/translator/docker/stemengine/packaging/pyinstaller/launch_backend.py` + +Notes: + +- This pass implements the plan as an executable scaffold and delivery workspace. +- It does not perform signed installer release publishing or production firewall automation yet (tracked as next phases in the plan). + +## 2026-07-08 Work Log — Integrations separation device selector + Windows x64 installer plan + +Status: implemented (frontend + docs). + +Completed in this pass: + +- Added a dedicated separation device option under Settings -> Integrations: + - `frontend/src/pages/Settings.tsx` + - New "Separation device" control allows choosing: + - `GPU (auto when available)` + - `CPU only` + - Persists via existing settings API as: + - `cpu_separation` (`true` for CPU mode, `false` for GPU-auto) + - `hw_profile: custom` to keep profile state consistent with manual override. + +- Created a formal implementation plan for shipping StemEngine as a Windows x64 installable executable with Jellyfin-style network-host behavior: + - `docs/plans/2026-07-08_StemEngine_Windows_x64_Installable_Executable_Plan.md` + - Plan includes packaging strategy, installer/service model, LAN access model, firewall/auth guardrails, release pipeline, and phase-gated validation. + +Validation: + +- Frontend typecheck: + - `cd frontend && npm run -s typecheck` (no type errors reported) + +Notes: + +- Backend runtime behavior for separation device was already wired through `cpu_separation`; this pass adds the requested Integrations-surface control and documentation plan. + +## 2026-07-07 Work Log — ONNX Runtime CUDA alignment + image build aliases + +Status: implemented and runtime-validated. + +Completed in this pass: + +- ONNX Runtime GPU compatibility fix: + - `backend/requirements.txt` + - Pinned `onnxruntime-gpu==1.20.1` to avoid `libcudart.so.13` runtime linkage in the current CUDA 12.4 stack. + +- Runtime validation in live container: + - Confirmed previous failure on `onnxruntime-gpu 1.27.0` (`ImportError: libcudart.so.13`). + - Applied compatibility hotfix in running container (`pip install onnxruntime-gpu==1.20.1`). + - Verified providers now load successfully: + - `['TensorrtExecutionProvider', 'CUDAExecutionProvider', 'CPUExecutionProvider']` + +- Added discrete/all-at-once image build helpers: + - `~/.bashrc` + - New functions: + - `se-img-build ` + - `se-img-build-all` + - New aliases: + - `se-build-image` -> `se-img-build` + - `se-build-images` -> `se-img-build-all` + - Supported image targets: + - `stemengine` + - `stemengine-webui` + - `stemengine-muon` + - `stemengine-fluid` + - `stemengine-tendrils` + - `stemengine-spirit` + - `omnizart-sidecar` + - `parakeet-sidecar` + +Notes: + +- Current deployed compose at `/home/translator/docker/stemengine.yml` pulls prebuilt registry images by default. +- The runtime hotfix is active now, but full persistence requires rebuilding/publishing the `stemengine` image with the pinned requirement and redeploying from that image. + +### 2026-07-07 Follow-up — one-command local deploy + compose local-build override + +Status: implemented and validated. + +Completed in this pass: + +- Added local-build override compose file: + - `/home/translator/docker/stemengine.local.override.yml` + - Services overridden to prefer local builds and never pull: + - `stemengine` + - `stemengine-muon` + - `stemengine-fluid` + - `stemengine-tendrils` + - `stemengine-spirit` + - `omnizart-sidecar` + - Each service now has: + - `pull_policy: never` + - explicit `build.context` + `build.dockerfile` + +- Added one-command deploy helper to `~/.bashrc`: + - `se-deploy-stemengine-local [tag]` + - Behavior: + - builds `git.imibimi.xyz/translator/stemengine:latest` + - tags image as `git.imibimi.xyz/translator/stemengine:` (timestamp default) + - recreates only `stemengine` with no pull: + - `docker compose -f /home/translator/docker/stemengine.yml -f /home/translator/docker/stemengine.local.override.yml up -d --no-deps --force-recreate --pull never stemengine` + +- Added local stack shortcut helpers to `~/.bashrc`: + - `se-up-local` + - runs full stack bring-up using base + override compose files. + - alias: `se-deploy-local` -> `se-deploy-stemengine-local` + +Validation: + +- `docker compose -f /home/translator/docker/stemengine.yml -f /home/translator/docker/stemengine.local.override.yml config` confirms: + - local `build` blocks are applied + - `pull_policy: never` is applied for overridden services +- interactive shell check confirms helpers are loaded: + - `type se-deploy-stemengine-local` + - `alias se-deploy-local` + - `alias se-up-local` + +## 2026-07-07 Work Log — Demucs stuck-at-100% fix + GPU device enforcement + +Status: implemented, deployed, and validated on live backend. + +Completed in this pass: + +- Fixed Demucs post-completion spin loop: + - `backend/services/separator_service.py` + - In `_run_demucs()`, the stdout polling loop now exits on EOF when the child process has finished. + - This resolves the observed state where jobs remained `running` at `100%` indefinitely while `separator_worker` consumed high CPU. + +- Enforced explicit Demucs device selection: + - `backend/services/separator_service.py` + - `_run_demucs()` now computes `demucs_device` and passes `--device` to the CLI. + - Behavior: + - `CPU_SEPARATION=true` => force `cpu` + - otherwise => use `cuda` when CUDA is available, else fallback `cpu` + - Job progress now includes selected device (`device=cuda` or `device=cpu`) for observability. + +- Hardened compose GPU intent for backend service: + - `docker/stemengine.yml` + - Added `gpus: all` for `stemengine`. + - Added explicit env defaults: + - `NVIDIA_VISIBLE_DEVICES=${NVIDIA_VISIBLE_DEVICES:-all}` + - `NVIDIA_DRIVER_CAPABILITIES=${NVIDIA_DRIVER_CAPABILITIES:-compute,utility}` + +Operational actions performed: + +- Recreated backend service: + - `docker compose -f /home/translator/docker/stemengine.yml up -d --force-recreate stemengine` +- Restarted backend after code patch: + - `docker restart stemengine` +- Cleared stale stuck job from queue: + - `DELETE /api/jobs/6cafc416-fe31-489f-b5f1-135b3b9a1eb2` + +Validation: + +- Fresh separation request accepted: + - `POST /api/separate/from-path` for `/app/media/imports/tUnE-yArDs - Heartbreak.flac` + - Job: `1a2b9beb-d79e-4bdf-929d-0db5d8a7a633` +- Live job state confirms GPU selection: + - `GET /api/jobs/1a2b9beb-d79e-4bdf-929d-0db5d8a7a633` + - progress includes: `Long-track tuning: ... device=cuda ...` +- Runtime capability checks inside container: + - `torch.cuda.is_available() == True` + - `torch.cuda.device_count() == 1` + - device name: `NVIDIA GeForce RTX 3060` + +Notes: + +- `nvidia-smi` inside container is functional; CUDA compute is available to PyTorch. +- ONNX Runtime currently fails import with `libcudart.so.13` missing in this image; this is separate from Demucs and may affect ONNX-model GPU paths until runtime/library versions are aligned. + ## 2026-04-29 Work Log — Per-renderer iframe audit remediation pass Implemented audit follow-up items across Spirit runtime, iframe Docker guards, and docs/spec drift. diff --git a/README.md b/README.md index 4650e40..b459706 100755 --- a/README.md +++ b/README.md @@ -132,6 +132,9 @@ All configuration is via environment variables. The `docker-compose.yml` accepts | `NAVSERVER` | — | Subsonic server URL (can also be set in the UI) | | `NAVUSER` | — | Subsonic username | | `NAVPASS` | — | Subsonic password | +| `SLOPPAK_SOURCE_ROOT` | `/app/media/uploads` | Converter source root for relative PSARC paths | +| `SLOPPAK_OUTPUT_ROOT` | `/app/media/sloppak` | Converter output root for generated `.sloppak` files | +| `SLOPPAK_PSARC_UNPACK_CMD` | empty | Optional PSARC unpack command template (`{input}` and `{output}` placeholders) used for deep Rocksmith metadata extraction when python patcher module is unavailable | ### Model Directories @@ -146,6 +149,31 @@ These are set automatically in `docker-compose.yml` and point inside the `/opt/` Each separation job creates a folder under `PROJECTS_DIR`: ``` + +--- + +## sloppak Converter Deep Extraction + +The PSARC -> sloppak converter can enrich packages with Rocksmith metadata (arrangements, lyrics, WEM inventory) when a PSARC unpacker is available. + +Unpacker resolution order: + +1. Python `patcher.unpack_psarc` module (if installed) +2. `SLOPPAK_PSARC_UNPACK_CMD` command template from environment + +Command template requirements: + +- Must include `{input}` and `{output}` placeholders. +- Example: + +```bash +SLOPPAK_PSARC_UNPACK_CMD="/usr/local/bin/psarc unpack {input} {output}" +``` + +Runtime verification endpoint: + +- `GET /api/sloppak/capabilities` + - Reports whether a usable unpacker is currently detected. Artist - Title/ Artist - Title.mp3 # Original source audio (copy) Artist - Title.elrc # ELRC lyrics with word-level timestamps (if aligned) diff --git a/SUMMARIES (Claude).md b/SUMMARIES (Claude).md index aecb9cb..94d3bb2 100644 --- a/SUMMARIES (Claude).md +++ b/SUMMARIES (Claude).md @@ -1,5 +1,412 @@ # StemEngine — Session Summaries +## 2026-07-09 + +### Implemented: Slice F then Slices A/B (MIDI editor + sloppak converter) + +Completed in `/home/translator/docker/appdata/stemengine`: + +- Slice F (lightweight MIDI editor) implemented end-to-end: + - backend: + - `backend/routers/midi_editor.py` + - `backend/services/midi_editor_service.py` + - frontend: + - `frontend/src/components/MidiLightEditor.tsx` + - integrated in `frontend/src/pages/ProjectDetail.tsx` +- Slice A/B converter implementation added: + - backend: + - `backend/routers/sloppak_converter.py` + - `backend/services/sloppak_converter_service.py` + - router wired in `backend/main.py` + - frontend: + - `frontend/src/components/SloppakConverterPanel.tsx` + - integrated in `frontend/src/pages/ProjectDetail.tsx` +- API/type updates: + - `frontend/src/api/client.ts` + - `frontend/src/types.ts` + +Behavior delivered: + +1. Session-based MIDI edits (notes + pitch bends) with save/discard flow. +2. Save writes a new sibling MIDI artifact; original MIDI remains unchanged. +3. sloppak queue supports single/batch enqueue and operational controls (pause/resume/cancel-queued/retry-failed/clear-finished). + +Validation: + +- Backend compile pass: + - `python3 -m compileall backend/main.py backend/routers/sloppak_converter.py backend/routers/midi_editor.py backend/services/sloppak_converter_service.py backend/services/midi_editor_service.py` +- Frontend typecheck pass: + - `cd frontend && npm run -s typecheck` + +### Follow-up implemented: deeper Rocksmith-aware extraction/normalization + +Completed in `/home/translator/docker/appdata/stemengine`: + +- Upgraded converter packaging logic in: + - `backend/services/sloppak_converter_service.py` +- Added optional unpack + normalization flow: + - tries Python `patcher.unpack_psarc` first, + - can use `SLOPPAK_PSARC_UNPACK_CMD` template fallback. +- Added normalized extraction outputs when unpack succeeds: + - song metadata (title/artist/album/year), + - arrangements (id/name/tuning/capo), + - WEM inventory, + - vocals XML lyrics. +- Added these into output package: + - enriched `manifest.json` rocksmith sections, + - `metadata/rocksmith-summary.json`, + - optional `lyrics.json`. + +Validation: + +- `python3 -m compileall backend/services/sloppak_converter_service.py` (pass) + +### Follow-up implemented: unpacker capability endpoint + env docs + +Completed in `/home/translator/docker/appdata/stemengine`: + +- Added `GET /api/sloppak/capabilities` for runtime unpacker health/status. +- Added converter env pass-through in `stemengine.yml`: + - `SLOPPAK_SOURCE_ROOT` + - `SLOPPAK_OUTPUT_ROOT` + - `SLOPPAK_PSARC_UNPACK_CMD` +- Updated `.env.example` with concrete command-template guidance. +- Updated `README.md` with deep extraction setup and capability endpoint verification. + +Validation: + +- `python3 -m compileall backend/routers/sloppak_converter.py backend/services/sloppak_converter_service.py` (pass) + +### Follow-up implemented: Slices C, D, and E (pitch bend pipeline) + +Completed in `/home/translator/docker/appdata/stemengine`: + +- Slice C (basic-pitch): + - Added bend policy routing (`auto|force_on|force_off`) through transcription endpoints. + - Implemented adaptive auto bend behavior and persisted `result.bend_stats` diagnostics. + - Touched: + - `backend/services/transcriber_service.py` + - `backend/routers/transcribe.py` + - `backend/routers/projects.py` + - `backend/services/project_reprocess_service.py` + +- Slice D (Omnizart): + - Added post-process bend derivation when native bends are missing. + - Auto behavior targets `vocal-contour`; other modes are opt-in via `force_on`. + - Added dense/polyphonic guardrails and derivation transparency metadata. + - Touched: + - `backend/services/omnizart_service.py` + +- Slice E (Frontend playback): + - Added pitch bend scheduling in MIDI playback. + - Added bend playback on/off toggle (default on) and intensity slider. + - Added bend event count visibility in playback status. + - Touched: + - `frontend/src/pages/ProjectDetail.tsx` + +Validation: + +- `python3 -m compileall backend/services/transcriber_service.py backend/services/omnizart_service.py backend/routers/transcribe.py backend/routers/projects.py backend/services/project_reprocess_service.py` (pass) +- `cd frontend && npm run -s typecheck` (pass) + +### Follow-up implemented: dedicated conversions page + explicit remaining-work checklist + +Completed in `/home/translator/docker/appdata/stemengine`: + +- Added dedicated conversion page and navigation: + - `frontend/src/pages/Conversions.tsx` + - `frontend/src/routes.ts` (`/conversions`) + - `frontend/src/App.tsx` route wiring + - `frontend/src/components/Sidebar.tsx` entry link + +- Updated plan with a clear "Remaining Work Checklist" section to distinguish done vs open items: + - `docs/plans/2026-07-09_StemEngine_PSARC_SlopPak_And_PitchBend_Integration_Plan.md` + +Validation: + +- `cd frontend && npm run -s typecheck` (pass) + +### Follow-up implemented: completed remaining plan work + mirrored to stemengine_master + +Completed in `/home/translator/docker/appdata/stemengine`: + +- Phase 3 converter efficiency items closed: + - restart hydration (`ensure_queue_hydrated` + startup hook) + - threaded bulk preflight checks for large batches + - snapshot/update stream endpoints + frontend SSE subscription +- Slice G evidence closed with runtime regression artifacts: + - source `/app/media/jobs/midi_regression_evidence/regression_source.mid` + - edited `/app/media/jobs/midi_regression_evidence/regression_source.edited.mid` + - source immutability verified after save/discard + +Mirror sync completed to `/home/translator/docker/stemengine_master/src`: + +- Synced touched backend/frontend files (sloppak service/router/startup, client, panel, routes, app/sidebar, conversions page). + +Validation: + +- appdata backend compile: pass +- appdata frontend typecheck: pass +- runtime container checks for hydrate/snapshot/stream: pass +- stemengine_master backend compile: pass +- stemengine_master frontend `npm ci` + typecheck: pass + +### PSARC to sloppak + pitch bend feature integration plan + +Completed in `/home/translator/docker/appdata/stemengine`: + +- Analyzed Slopsmith engineup references for converter and queue behavior: + - `slopsmith-plugin-sloppak-converter` + - `slopsmith-plugin-midi` + - `slopsmith-plugin-notedetect` +- Compared against StemEngine backend/frontend capabilities and identified implementation gaps. +- Produced a phased architecture and delivery plan in: + - `docs/plans/2026-07-09_StemEngine_PSARC_SlopPak_And_PitchBend_Integration_Plan.md` + +Planned feature scope: + +1. Discrete + batch PSARC to sloppak conversion endpoints and queue UX. +2. Queue efficiency and resilience patterns (dedupe, reason codes, pause/resume, persistence). +3. Automatic pitch bend policy for basic-pitch and contour-derived bends for Omnizart. +4. MIDI player pitch bend playback support and user toggle. +5. Lightweight MIDI editor (notes + bends) with save/discard persistence workflow. + +Scope update: + +- Plan expanded to include an in-app lightweight MIDI editor that saves user edits as a new MIDI artifact while preserving the original source file. + +Status: + +- Documentation/planning complete. +- Code implementation intentionally deferred to the next execution phase. + +### StemEngine image-size investigation and shrink plan + +Completed in `/home/translator/docker/appdata/stemengine`: + +- Investigated why `stemengine:latest` is reported around `30.3GB`. +- Confirmed the main image is the dominant local image footprint; additional host pressure includes large Docker build cache. +- Reviewed Dockerfile/layer history and dependency manifests to isolate high-impact contributors. +- Documented full analysis + phased reduction strategy in: + - `docs/plans/2026-07-09_StemEngine_Image_Size_Analysis_And_Reduction_Plan.md` + +Key findings: + +- Largest layers are Python dependency installation and CUDA torch wheel install. +- Runtime image currently includes dev/watch tooling and compiler toolchain. +- Build cache is a separate but major storage contributor. + +Next implementation phases (planned): + +1. Split dev/runtime Dockerfiles and remove dev-only runtime dependencies. +2. Split dependency profiles and image variants (core/full). +3. Rationalize CUDA/cuDNN footprint while preserving compatibility constraints. + +## 2026-07-08 + +### Final path normalization: workspace now uses ./src only + +Follow-up refinement completed in `/home/translator/docker/stemengine`: + +- Removed workspace-level `./src/stemengine` assumptions and standardized on `./src`. +- Updated scripts, compose, Dockerfiles, CI source-root env, and PyInstaller defaults to use `./src`. + +Layman-friendly bootstrap improvement: + +- `scripts/bootstrap-source.sh` and `scripts/bootstrap-source.ps1` now: + - clone/update directly into `./src`, + - fall back gracefully when requested ref (default `main`) is missing, + - keep current branch instead of hard-failing. + +Validation: + +- bootstrap run updated `./src` successfully and printed graceful missing-ref fallback. +- source readiness verified (`./src/frontend/package.json` and `./src/backend/main.py`). + +## 2026-07-08 + +### Source-root moved to ./src and dev docker-compose stack implemented + +Implemented in `/home/translator/docker/stemengine`: + +- Packaging/build script defaults now target local source path: + - `/home/translator/docker/stemengine/src/stemengine` + +- Added source bootstrap scripts: + - `scripts/bootstrap-source.sh` + - `scripts/bootstrap-source.ps1` + +- Added root-level dev stack: + - `docker-compose.yml` + - `docker/Dockerfile.backend.dev` + - `docker/Dockerfile.webui.dev` + +Dev stack behavior: + +- image build clones from `https://git.imibimi.xyz/stemengine`, +- runtime bind-mount uses `./src/stemengine`, +- backend open access + reload on `0.0.0.0:8000`, +- Vite open access + hot reload on `0.0.0.0:5173`. + +Docs updated: + +- `README.md` now documents new source root and docker compose quickstart. +- `.gitignore` updated for local source/dev artifacts. + +Validation: + +- workspace file listing and README grep checks confirm implementation. + +## 2026-07-08 + +### One-command release build script added + +Implemented in `/home/translator/docker/stemengine`: + +- Added `scripts/build-release.ps1` as a true one-command wrapper. + - Executes runtime build (`build-all.ps1`) then MSI build (`build-installer.ps1`). + +- Updated `README.md` wording: + - now documents `build-release.ps1` under "One-command release build" + - keeps old two-command flow under "Two-step equivalent" + +Validation: + +- scripts folder listing confirms `build-release.ps1` is present. + +## 2026-07-08 + +### Strict tag validation enabled and layperson release guide added + +Follow-up completed in `/home/translator/docker/stemengine`: + +- CI behavior changed in `.github/workflows/windows-packaging.yml`: + - tag-triggered builds now fail fast on malformed tags, + - valid release tags remain `vX.Y.Z` (or `vX.Y.Z.W`), + - fallback `1.0.` remains for non-tag runs only. + +- Documentation updates: + - `README.md` updated to describe strict/fallback split. + - New layperson-friendly guide added: + - `docs/LAYPERSON_RELEASE_TAGS.md` + +Validation: + +- grep verification confirms strict `throw` path exists and README links to layperson guide. + +## 2026-07-08 + +### CI installer version now derived from release tags + +Follow-up enhancement in `/home/translator/docker/stemengine`: + +- `.github/workflows/windows-packaging.yml` now supports release-tag versioning. + - push tag trigger: `v*` + - resolve step maps `vX.Y.Z` -> `X.Y.Z` + - fallback for non-numeric tags: `1.0.` + - MSI build consumes resolved `$env:PRODUCT_VERSION` + - artifact names include resolved version suffix + +- `README.md` updated with release-tag versioning behavior. + +Validation: + +- workflow grep checks confirmed tag trigger, version resolver, version-fed installer build, and versioned artifact names. + +## 2026-07-08 + +### Windows packaging next steps completed (WiX harvest + ops scripts + CI) + +Follow-up implementation completed in `/home/translator/docker/stemengine`: + +- WiX harvest/build wiring: + - `packaging/wix/Product.wxs` switched to build-time variables + - `packaging/wix/RuntimeFiles.wxs` added with `Files Include="!(bindpath.runtime)\\**"` + - `scripts/build-installer.ps1` added to build MSI from assembled runtime payload + +- Windows operational scripts added: + - `scripts/windows/install-service.ps1` + - `scripts/windows/uninstall-service.ps1` + - `scripts/windows/configure-firewall.ps1` + - `scripts/windows/remove-firewall.ps1` + +- CI workflow added: + - `.github/workflows/windows-packaging.yml` + - Builds runtime, validates layout, builds MSI, uploads artifacts. + +- Docs updated: + - `README.md` + - `docs/IMPLEMENTATION_TRACKER.md` + - `docs/VALIDATION_CHECKLIST.md` + +Validation: + +- file inventory verification confirms new workflow/scripts/templates present. + +## 2026-07-08 + +### Windows x64 executable implementation scaffold under ~/docker/stemengine + +Implemented a concrete plan execution workspace at: + +- `/home/translator/docker/stemengine` + +Delivered artifacts: + +- Build/runtime scripts: + - `scripts/build-webui.ps1` + - `scripts/build-backend.ps1` + - `scripts/assemble-runtime.ps1` + - `scripts/build-all.ps1` + - `scripts/validate-runtime.ps1` +- Packaging templates: + - `packaging/pyinstaller/launch_backend.py` + - `packaging/pyinstaller/stemengine_backend.spec` + - `packaging/winsw/stemengine-service.xml` + - `packaging/wix/Product.wxs` +- Docs/config: + - `README.md` + - `config/settings.template.json` + - `docs/IMPLEMENTATION_TRACKER.md` + - `docs/VALIDATION_CHECKLIST.md` + +Scope lock enforced in scaffold: + +- local Windows distribution targets backend + standard WebUI only, +- SingUI is out of scope for this packaging path. + +Validation performed: + +- file structure verification via `find` on workspace, +- Python syntax check passed for `launch_backend.py`. + +## 2026-07-08 + +### Integrations CPU/GPU separation selector + Windows x64 packaging plan + +Implemented: + +- Added a new "Separation device" control in Settings -> Integrations: + - `frontend/src/pages/Settings.tsx` + - Option values: + - GPU (auto when available) + - CPU only + - Save action persists through existing settings API by writing: + - `cpu_separation` + - `hw_profile: custom` + +- Added new planning document for a Jellyfin-style Windows distribution path: + - `docs/plans/2026-07-08_StemEngine_Windows_x64_Installable_Executable_Plan.md` + - Covers: + - installer/runtime architecture, + - service + LAN-access model, + - firewall/auth/security baseline, + - phased implementation and release gates. + +Validation: + +- frontend typecheck passed (`npm run -s typecheck` from `frontend/`). + ## 2026-04-29 ### Per-renderer iframe audit remediation (implementation + docs) diff --git a/backend/main.py b/backend/main.py index c075726..1e11a12 100755 --- a/backend/main.py +++ b/backend/main.py @@ -41,12 +41,13 @@ from fastapi.staticfiles import StaticFiles from fastapi.responses import FileResponse from . import job_manager as jm -from .routers import separate, transcribe, abc_convert, pipeline, jobs, lrc, subsonic, projects, models, lyric_transcribe, system, settings, karaoke_sek, pixabay, room, local_library, gloss +from .routers import separate, transcribe, abc_convert, pipeline, jobs, lrc, subsonic, projects, models, lyric_transcribe, system, settings, karaoke_sek, pixabay, room, local_library, gloss, sloppak_converter, midi_editor from .config import SUBSONIC_CONFIG_PATH from .app_settings import settings as app_settings from .db import init_db from .services.local_library_service import seed_music_library_path_from_env from .services.gloss_service import initialize_gloss_runtime +from .services import sloppak_converter_service _log = logging.getLogger(__name__) @@ -111,6 +112,16 @@ async def lifespan(app): initialize_gloss_runtime() except Exception: _log.exception("Startup gloss runtime initialization failed") + try: + hydrate = sloppak_converter_service.ensure_queue_hydrated() + if hydrate.get("resubmitted", 0) or hydrate.get("requeued_running", 0): + _log.info( + "Startup sloppak queue hydrate: resubmitted=%s requeued_running=%s", + hydrate.get("resubmitted", 0), + hydrate.get("requeued_running", 0), + ) + except Exception: + _log.exception("Startup sloppak queue hydration failed") for issue in app_settings.validate_startup(): _log.warning("Startup check: %s", issue) yield @@ -134,6 +145,8 @@ def _include_routers(app: FastAPI) -> None: app.include_router(karaoke_sek.router) app.include_router(pixabay.router) app.include_router(room.router) + app.include_router(sloppak_converter.router) + app.include_router(midi_editor.router) def _mount_static(app: FastAPI) -> None: diff --git a/backend/requirements.txt b/backend/requirements.txt index 56618bc..0dd6b54 100755 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -40,7 +40,8 @@ hyper-connections>=0.0.10 # MDX-Net dependencies (ONNX models from UVR) # onnxruntime-gpu provides CUDAExecutionProvider so both MDX-Net and basic-pitch # automatically run on the GPU when available (conflicts with onnxruntime CPU package). -onnxruntime-gpu>=1.16.0 +# Pin below 1.27 because newer wheels in this stack require libcudart.so.13. +onnxruntime-gpu==1.20.1 onnx>=1.14.0 # HuggingFace CLI (for BS-RoFormer model download) diff --git a/backend/routers/midi_editor.py b/backend/routers/midi_editor.py new file mode 100644 index 0000000..1342033 --- /dev/null +++ b/backend/routers/midi_editor.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from typing import Any, Optional + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel, Field + +from ..services import midi_editor_service as svc + + +router = APIRouter(prefix="/api/midi/edit", tags=["midi-editor"]) + + +class CreateSessionRequest(BaseModel): + job_id: str = Field(..., min_length=1) + filename: Optional[str] = None + + +class ApplyOpsRequest(BaseModel): + ops: list[dict[str, Any]] = Field(default_factory=list) + + +@router.post("/sessions") +def create_session(req: CreateSessionRequest): + try: + return svc.create_session(req.job_id, req.filename) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.get("/sessions/{session_id}") +def get_session(session_id: str): + try: + return svc.get_session(session_id) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@router.patch("/sessions/{session_id}") +def apply_ops(session_id: str, req: ApplyOpsRequest): + try: + return svc.apply_ops(session_id, req.ops) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.post("/sessions/{session_id}/save") +def save_session(session_id: str): + try: + return svc.save_session(session_id) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.delete("/sessions/{session_id}") +def discard_session(session_id: str): + try: + return svc.discard_session(session_id) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc \ No newline at end of file diff --git a/backend/routers/projects.py b/backend/routers/projects.py index fef9abb..861f694 100755 --- a/backend/routers/projects.py +++ b/backend/routers/projects.py @@ -73,6 +73,7 @@ def _validate_mp3_bitrate(value: Optional[str]) -> Optional[str]: class TranscribeRequest(BaseModel): stem_name: str # which stem to transcribe (key from sep job) mode: str = DEFAULT_TRANSCRIBE_MODE + bend_policy: str = "auto" # auto | force_on | force_off class AbcRequest(BaseModel): @@ -182,7 +183,7 @@ def transcribe_project(project_id: str, body: TranscribeRequest): """Transcribe a specific stem from this project to MIDI.""" project = _get_project_or_404(project_id) try: - return enqueue_project_transcription(project, body.stem_name, body.mode) + return enqueue_project_transcription(project, body.stem_name, body.mode, body.bend_policy) except ProjectOperationError as exc: raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc diff --git a/backend/routers/sloppak_converter.py b/backend/routers/sloppak_converter.py new file mode 100644 index 0000000..a39bc2a --- /dev/null +++ b/backend/routers/sloppak_converter.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +from fastapi import APIRouter +from fastapi.responses import StreamingResponse +from pydantic import BaseModel, Field + +from ..services import sloppak_converter_service as svc + + +router = APIRouter(prefix="/api/sloppak", tags=["sloppak"]) + + +class EnqueueRequest(BaseModel): + filename: str = Field(..., min_length=1) + reconvert: bool = False + + +class EnqueueBulkRequest(BaseModel): + filenames: list[str] = Field(default_factory=list) + reconvert: bool = False + + +@router.get("/jobs") +def list_jobs(): + return { + "paused": svc.is_paused(), + "jobs": svc.list_conversion_jobs(), + } + + +@router.get("/snapshot") +def snapshot(): + return svc.get_snapshot() + + +@router.get("/updates") +def updates(): + return StreamingResponse( + svc.stream_updates(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "Connection": "keep-alive"}, + ) + + +@router.post("/hydrate") +def hydrate(): + return svc.ensure_queue_hydrated() + + +@router.get("/capabilities") +def capabilities(): + return svc.get_converter_capabilities() + + +@router.post("/enqueue") +def enqueue(req: EnqueueRequest): + return svc.enqueue_conversion(req.filename, reconvert=req.reconvert) + + +@router.post("/enqueue-bulk") +def enqueue_bulk(req: EnqueueBulkRequest): + return svc.enqueue_bulk(req.filenames, reconvert=req.reconvert) + + +@router.post("/pause") +def pause_queue(): + return svc.pause_queue() + + +@router.post("/resume") +def resume_queue(): + return svc.resume_queue() + + +@router.post("/jobs/cancel-queued") +def cancel_queued(): + return svc.cancel_queued() + + +@router.post("/jobs/retry-failed") +def retry_failed(): + return svc.retry_failed() + + +@router.post("/jobs/clear-finished") +def clear_finished(): + return svc.clear_finished() \ No newline at end of file diff --git a/backend/routers/transcribe.py b/backend/routers/transcribe.py index d470d81..6549f02 100755 --- a/backend/routers/transcribe.py +++ b/backend/routers/transcribe.py @@ -12,6 +12,7 @@ from ..services.upload_io import save_upload_file router = APIRouter(prefix="/api/transcribe", tags=["transcribe"]) VALID_MODES = {m["id"] for m in TRANSCRIBE_MODES} +_VALID_BEND_POLICIES = {"auto", "force_on", "force_off"} # Preferred stem order for transcription (drums are skipped; vocals last since # Omnizart's "music" mode is most useful on pitched instruments). @@ -70,6 +71,7 @@ def get_modes(): async def transcribe( file: UploadFile = File(...), mode: str = Form(DEFAULT_TRANSCRIBE_MODE), + bend_policy: str = Form("auto"), _admin_guard: None = Depends(require_admin_token), ): if mode not in VALID_MODES: @@ -77,13 +79,16 @@ async def transcribe( if mode.startswith("omnizart-") and not OMNIZART_ENABLED: raise HTTPException(status_code=503, detail="Omnizart is not available on this server (OMNIZART_ENABLED=false).") + if bend_policy not in _VALID_BEND_POLICIES: + raise HTTPException(status_code=400, detail=f"Unknown bend_policy: {bend_policy}") + job_id = jm.create_job("transcribe") upload_path = UPLOAD_DIR / job_id / _safe_audio_upload_filename(file.filename) upload_path.parent.mkdir(parents=True, exist_ok=True) await save_upload_file(file, upload_path) - jm.submit_gpu(run_transcription, job_id, str(upload_path), mode) + jm.submit_gpu(run_transcription, job_id, str(upload_path), mode, bend_policy) return {"job_id": job_id} @@ -91,6 +96,7 @@ async def transcribe( async def transcribe_from_project( project_name: str = Form(...), mode: str = Form(DEFAULT_TRANSCRIBE_MODE), + bend_policy: str = Form("auto"), _admin_guard: None = Depends(require_admin_token), ): """ @@ -101,6 +107,8 @@ async def transcribe_from_project( raise HTTPException(status_code=400, detail=f"Unknown mode: {mode}") if mode.startswith("omnizart-") and not OMNIZART_ENABLED: raise HTTPException(status_code=503, detail="Omnizart is not available on this server (OMNIZART_ENABLED=false).") + if bend_policy not in _VALID_BEND_POLICIES: + raise HTTPException(status_code=400, detail=f"Unknown bend_policy: {bend_policy}") safe_project_name = _safe_project_name(project_name) stems_dir = PROJECTS_DIR / safe_project_name / "Stems" @@ -133,5 +141,5 @@ async def transcribe_from_project( raise HTTPException(status_code=404, detail="No suitable (non-drum) stem found in project") job_id = jm.create_job("transcribe") - jm.submit_gpu(run_transcription, job_id, str(stem_file), mode) + jm.submit_gpu(run_transcription, job_id, str(stem_file), mode, bend_policy) return {"job_id": job_id, "stem": stem_file.name} diff --git a/backend/services/midi_editor_service.py b/backend/services/midi_editor_service.py new file mode 100644 index 0000000..7ee340e --- /dev/null +++ b/backend/services/midi_editor_service.py @@ -0,0 +1,311 @@ +from __future__ import annotations + +import logging +import threading +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import pretty_midi + +from .. import job_manager as jm + + +log = logging.getLogger(__name__) + +_SESSIONS_LOCK = threading.Lock() +_SESSIONS: dict[str, dict[str, Any]] = {} + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _clamp(value: float, lo: float, hi: float) -> float: + return max(lo, min(hi, value)) + + +def _resolve_midi_path(job_id: str, filename: str | None) -> Path: + job = jm.get_job(job_id) + if not job: + raise ValueError("job not found") + + files = [f for f in (job.get("files") or []) if str(f.get("name", "")).lower().endswith(".mid")] + if not files: + raise ValueError("no midi file found on job") + + chosen = None + if filename: + for item in files: + if item.get("name") == filename: + chosen = item + break + if not chosen: + raise ValueError("midi filename not found on job") + else: + chosen = files[0] + + path = Path(str(chosen.get("path", ""))) + if not path.exists() or not path.is_file(): + raise ValueError("midi file is missing from disk") + return path + + +def create_session(job_id: str, filename: str | None = None) -> dict[str, Any]: + source_path = _resolve_midi_path(job_id, filename) + midi = pretty_midi.PrettyMIDI(str(source_path)) + + tracks: list[dict[str, Any]] = [] + for idx, instrument in enumerate(midi.instruments): + notes = [ + { + "id": uuid4().hex[:10], + "pitch": int(note.pitch), + "start": float(note.start), + "duration": float(max(0.001, note.end - note.start)), + "velocity": int(note.velocity), + } + for note in instrument.notes + ] + bends = [ + { + "id": uuid4().hex[:10], + "time": float(bend.time), + "value": int(bend.pitch), + } + for bend in instrument.pitch_bends + ] + tracks.append( + { + "index": idx, + "name": instrument.name or f"Track {idx + 1}", + "program": int(instrument.program), + "is_drum": bool(instrument.is_drum), + "notes": notes, + "bends": bends, + } + ) + + session_id = uuid4().hex + payload = { + "id": session_id, + "created_at": _now_iso(), + "source_job_id": job_id, + "source_filename": source_path.name, + "source_path": str(source_path), + "tracks": tracks, + "editor_version": "stemengine-midi-editor-v1", + } + + with _SESSIONS_LOCK: + _SESSIONS[session_id] = payload + return payload + + +def get_session(session_id: str) -> dict[str, Any]: + with _SESSIONS_LOCK: + session = _SESSIONS.get(session_id) + if not session: + raise ValueError("session not found") + return session + + +def discard_session(session_id: str) -> dict[str, Any]: + with _SESSIONS_LOCK: + existed = _SESSIONS.pop(session_id, None) is not None + if not existed: + raise ValueError("session not found") + return {"discarded": True} + + +def apply_ops(session_id: str, ops: list[dict[str, Any]]) -> dict[str, Any]: + with _SESSIONS_LOCK: + session = _SESSIONS.get(session_id) + if not session: + raise ValueError("session not found") + + for op in ops: + _apply_single_op(session, op) + + return session + + +def _get_track(session: dict[str, Any], track_index: int) -> dict[str, Any]: + tracks: list[dict[str, Any]] = session["tracks"] + if track_index < 0 or track_index >= len(tracks): + raise ValueError("track_index out of range") + return tracks[track_index] + + +def _apply_single_op(session: dict[str, Any], op: dict[str, Any]) -> None: + op_name = str(op.get("op", "")).strip() + if not op_name: + raise ValueError("op is required") + + track_index = int(op.get("track_index", 0)) + track = _get_track(session, track_index) + + if op_name == "add_note": + start = float(op.get("start", 0.0)) + duration = float(op.get("duration", 0.5)) + note = { + "id": uuid4().hex[:10], + "pitch": int(_clamp(float(op.get("pitch", 60)), 0, 127)), + "start": _clamp(start, 0.0, 86400.0), + "duration": _clamp(duration, 0.001, 60.0), + "velocity": int(_clamp(float(op.get("velocity", 100)), 1, 127)), + } + track["notes"].append(note) + track["notes"].sort(key=lambda x: (x["start"], x["pitch"])) + return + + if op_name == "update_note": + note_id = str(op.get("note_id", "")) + note = next((n for n in track["notes"] if n["id"] == note_id), None) + if not note: + raise ValueError("note not found") + if "pitch" in op: + note["pitch"] = int(_clamp(float(op["pitch"]), 0, 127)) + if "start" in op: + note["start"] = _clamp(float(op["start"]), 0.0, 86400.0) + if "duration" in op: + note["duration"] = _clamp(float(op["duration"]), 0.001, 60.0) + if "velocity" in op: + note["velocity"] = int(_clamp(float(op["velocity"]), 1, 127)) + track["notes"].sort(key=lambda x: (x["start"], x["pitch"])) + return + + if op_name == "delete_note": + note_id = str(op.get("note_id", "")) + track["notes"] = [n for n in track["notes"] if n["id"] != note_id] + return + + if op_name == "quantize_notes": + step = float(op.get("step", 0.25)) + if step <= 0: + raise ValueError("quantize step must be > 0") + for note in track["notes"]: + note["start"] = round(note["start"] / step) * step + note["duration"] = max(step, round(note["duration"] / step) * step) + track["notes"].sort(key=lambda x: (x["start"], x["pitch"])) + return + + if op_name == "add_bend": + bend = { + "id": uuid4().hex[:10], + "time": _clamp(float(op.get("time", 0.0)), 0.0, 86400.0), + "value": int(_clamp(float(op.get("value", 0)), -8192, 8191)), + } + track["bends"].append(bend) + track["bends"].sort(key=lambda x: x["time"]) + return + + if op_name == "update_bend": + bend_id = str(op.get("bend_id", "")) + bend = next((b for b in track["bends"] if b["id"] == bend_id), None) + if not bend: + raise ValueError("bend not found") + if "time" in op: + bend["time"] = _clamp(float(op["time"]), 0.0, 86400.0) + if "value" in op: + bend["value"] = int(_clamp(float(op["value"]), -8192, 8191)) + track["bends"].sort(key=lambda x: x["time"]) + return + + if op_name == "delete_bend": + bend_id = str(op.get("bend_id", "")) + track["bends"] = [b for b in track["bends"] if b["id"] != bend_id] + return + + raise ValueError(f"unsupported op: {op_name}") + + +def save_session(session_id: str) -> dict[str, Any]: + with _SESSIONS_LOCK: + session = _SESSIONS.get(session_id) + if not session: + raise ValueError("session not found") + + source_path = Path(session["source_path"]) + if not source_path.exists() or not source_path.is_file(): + raise ValueError("source midi file is missing") + + midi = pretty_midi.PrettyMIDI(str(source_path)) + tracks: list[dict[str, Any]] = session["tracks"] + + while len(midi.instruments) < len(tracks): + midi.instruments.append(pretty_midi.Instrument(program=0, is_drum=False, name="New Track")) + + for idx, track in enumerate(tracks): + instrument = midi.instruments[idx] + instrument.program = int(track.get("program", 0)) + instrument.is_drum = bool(track.get("is_drum", False)) + instrument.name = str(track.get("name", instrument.name or f"Track {idx + 1}")) + + notes = [] + for item in track.get("notes", []): + start = float(item["start"]) + duration = float(item["duration"]) + notes.append( + pretty_midi.Note( + velocity=int(item["velocity"]), + pitch=int(item["pitch"]), + start=max(0.0, start), + end=max(0.001, start + duration), + ) + ) + notes.sort(key=lambda x: (x.start, x.pitch)) + + bends = [] + for item in track.get("bends", []): + bends.append( + pretty_midi.PitchBend( + pitch=int(item["value"]), + time=max(0.0, float(item["time"])), + ) + ) + bends.sort(key=lambda x: x.time) + + instrument.notes = notes + instrument.pitch_bends = bends + + edited_name = f"{source_path.stem}.edited.mid" + edited_path = source_path.with_name(edited_name) + suffix = 2 + while edited_path.exists(): + edited_path = source_path.with_name(f"{source_path.stem}.edited.{suffix}.mid") + suffix += 1 + + midi.write(str(edited_path)) + + summary = { + "track_count": len(tracks), + "note_count": sum(len(t.get("notes", [])) for t in tracks), + "bend_count": sum(len(t.get("bends", [])) for t in tracks), + } + + save_job_id = jm.create_job("midi-edit-save") + jm.update_job( + save_job_id, + status=jm.JobStatus.DONE, + progress="MIDI edits saved.", + files=[{"name": edited_path.name, "path": str(edited_path), "type": "midi"}], + result={ + "source_midi_path": str(source_path), + "edited_midi_path": str(edited_path), + "midi_edit_summary": summary, + "midi_editor_version": session.get("editor_version", "stemengine-midi-editor-v1"), + "saved_at": _now_iso(), + }, + ) + + session["last_saved_path"] = str(edited_path) + session["last_saved_job_id"] = save_job_id + + return { + "job_id": save_job_id, + "filename": edited_path.name, + "path": str(edited_path), + "summary": summary, + } \ No newline at end of file diff --git a/backend/services/omnizart_service.py b/backend/services/omnizart_service.py index 17e845e..171750c 100644 --- a/backend/services/omnizart_service.py +++ b/backend/services/omnizart_service.py @@ -36,6 +36,8 @@ import time from pathlib import Path from typing import Optional +import pretty_midi + from .. import job_manager as jm from ..config import JOBS_DIR, OMNIZART_SIDECAR_URL @@ -56,7 +58,7 @@ _POLL_INTERVAL: float = 5.0 # Public entry point # --------------------------------------------------------------------------- -def run_omnizart_transcription(job_id: str, input_path: str, mode_id: str) -> None: +def run_omnizart_transcription(job_id: str, input_path: str, mode_id: str, bend_policy: str = "auto") -> None: """ Background task: transcribe *input_path* to MIDI using Omnizart. @@ -64,16 +66,16 @@ def run_omnizart_transcription(job_id: str, input_path: str, mode_id: str) -> No Routes to sidecar HTTP or local subprocess depending on OMNIZART_SIDECAR_URL. """ if OMNIZART_SIDECAR_URL: - _run_via_sidecar(job_id, input_path, mode_id) + _run_via_sidecar(job_id, input_path, mode_id, bend_policy) else: - _run_via_subprocess(job_id, input_path, mode_id) + _run_via_subprocess(job_id, input_path, mode_id, bend_policy) # --------------------------------------------------------------------------- # Sidecar HTTP path (Phase 3 / production) # --------------------------------------------------------------------------- -def _run_via_sidecar(job_id: str, input_path: str, mode_id: str) -> None: +def _run_via_sidecar(job_id: str, input_path: str, mode_id: str, bend_policy: str) -> None: """ POST the audio file to the omnizart-sidecar HTTP service and save the returned MIDI bytes to the jobs directory. @@ -122,6 +124,7 @@ def _run_via_sidecar(job_id: str, input_path: str, mode_id: str) -> None: dest = output_dir / mid_name dest.write_bytes(response.content) + bend_stats = _postprocess_omnizart_bends(dest, mode, bend_policy) log.debug("[%s] omnizart sidecar complete — output=%s", job_id, dest.name) jm.update_job( @@ -129,6 +132,7 @@ def _run_via_sidecar(job_id: str, input_path: str, mode_id: str) -> None: status=jm.JobStatus.DONE, progress="Transcription complete.", files=[{"name": dest.name, "path": str(dest)}], + result={"bend_stats": bend_stats}, ) except Exception as exc: @@ -140,7 +144,7 @@ def _run_via_sidecar(job_id: str, input_path: str, mode_id: str) -> None: # Local subprocess path (development / Option B) # --------------------------------------------------------------------------- -def _run_via_subprocess(job_id: str, input_path: str, mode_id: str) -> None: +def _run_via_subprocess(job_id: str, input_path: str, mode_id: str, bend_policy: str) -> None: """ Convert the input to WAV, invoke `omnizart {mode} transcribe ...` as a subprocess, capture the output .mid file and store it in the jobs directory. @@ -208,12 +212,15 @@ def _run_via_subprocess(job_id: str, input_path: str, mode_id: str) -> None: dest = output_dir / mid_name dest.write_bytes(expected_output.read_bytes()) + bend_stats = _postprocess_omnizart_bends(dest, mode, bend_policy) + log.debug("[%s] omnizart subprocess complete — output=%s", job_id, dest.name) jm.update_job( job_id, status=jm.JobStatus.DONE, progress="Transcription complete.", files=[{"name": dest.name, "path": str(dest)}], + result={"bend_stats": bend_stats}, ) except Exception as exc: @@ -253,3 +260,135 @@ def _convert_to_wav(src: Path, dst: Path) -> None: def _fail(job_id: str, message: str) -> None: jm.update_job(job_id, status=jm.JobStatus.ERROR, error=message, progress="Error.") + + +def _postprocess_omnizart_bends(mid_path: Path, mode: str, bend_policy: str) -> dict: + midi = pretty_midi.PrettyMIDI(str(mid_path)) + native_bend_count = sum(len(inst.pitch_bends) for inst in midi.instruments) + + stats = { + "engine": "omnizart", + "mode": mode, + "bend_policy": bend_policy, + "native_bend_count": int(native_bend_count), + "derived_bend_count": 0, + "bend_event_count": int(native_bend_count), + "derivation_applied": False, + "derivation_reason": "native_bends_present" if native_bend_count > 0 else "not_requested", + } + + if native_bend_count > 0: + return stats + + derive = False + if bend_policy == "force_on": + derive = True + stats["derivation_reason"] = "forced_on" + elif bend_policy == "force_off": + derive = False + stats["derivation_reason"] = "forced_off" + else: + derive = mode == "vocal-contour" + stats["derivation_reason"] = "auto_vocal_contour" if derive else "auto_opt_in_required" + + if not derive: + return stats + + if _is_polyphonic_dense(midi): + stats["derivation_reason"] = "guard_dense_polyphonic" + return stats + + added = 0 + for instrument in midi.instruments: + if instrument.is_drum: + continue + if instrument.pitch_bends: + continue + derived = _derive_bends_from_notes(instrument.notes, mode) + if derived: + instrument.pitch_bends = derived + added += len(derived) + + if added > 0: + midi.write(str(mid_path)) + + stats["derived_bend_count"] = int(added) + stats["bend_event_count"] = int(native_bend_count + added) + stats["derivation_applied"] = added > 0 + if added == 0 and stats["derivation_reason"] in {"forced_on", "auto_vocal_contour"}: + stats["derivation_reason"] = "no_eligible_note_transitions" + + return stats + + +def _is_polyphonic_dense(midi: pretty_midi.PrettyMIDI) -> bool: + events: list[tuple[float, int]] = [] + for instrument in midi.instruments: + if instrument.is_drum: + continue + for note in instrument.notes: + events.append((float(note.start), 1)) + events.append((float(note.end), -1)) + + if not events: + return False + + events.sort(key=lambda item: (item[0], -item[1])) + active = 0 + max_active = 0 + dense_time = 0.0 + prev_t = events[0][0] + + for t, delta in events: + dt = max(0.0, t - prev_t) + if active >= 3: + dense_time += dt + active += delta + max_active = max(max_active, active) + prev_t = t + + total_time = max(0.001, events[-1][0] - events[0][0]) + dense_ratio = dense_time / total_time + return max_active >= 4 or dense_ratio > 0.35 + + +def _derive_bends_from_notes(notes: list[pretty_midi.Note], mode: str) -> list[pretty_midi.PitchBend]: + if len(notes) < 2: + return [] + + sorted_notes = sorted(notes, key=lambda n: (float(n.start), float(n.pitch))) + bend_range_semitones = 2.0 + max_delta_semitones = 6.0 if mode == "vocal-contour" else 2.5 + bends: list[pretty_midi.PitchBend] = [pretty_midi.PitchBend(pitch=0, time=max(0.0, float(sorted_notes[0].start)))] + + for idx in range(1, len(sorted_notes)): + prev_note = sorted_notes[idx - 1] + note = sorted_notes[idx] + gap = float(note.start) - float(prev_note.end) + if gap > 0.20: + continue + + semitone_delta = float(note.pitch) - float(prev_note.pitch) + if abs(semitone_delta) < 0.25: + continue + + semitone_delta = max(-max_delta_semitones, min(max_delta_semitones, semitone_delta)) + midi_value = int(max(-8192, min(8191, round((semitone_delta / bend_range_semitones) * 8192)))) + if midi_value == 0: + continue + + transition_time = max(float(prev_note.start), float(prev_note.end) - 0.035) + settle_time = min(float(note.end), float(note.start) + 0.060) + bends.append(pretty_midi.PitchBend(pitch=midi_value, time=max(0.0, transition_time))) + bends.append(pretty_midi.PitchBend(pitch=0, time=max(0.0, settle_time))) + + tail = max(float(sorted_notes[-1].end), 0.0) + bends.append(pretty_midi.PitchBend(pitch=0, time=tail)) + + # Keep only one bend per timestamp (last event wins) and sorted ordering. + merged: dict[float, int] = {} + for bend in bends: + key = round(float(bend.time), 4) + merged[key] = int(max(-8192, min(8191, bend.pitch))) + + return [pretty_midi.PitchBend(pitch=value, time=time) for time, value in sorted(merged.items())] diff --git a/backend/services/project_reprocess_service.py b/backend/services/project_reprocess_service.py index 27aa09d..6cbac69 100644 --- a/backend/services/project_reprocess_service.py +++ b/backend/services/project_reprocess_service.py @@ -70,7 +70,7 @@ def enqueue_stem_reprocess( return {"job_id": job_id} -def enqueue_project_transcription(project: dict, stem_name: str, mode: str) -> dict: +def enqueue_project_transcription(project: dict, stem_name: str, mode: str, bend_policy: str = "auto") -> dict: """Queue stem transcription for a project and persist the stem→job link.""" valid_modes = {mode_entry["id"] for mode_entry in TRANSCRIBE_MODES} if mode not in valid_modes: @@ -88,7 +88,7 @@ def enqueue_project_transcription(project: dict, stem_name: str, mode: str) -> d job_id = jm.create_job("transcribe") projects_repo.set_transcription_job(project["id"], stem_name, job_id) - jm.submit_gpu(run_transcription, job_id, stem_path, mode) + jm.submit_gpu(run_transcription, job_id, stem_path, mode, bend_policy) return {"job_id": job_id} diff --git a/backend/services/separator_service.py b/backend/services/separator_service.py index e39bf51..f2ac615 100755 --- a/backend/services/separator_service.py +++ b/backend/services/separator_service.py @@ -1493,6 +1493,16 @@ def _run_demucs( elif ext == 'flac': fmt_flag = ['--flac'] + demucs_device = 'cpu' + if not CPU_SEPARATION: + try: + import torch + + if torch.cuda.is_available() and torch.cuda.device_count() > 0: + demucs_device = 'cuda' + except Exception: + demucs_device = 'cpu' + # Determine segment size: check env override first, then auto-scale by duration _dur = get_audio_duration_secs(input_path) _use_long_track_tuning = should_chunk(_dur, LONG_TRACK_CHUNKING_MODE, LONG_TRACK_DURATION_SECS) @@ -1526,6 +1536,7 @@ def _run_demucs( cmd = [ sys.executable, '-m', 'demucs', '--name', model, + '--device', demucs_device, '--out', str(temp_out), *fmt_flag, *power_flags, @@ -1537,10 +1548,12 @@ def _run_demucs( jm.update_job( job_id, progress=( - f"Long-track tuning: segment={_segment}s, overlap={_overlap or 0.25}x " + f"Long-track tuning: segment={_segment}s, overlap={_overlap or 0.25}x, device={demucs_device} " f"(duration={_dur:.0f}s)..." ), ) + else: + jm.update_job(job_id, progress=f"Demucs device: {demucs_device}...") process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, @@ -1563,6 +1576,9 @@ def _run_demucs( line = process.stdout.readline() if line: jm.update_job(job_id, progress=line.rstrip()) + elif process.poll() is not None: + # EOF with a finished subprocess: stop polling. + break elif process.poll() is not None: break diff --git a/backend/services/sloppak_converter_service.py b/backend/services/sloppak_converter_service.py new file mode 100644 index 0000000..6f5cb6a --- /dev/null +++ b/backend/services/sloppak_converter_service.py @@ -0,0 +1,751 @@ +from __future__ import annotations + +import json +import logging +import os +import shlex +import shutil +import subprocess +import tempfile +import threading +import time +import zipfile +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +import xml.etree.ElementTree as ET + +from .. import job_manager as jm +from ..config import MEDIA_DIR + + +log = logging.getLogger(__name__) + +_PAUSE_EVENT = threading.Event() +_PAUSE_EVENT.set() + +_STATE_LOCK = threading.Lock() +_QUEUE_HYDRATED = False +_ACTIVE_SUBMISSIONS: set[str] = set() + +_SOURCE_ROOT = Path(os.environ.get("SLOPPAK_SOURCE_ROOT", str(MEDIA_DIR / "uploads"))).resolve() +_OUTPUT_ROOT = Path(os.environ.get("SLOPPAK_OUTPUT_ROOT", str(MEDIA_DIR / "sloppak"))).resolve() + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _normalize_rel_psarc(filename: str) -> str: + rel = (filename or "").strip().replace("\\", "/") + if not rel: + raise ValueError("filename is required") + rel_path = Path(rel) + if rel_path.is_absolute() or ".." in rel_path.parts: + raise ValueError("filename must be a relative path under source root") + if rel_path.suffix.lower() != ".psarc": + raise ValueError("filename must end with .psarc") + return rel_path.as_posix() + + +def _resolve_source_file(rel_psarc: str) -> Path: + source = (_SOURCE_ROOT / rel_psarc).resolve() + source.relative_to(_SOURCE_ROOT) + return source + + +def _output_path_for(rel_psarc: str) -> Path: + rel = Path(rel_psarc) + return (_OUTPUT_ROOT / rel).with_suffix(".sloppak") + + +def _job_source_filename(job: dict) -> str: + result = job.get("result") if isinstance(job.get("result"), dict) else {} + return str(result.get("source_filename", "")).strip() + + +def list_conversion_jobs() -> list[dict]: + jobs = [j for j in jm.list_all_jobs() if j.get("job_type") == "sloppak_convert"] + jobs.sort(key=lambda x: x.get("created_at") or "", reverse=True) + return jobs + + +def get_snapshot() -> dict[str, Any]: + ensure_queue_hydrated() + return { + "paused": is_paused(), + "jobs": list_conversion_jobs(), + "capabilities": get_converter_capabilities(), + } + + +def stream_updates(*, poll_interval_sec: float = 1.0): + ensure_queue_hydrated() + last_payload = "" + # Retry hint helps reconnect logic recover quickly after restarts/network blips. + yield "retry: 1500\n\n" + while True: + payload = json.dumps(get_snapshot(), ensure_ascii=True, sort_keys=True) + if payload != last_payload: + last_payload = payload + yield f"event: snapshot\ndata: {payload}\n\n" + time.sleep(max(0.2, float(poll_interval_sec))) + + +def is_paused() -> bool: + return not _PAUSE_EVENT.is_set() + + +def get_converter_capabilities() -> dict[str, Any]: + ensure_queue_hydrated() + unpack = _get_unpacker_capability() + return { + "source_root": str(_SOURCE_ROOT), + "output_root": str(_OUTPUT_ROOT), + "source_root_exists": _SOURCE_ROOT.exists(), + "output_root_exists": _OUTPUT_ROOT.exists(), + "unpacker": unpack, + "ready": bool(unpack.get("available")), + } + + +def enqueue_conversion(filename: str, *, reconvert: bool = False) -> dict: + ensure_queue_hydrated() + try: + rel_psarc = _normalize_rel_psarc(filename) + except ValueError as exc: + return {"skipped": True, "filename": filename, "reason": str(exc)} + + try: + source_file = _resolve_source_file(rel_psarc) + except Exception: + return {"skipped": True, "filename": rel_psarc, "reason": "not_found"} + + if not source_file.is_file(): + return {"skipped": True, "filename": rel_psarc, "reason": "not_found"} + + output_path = _output_path_for(rel_psarc) + if output_path.exists() and not reconvert: + return { + "skipped": True, + "filename": rel_psarc, + "reason": "already_converted", + "output_path": str(output_path), + } + + for existing in list_conversion_jobs(): + if _job_source_filename(existing) != rel_psarc: + continue + status = existing.get("status") + if status == "queued": + return { + "skipped": True, + "filename": rel_psarc, + "reason": "already_queued", + "job_id": existing.get("id"), + } + if status == "running": + return { + "skipped": True, + "filename": rel_psarc, + "reason": "already_running", + "job_id": existing.get("id"), + } + + job_id = jm.create_job("sloppak_convert") + jm.update_job( + job_id, + progress="Queued for conversion.", + result={ + "source_filename": rel_psarc, + "output_path": str(output_path), + "conversion_kind": "psarc_to_sloppak", + "queued_at": _now_iso(), + }, + ) + _submit_conversion(job_id, rel_psarc) + return {"ok": True, "filename": rel_psarc, "job_id": job_id} + + +def enqueue_bulk(filenames: list[str], *, reconvert: bool = False) -> dict: + ensure_queue_hydrated() + enqueued: list[dict] = [] + skipped: list[dict] = [] + existing_jobs = list_conversion_jobs() + existing_status_by_source: dict[str, str] = {} + for job in existing_jobs: + src = _job_source_filename(job) + status = str(job.get("status") or "") + if src and status in {"queued", "running"}: + existing_status_by_source[src] = status + + use_threaded_preflight = len(filenames) >= 40 + if use_threaded_preflight: + max_workers = max(2, min(8, (os.cpu_count() or 4))) + with ThreadPoolExecutor(max_workers=max_workers) as pool: + preflight_results = list( + pool.map( + lambda item: _preflight_filename(item, reconvert=reconvert, existing_status=existing_status_by_source), + filenames, + ) + ) + else: + preflight_results = [ + _preflight_filename(item, reconvert=reconvert, existing_status=existing_status_by_source) + for item in filenames + ] + + for preflight in preflight_results: + if preflight.get("ok"): + rel_psarc = str(preflight["filename"]) + res = enqueue_conversion(rel_psarc, reconvert=reconvert) + if res.get("ok"): + enqueued.append(res) + else: + skipped.append(res) + else: + skipped.append(preflight) + + return { + "enqueued": enqueued, + "skipped": skipped, + "summary": { + "requested": len(filenames), + "enqueued": len(enqueued), + "skipped": len(skipped), + }, + } + + +def pause_queue() -> dict: + ensure_queue_hydrated() + _PAUSE_EVENT.clear() + return {"paused": True} + + +def resume_queue() -> dict: + ensure_queue_hydrated() + _PAUSE_EVENT.set() + return {"paused": False} + + +def cancel_queued() -> dict: + ensure_queue_hydrated() + count = 0 + for job in list_conversion_jobs(): + if job.get("status") == "queued": + jm.cancel_job(job["id"]) + count += 1 + return {"cancelled": count} + + +def clear_finished() -> dict: + ensure_queue_hydrated() + deleted = 0 + for job in list_conversion_jobs(): + if job.get("status") in {"done", "error", "cancelled"}: + if jm.delete_job(job["id"]): + deleted += 1 + return {"deleted": deleted} + + +def retry_failed() -> dict: + ensure_queue_hydrated() + failed_jobs = [j for j in list_conversion_jobs() if j.get("status") == "error"] + retried = 0 + skipped = 0 + for job in failed_jobs: + source_filename = _job_source_filename(job) + if not source_filename: + skipped += 1 + continue + response = enqueue_conversion(source_filename, reconvert=True) + if response.get("ok"): + retried += 1 + else: + skipped += 1 + return {"retried": retried, "skipped": skipped} + + +def _run_convert_job(job_id: str, rel_psarc: str) -> None: + try: + while not _PAUSE_EVENT.is_set(): + if jm.should_cancel(job_id): + jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.") + return + jm.update_job(job_id, progress="Paused.") + time.sleep(0.25) + + jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Preparing conversion...") + source_file = _resolve_source_file(rel_psarc) + if not source_file.is_file(): + jm.update_job( + job_id, + status=jm.JobStatus.ERROR, + progress="Error.", + error=f"Source file not found: {rel_psarc}", + ) + return + + if jm.should_cancel(job_id): + jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.") + return + + output_path = _output_path_for(rel_psarc) + output_path.parent.mkdir(parents=True, exist_ok=True) + _build_sloppak_archive(job_id, source_file, output_path, rel_psarc) + + result = { + "source_filename": rel_psarc, + "output_path": str(output_path), + "conversion_kind": "psarc_to_sloppak", + "finished_at": _now_iso(), + "output_size": output_path.stat().st_size, + } + jm.update_job( + job_id, + status=jm.JobStatus.DONE, + progress="Conversion complete.", + files=[{"name": output_path.name, "path": str(output_path), "type": "sloppak"}], + result=result, + ) + except Exception as exc: + log.exception("sloppak conversion failed for job %s", job_id) + jm.update_job(job_id, status=jm.JobStatus.ERROR, progress="Error.", error=str(exc)) + + +def ensure_queue_hydrated() -> dict[str, int]: + global _QUEUE_HYDRATED + with _STATE_LOCK: + if _QUEUE_HYDRATED: + return {"resubmitted": 0, "requeued_running": 0} + + jobs = list_conversion_jobs() + resubmitted = 0 + requeued_running = 0 + for job in jobs: + status = str(job.get("status") or "") + if status not in {"queued", "running"}: + continue + + rel_psarc = _job_source_filename(job) + job_id = str(job.get("id") or "") + if not rel_psarc or not job_id: + continue + + if status == "running": + requeued_running += 1 + jm.update_job( + job_id, + status=jm.JobStatus.QUEUED, + progress="Recovered after restart; re-queued.", + ) + + if _submit_conversion(job_id, rel_psarc): + resubmitted += 1 + + with _STATE_LOCK: + _QUEUE_HYDRATED = True + if resubmitted or requeued_running: + log.info( + "sloppak queue hydrated: resubmitted=%s requeued_running=%s", + resubmitted, + requeued_running, + ) + return {"resubmitted": resubmitted, "requeued_running": requeued_running} + + +def _submit_conversion(job_id: str, rel_psarc: str) -> bool: + with _STATE_LOCK: + if job_id in _ACTIVE_SUBMISSIONS: + return False + _ACTIVE_SUBMISSIONS.add(job_id) + + def _runner() -> None: + try: + _run_convert_job(job_id, rel_psarc) + finally: + with _STATE_LOCK: + _ACTIVE_SUBMISSIONS.discard(job_id) + + jm.submit(_runner) + return True + + +def _preflight_filename( + filename: str, + *, + reconvert: bool, + existing_status: dict[str, str], +) -> dict[str, Any]: + try: + rel_psarc = _normalize_rel_psarc(filename) + except ValueError as exc: + return {"skipped": True, "filename": filename, "reason": str(exc)} + + status = existing_status.get(rel_psarc) + if status == "queued": + return {"skipped": True, "filename": rel_psarc, "reason": "already_queued"} + if status == "running": + return {"skipped": True, "filename": rel_psarc, "reason": "already_running"} + + try: + source_file = _resolve_source_file(rel_psarc) + except Exception: + return {"skipped": True, "filename": rel_psarc, "reason": "not_found"} + + if not source_file.is_file(): + return {"skipped": True, "filename": rel_psarc, "reason": "not_found"} + + output_path = _output_path_for(rel_psarc) + if output_path.exists() and not reconvert: + return { + "skipped": True, + "filename": rel_psarc, + "reason": "already_converted", + "output_path": str(output_path), + } + + return {"ok": True, "filename": rel_psarc} + + +def _build_sloppak_archive(job_id: str, source_file: Path, output_path: Path, rel_psarc: str) -> None: + jm.update_job(job_id, progress="Analyzing Rocksmith payload...") + rocksmith = _extract_rocksmith_context(source_file) + + manifest = { + "format": "sloppak", + "version": 1, + "metadata": { + "title": rocksmith.get("song", {}).get("title") or Path(rel_psarc).stem, + "artist": rocksmith.get("song", {}).get("artist") or "", + "album": rocksmith.get("song", {}).get("album") or "", + "year": rocksmith.get("song", {}).get("year") or 0, + "source_file": rel_psarc, + "converted_at": _now_iso(), + "converter": "stemengine.slice_ab", + }, + "contents": { + "source_psarc": f"source/{Path(rel_psarc).name}", + }, + "rocksmith": { + "unpack": { + "succeeded": bool(rocksmith.get("unpack_succeeded")), + "method": rocksmith.get("unpack_method"), + "error": rocksmith.get("unpack_error"), + }, + "arrangements": rocksmith.get("arrangements", []), + "audio": { + "wem_count": len(rocksmith.get("wem_files", [])), + "wem_files": rocksmith.get("wem_files", []), + }, + }, + } + + with tempfile.TemporaryDirectory(prefix="stemengine_sloppak_") as tmp: + tmp_dir = Path(tmp) + src_dir = tmp_dir / "source" + meta_dir = tmp_dir / "metadata" + src_dir.mkdir(parents=True, exist_ok=True) + meta_dir.mkdir(parents=True, exist_ok=True) + staged_psarc = src_dir / Path(rel_psarc).name + staged_psarc.write_bytes(source_file.read_bytes()) + + lyrics = rocksmith.get("lyrics", []) + if lyrics: + lyrics_path = tmp_dir / "lyrics.json" + lyrics_path.write_text(json.dumps(lyrics, ensure_ascii=True, indent=2), encoding="utf-8") + manifest["contents"]["lyrics"] = "lyrics.json" + manifest["metadata"]["lyrics_source"] = "xml-vocals" + + summary_path = meta_dir / "rocksmith-summary.json" + summary_path.write_text( + json.dumps( + { + "song": rocksmith.get("song", {}), + "arrangements": rocksmith.get("arrangements", []), + "wem_files": rocksmith.get("wem_files", []), + }, + ensure_ascii=True, + indent=2, + ), + encoding="utf-8", + ) + manifest["contents"]["metadata"] = { + "summary": "metadata/rocksmith-summary.json", + } + + manifest_path = tmp_dir / "manifest.json" + manifest_path.write_text(json.dumps(manifest, ensure_ascii=True, indent=2), encoding="utf-8") + + tmp_output = output_path.with_suffix(output_path.suffix + ".tmp") + with zipfile.ZipFile(tmp_output, "w", compression=zipfile.ZIP_DEFLATED) as zf: + zf.write(manifest_path, "manifest.json") + zf.write(staged_psarc, f"source/{staged_psarc.name}") + if lyrics: + zf.write(tmp_dir / "lyrics.json", "lyrics.json") + zf.write(summary_path, "metadata/rocksmith-summary.json") + tmp_output.replace(output_path) + + +def _extract_rocksmith_context(source_file: Path) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="stemengine_psarc_extract_") as tmp: + extract_dir = Path(tmp) + unpack_succeeded = _try_unpack_psarc(source_file, extract_dir) + + context: dict[str, Any] = { + "unpack_succeeded": unpack_succeeded.get("ok", False), + "unpack_method": unpack_succeeded.get("method"), + "unpack_error": unpack_succeeded.get("error"), + "song": { + "title": "", + "artist": "", + "album": "", + "year": 0, + }, + "arrangements": [], + "lyrics": [], + "wem_files": [], + } + + if not unpack_succeeded.get("ok"): + return context + + context["song"], context["arrangements"] = _extract_song_and_arrangements(extract_dir) + context["lyrics"] = _extract_lyrics(extract_dir) + context["wem_files"] = sorted( + [p.relative_to(extract_dir).as_posix() for p in extract_dir.rglob("*.wem")] + ) + return context + + +def _try_unpack_psarc(source_file: Path, extract_dir: Path) -> dict[str, Any]: + # Preferred path when available: Slopsmith-compatible Python unpacker. + try: + from patcher import unpack_psarc as patcher_unpack_psarc # type: ignore[import-not-found] + + patcher_unpack_psarc(str(source_file), str(extract_dir)) + return {"ok": True, "method": "python-patcher"} + except Exception: + pass + + # Optional CLI path: user can provide explicit unpack command template. + # Template supports {input} and {output} placeholders. + # Example: SLOPPAK_PSARC_UNPACK_CMD="/usr/local/bin/psarc unpack {input} {output}" + raw_cmd = (os.environ.get("SLOPPAK_PSARC_UNPACK_CMD") or "").strip() + if not raw_cmd: + return { + "ok": False, + "method": None, + "error": "No PSARC unpacker available (patcher module or SLOPPAK_PSARC_UNPACK_CMD).", + } + + try: + rendered = raw_cmd.format(input=str(source_file), output=str(extract_dir)) + args = shlex.split(rendered) + run = subprocess.run(args, capture_output=True, text=True) + if run.returncode != 0: + return { + "ok": False, + "method": "env-command", + "error": (run.stderr or run.stdout or "psarc unpack command failed").strip()[:500], + } + return {"ok": True, "method": "env-command"} + except Exception as exc: + return {"ok": False, "method": "env-command", "error": str(exc)} + + +def _get_unpacker_capability() -> dict[str, Any]: + try: + from patcher import unpack_psarc as _unused # type: ignore[import-not-found] + + return { + "available": True, + "method": "python-patcher", + "detail": "Using patcher.unpack_psarc", + } + except Exception: + pass + + raw_cmd = (os.environ.get("SLOPPAK_PSARC_UNPACK_CMD") or "").strip() + if not raw_cmd: + return { + "available": False, + "method": None, + "detail": "Set SLOPPAK_PSARC_UNPACK_CMD or install patcher module.", + } + + try: + rendered = raw_cmd.format(input="/path/to/source.psarc", output="/path/to/extract") + args = shlex.split(rendered) + if not args: + return { + "available": False, + "method": "env-command", + "detail": "SLOPPAK_PSARC_UNPACK_CMD is empty after template rendering.", + } + + binary = args[0] + binary_exists = os.path.isabs(binary) and Path(binary).exists() or bool(shutil.which(binary)) + return { + "available": bool(binary_exists), + "method": "env-command", + "detail": "Using SLOPPAK_PSARC_UNPACK_CMD template.", + "binary": binary, + "binary_found": bool(binary_exists), + } + except Exception as exc: + return { + "available": False, + "method": "env-command", + "detail": f"Invalid SLOPPAK_PSARC_UNPACK_CMD: {exc}", + } + + +def _extract_song_and_arrangements(extract_dir: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]: + song: dict[str, Any] = { + "title": "", + "artist": "", + "album": "", + "year": 0, + } + arrangements: list[dict[str, Any]] = [] + + json_files = sorted(extract_dir.rglob("*.json")) + for file_path in json_files: + if file_path.stat().st_size > 5_000_000: + continue + payload = _read_json(file_path) + if not isinstance(payload, dict): + continue + + _merge_song_fields(song, payload) + _append_arrangements(arrangements, payload, file_path) + + _dedupe_arrangements(arrangements) + return song, arrangements + + +def _read_json(path: Path) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8", errors="replace")) + except Exception: + return None + + +def _merge_song_fields(song: dict[str, Any], payload: dict[str, Any]) -> None: + def pick(*keys: str) -> str: + for key in keys: + value = payload.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return "" + + title = pick("title", "Title", "SongName") + artist = pick("artist", "ArtistName", "artistName") + album = pick("album", "AlbumName", "albumName") + year_raw = payload.get("SongYear") or payload.get("year") or payload.get("Year") + + if title and not song["title"]: + song["title"] = title + if artist and not song["artist"]: + song["artist"] = artist + if album and not song["album"]: + song["album"] = album + if not song["year"]: + try: + year = int(year_raw) + if 1900 <= year <= 2200: + song["year"] = year + except Exception: + pass + + +def _append_arrangements(arrangements: list[dict[str, Any]], payload: dict[str, Any], path: Path) -> None: + candidates: list[dict[str, Any]] = [] + if isinstance(payload.get("Arrangements"), list): + for item in payload["Arrangements"]: + if isinstance(item, dict): + candidates.append(item) + if payload.get("ArrangementProperties") or payload.get("Tuning"): + candidates.append(payload) + + for item in candidates: + name = _first_text(item, "name", "Name", "ArrangementName", "arrangementName") + if not name: + base = path.stem + if base: + name = base + tuning = item.get("Tuning") or item.get("tuning") or [] + if not isinstance(tuning, list): + tuning = [] + capo = item.get("Capo") + if capo is None: + capo = item.get("capo") + try: + capo_value = int(capo) if capo is not None else 0 + except Exception: + capo_value = 0 + + normalized_tuning: list[int] = [] + for value in tuning: + try: + normalized_tuning.append(int(value)) + except Exception: + continue + + arrangements.append( + { + "id": _first_text(item, "PersistentID", "persistentID") or name.lower().replace(" ", "_"), + "name": name, + "tuning": normalized_tuning[:6], + "capo": capo_value, + } + ) + + +def _first_text(payload: dict[str, Any], *keys: str) -> str: + for key in keys: + value = payload.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return "" + + +def _dedupe_arrangements(arrangements: list[dict[str, Any]]) -> None: + seen: set[tuple[str, str]] = set() + deduped: list[dict[str, Any]] = [] + for item in arrangements: + key = (str(item.get("id", "")), str(item.get("name", ""))) + if key in seen: + continue + seen.add(key) + deduped.append(item) + arrangements.clear() + arrangements.extend(deduped) + + +def _extract_lyrics(extract_dir: Path) -> list[dict[str, Any]]: + lyrics: list[dict[str, Any]] = [] + for xml_path in sorted(extract_dir.rglob("*.xml")): + try: + root = ET.parse(xml_path).getroot() + except Exception: + continue + if root.tag != "vocals": + continue + for vocal in root.findall("vocal"): + try: + t = float(vocal.get("time", "0")) + except Exception: + t = 0.0 + try: + d = float(vocal.get("length", "0")) + except Exception: + d = 0.0 + word = (vocal.get("lyric") or "").strip() + if not word: + continue + lyrics.append({"t": round(t, 3), "d": round(d, 3), "w": word}) + return lyrics \ No newline at end of file diff --git a/backend/services/transcriber_service.py b/backend/services/transcriber_service.py index 6b2c97c..e535932 100755 --- a/backend/services/transcriber_service.py +++ b/backend/services/transcriber_service.py @@ -6,7 +6,9 @@ Routes to the appropriate engine based on the mode prefix: 'omnizart-*' → Omnizart CLI or sidecar HTTP (requires OMNIZART_ENABLED=true) """ import logging +from statistics import mean from pathlib import Path +from typing import Any from .. import job_manager as jm from ..config import JOBS_DIR, TRANSCRIBE_PRESETS, OMNIZART_ENABLED @@ -14,7 +16,10 @@ from ..config import JOBS_DIR, TRANSCRIBE_PRESETS, OMNIZART_ENABLED log = logging.getLogger(__name__) -def run_transcription(job_id: str, input_path: str, mode: str) -> None: +_BEND_POLICY_VALUES = {"auto", "force_on", "force_off"} + + +def run_transcription(job_id: str, input_path: str, mode: str, bend_policy: str = "auto") -> None: """ Background task: transcribe audio to MIDI. @@ -23,6 +28,8 @@ def run_transcription(job_id: str, input_path: str, mode: str) -> None: 'omnizart-vocal' → Omnizart vocal mode 'omnizart-vocal-contour' → Omnizart vocal-contour mode """ + resolved_bend_policy = _normalize_bend_policy(bend_policy) + if mode.startswith("omnizart-"): if not OMNIZART_ENABLED: jm.update_job( @@ -33,16 +40,16 @@ def run_transcription(job_id: str, input_path: str, mode: str) -> None: ) return from .omnizart_service import run_omnizart_transcription - run_omnizart_transcription(job_id, input_path, mode) + run_omnizart_transcription(job_id, input_path, mode, resolved_bend_policy) else: - _run_basic_pitch(job_id, input_path, mode) + _run_basic_pitch(job_id, input_path, mode, resolved_bend_policy) # --------------------------------------------------------------------------- # basic-pitch (original engine — unchanged) # --------------------------------------------------------------------------- -def _run_basic_pitch(job_id: str, input_path: str, mode: str) -> None: +def _run_basic_pitch(job_id: str, input_path: str, mode: str, bend_policy: str) -> None: # Reuse global GPU safety limits (VRAM reserve + TF32 settings) before # loading any transcription backend for basic-pitch. try: @@ -66,15 +73,30 @@ def _run_basic_pitch(job_id: str, input_path: str, mode: str) -> None: jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.") return - params = TRANSCRIBE_PRESETS.get(mode, TRANSCRIBE_PRESETS["polyphonic"]) + params = dict(TRANSCRIBE_PRESETS.get(mode, TRANSCRIBE_PRESETS["polyphonic"])) + # Always infer with bends enabled for adaptive modes; policy is applied + # after prediction by either keeping or stripping bend events. + if mode in {"polyphonic", "melody"}: + params["multiple_pitch_bends"] = True jm.update_job(job_id, progress=f"Transcribing with basic-pitch ({mode})...") - _, midi_data, _ = predict( + _, midi_data, note_events = predict( input_path, ICASSP_2022_MODEL_PATH, **params, ) + confidence_summary = _summarize_confidence(note_events) + stability = _estimate_note_stability(midi_data) + allow_bends, decision = _resolve_basic_pitch_bend_policy( + mode=mode, + bend_policy=bend_policy, + confidence_summary=confidence_summary, + note_stability=stability, + ) + if not allow_bends: + _strip_pitch_bends(midi_data) + if jm.should_cancel(job_id): jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.") return @@ -83,14 +105,121 @@ def _run_basic_pitch(job_id: str, input_path: str, mode: str) -> None: dest = output_dir / output_name midi_data.write(str(dest)) + bend_event_count = _count_pitch_bends(midi_data) + bend_stats = { + "engine": "basic-pitch", + "mode": mode, + "bend_policy": bend_policy, + "bend_policy_decision": decision, + "bend_event_count": bend_event_count, + "bend_confidence_summary": confidence_summary, + "segment_stability": stability, + "bends_enabled": bool(allow_bends), + } + log.debug("[%s] basic-pitch complete — output=%s", job_id, dest.name) jm.update_job( job_id, status=jm.JobStatus.DONE, progress="Transcription complete.", files=[{"name": dest.name, "path": str(dest)}], + result={"bend_stats": bend_stats}, ) except Exception as exc: log.debug("[%s] basic-pitch failed — %s: %s", job_id, type(exc).__name__, exc) jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.") + + +def _normalize_bend_policy(value: str | None) -> str: + cleaned = str(value or "auto").strip().lower() + return cleaned if cleaned in _BEND_POLICY_VALUES else "auto" + + +def _summarize_confidence(note_events: Any) -> dict[str, Any]: + values: list[float] = [] + if isinstance(note_events, list): + for item in note_events: + if isinstance(item, dict): + for key in ("confidence", "amplitude", "velocity", "score"): + raw = item.get(key) + if isinstance(raw, (int, float)): + values.append(float(raw)) + break + elif isinstance(item, (tuple, list)) and item: + tail = item[-1] + if isinstance(tail, (int, float)): + values.append(float(tail)) + + if not values: + return { + "count": 0, + "avg": None, + "min": None, + "max": None, + } + + return { + "count": len(values), + "avg": float(mean(values)), + "min": float(min(values)), + "max": float(max(values)), + } + + +def _estimate_note_stability(midi_data: Any) -> float: + deltas: list[float] = [] + for instrument in getattr(midi_data, "instruments", []): + notes = sorted(getattr(instrument, "notes", []) or [], key=lambda n: float(getattr(n, "start", 0.0))) + for idx in range(1, len(notes)): + prev = notes[idx - 1] + cur = notes[idx] + prev_pitch = float(getattr(prev, "pitch", 0.0)) + cur_pitch = float(getattr(cur, "pitch", 0.0)) + deltas.append(abs(cur_pitch - prev_pitch)) + + if not deltas: + return 0.0 + + # 1.0 = highly stable, 0.0 = highly unstable. + avg_delta = float(mean(deltas)) + return max(0.0, min(1.0, 1.0 - (avg_delta / 12.0))) + + +def _resolve_basic_pitch_bend_policy( + *, + mode: str, + bend_policy: str, + confidence_summary: dict[str, Any], + note_stability: float, +) -> tuple[bool, str]: + if bend_policy == "force_on": + return True, "forced_on" + if bend_policy == "force_off": + return False, "forced_off" + + if mode in {"guitar", "bass"}: + return True, "auto_guitar_bass" + if mode == "monophonic": + return False, "auto_monophonic" + + confidence_avg = confidence_summary.get("avg") + enough_confidence = isinstance(confidence_avg, (int, float)) and float(confidence_avg) >= 0.45 + stable_segments = note_stability >= 0.55 + if enough_confidence and stable_segments: + return True, "auto_adaptive_on" + return False, "auto_adaptive_off" + + +def _strip_pitch_bends(midi_data: Any) -> None: + for instrument in getattr(midi_data, "instruments", []): + if hasattr(instrument, "pitch_bends"): + instrument.pitch_bends = [] + + +def _count_pitch_bends(midi_data: Any) -> int: + count = 0 + for instrument in getattr(midi_data, "instruments", []): + bends = getattr(instrument, "pitch_bends", []) or [] + count += len(bends) + return count diff --git a/docs/plans/2026-07-08_StemEngine_Windows_x64_Installable_Executable_Plan.md b/docs/plans/2026-07-08_StemEngine_Windows_x64_Installable_Executable_Plan.md new file mode 100644 index 0000000..88cc950 --- /dev/null +++ b/docs/plans/2026-07-08_StemEngine_Windows_x64_Installable_Executable_Plan.md @@ -0,0 +1,279 @@ +# StemEngine Windows x64 Installable Executable Plan (Jellyfin-style Network Host) + +Date: 2026-07-08 +Owner: StemEngine +Status: proposed + +## 1. Goal + +Ship StemEngine as an installable Windows x64 application that: + +- runs as a local network-accessible service (LAN reachable), similar to Jellyfin's server model, +- hosts the Web UI from the same local install, +- installs/upgrades cleanly with an installer, +- supports CPU and NVIDIA GPU execution modes, +- is observable and recoverable by non-developer users. + +Scope clarification for this local Windows distribution: + +- Frontend target is the standard WebUI only. +- SingUI is explicitly out of scope for this installer/runtime plan. +- Packaging, QA, and release gates in this document apply to backend + standard WebUI only. + +## 2. Jellyfin-style baseline to emulate + +Jellyfin's model to mirror: + +- server process hosted locally on Windows, +- web client served by that process, +- default localhost access plus configurable LAN bind, +- predictable install location and upgrade path, +- optional service semantics (auto-start/always-on behavior). + +Target StemEngine shape: + +- a single installed "StemEngine Server" runtime (backend + static Web UI assets), +- browser access at a stable URL (default: http://localhost:3001), +- optional LAN access (for other devices on the same network), +- optional Windows service mode for headless/always-on operation. + +Frontend note: + +- "Web UI assets" here means the standard WebUI build artifacts. +- SingUI assets are not included in the local Windows installer payload for this plan. + +## 3. Packaging strategy + +## 3.1 Runtime packaging approach + +Primary approach (recommended): + +- Freeze backend Python runtime into a packaged executable tree (not one-file), using PyInstaller or Nuitka. +- Include prebuilt frontend dist in the install payload. +- Run FastAPI with an embedded ASGI server process wrapper. +- Bundle ffmpeg binaries required by processing paths. +- Keep model assets externalized in a writable data directory to avoid huge installer updates. + +Why this approach: + +- predictable startup and dependency closure on Windows, +- straightforward code signing and installer integration, +- allows patching backend/frontend without requiring Docker. + +## 3.2 Installer technology + +Use WiX Toolset v4 or NSIS for MSI/EXE installer generation. + +Installer must provide: + +- per-machine install under Program Files, +- Start Menu shortcuts, +- optional desktop shortcut, +- uninstall entry, +- upgrade-safe migrations for config/data, +- optional firewall rule creation for selected port, +- optional service registration checkbox. + +## 3.3 Service and process modes + +Ship two launch modes: + +- Desktop mode: user launches StemEngine app; tray icon and "Open Web UI" action. +- Service mode: Windows service starts on boot and runs without user login. + +Service wrapper options: + +- winsw (Windows Service Wrapper) around backend executable, or +- native service host in a tiny .NET helper. + +## 4. Configuration and data layout (Windows) + +Recommended directory layout: + +- Binaries: C:\Program Files\StemEngine\ +- Mutable app data: C:\ProgramData\StemEngine\ +- Logs: C:\ProgramData\StemEngine\logs\ +- Settings/config: C:\ProgramData\StemEngine\config\settings.json +- Model cache/downloads: C:\ProgramData\StemEngine\models\ +- Job output/projects: C:\ProgramData\StemEngine\projects\ and jobs\ + +Rules: + +- keep Program Files read-only at runtime, +- keep all mutable and large data in ProgramData, +- preserve ProgramData across upgrades/uninstalls unless user explicitly chooses cleanup. + +## 5. Networking plan (LAN-access similar to Jellyfin) + +## 5.1 Bind behavior + +Default: + +- bind backend to 127.0.0.1 (safe local-only first run). + +Optional LAN mode: + +- bind to 0.0.0.0 with explicit UI toggle, +- show warning + trust model text before enabling. + +## 5.2 Port and discovery + +Initial defaults: + +- Web/API port: 3001 (or selected final port), +- optional second internal port only if required by split services. + +Future enhancement: + +- optional mDNS/SSDP announcement for discovery on trusted LAN. + +## 5.3 Windows Firewall integration + +Installer option: + +- "Allow StemEngine on private networks" (recommended), +- creates inbound TCP rule for configured port/profile, +- avoid public-network default opening. + +## 6. GPU/CPU runtime design + +Runtime policy: + +- CPU mode always available. +- GPU mode available when NVIDIA runtime is detected and dependencies load. +- mismatch fallback: automatic downgrade to CPU with explicit status in UI/logs. + +Packaging notes: + +- keep CUDA-dependent Python wheels aligned and pinned, +- provide a startup self-check screen/report listing detected providers, +- persist user-selected separation mode in settings and respect it across restarts. + +## 7. Security baseline for LAN mode + +Minimum controls before broad LAN exposure: + +- first-run admin credential setup (or strong local token) when LAN mode is enabled, +- CSRF protections for mutating endpoints, +- input/path hardening validation retained, +- optional CORS narrowing in packaged defaults, +- local audit logs for settings changes and remote requests. + +## 8. Build and release pipeline + +## 8.1 CI artifacts + +Produce signed artifacts per release: + +- StemEngine-Windows-x64-Installer.exe (or .msi), +- portable zip (advanced users), +- checksums + SBOM, +- release notes with migration notes. + +## 8.2 Signing + +- sign executables and installer with code-sign certificate, +- timestamp signatures, +- verify SmartScreen reputation strategy (consistent publisher identity). + +## 8.3 Upgrade mechanics + +- in-place upgrade preserving ProgramData, +- schema migration hooks for settings/jobs/projects metadata, +- rollback-safe migration checkpoints for critical config changes. + +## 9. Implementation phases + +Phase 0: readiness audit (1-2 weeks) + +- inventory Python runtime deps, GPU deps, ffmpeg requirements, +- identify Linux-only assumptions and replace with cross-platform abstractions, +- define Windows path/env defaults and migration rules. +- confirm frontend packaging target is standard WebUI only (no SingUI build/dependency path). + +Exit criteria: + +- Windows compatibility gap list and remediation tasks approved. + +Phase 1: local Windows runnable build (1-2 weeks) + +- backend launches on Windows via one command, +- standard WebUI frontend dist served by backend, +- basic processing flow works in CPU mode. + +Exit criteria: + +- end-to-end separation/transcription on local host with persisted settings. + +Phase 2: frozen runtime packaging (2-3 weeks) + +- produce packaged backend runtime (PyInstaller/Nuitka), +- embed standard WebUI static assets, +- include ffmpeg/runtime probes. + +Exit criteria: + +- portable unpacked build runs on clean Windows x64 VM. + +Phase 3: installer + service mode (2-3 weeks) + +- implement installer, +- install/uninstall/upgrade paths, +- optional service registration and firewall rule support. + +Exit criteria: + +- clean install, upgrade, uninstall acceptance tests pass. + +Phase 4: LAN + security hardening (2 weeks) + +- LAN toggle UX and guardrails, +- authentication hardening for network mode, +- request logging and diagnostics UI. + +Exit criteria: + +- LAN access from secondary device works with auth and clear operator controls. + +Phase 5: release engineering + public beta (1-2 weeks) + +- signed artifacts in CI, +- beta channel telemetry + crash reporting, +- docs and support playbooks. + +Exit criteria: + +- beta quality gate passed; release candidate approved. + +## 10. Validation matrix + +Must-pass test matrix: + +- Windows 11 x64 clean VM install/uninstall/upgrade, +- CPU-only machine flow, +- NVIDIA GPU machine flow, +- standard WebUI functional smoke (dashboard/settings/process flow), +- LAN access from second device on private subnet, +- service auto-start after reboot, +- long-running job stability and log rotation, +- data preservation across upgrades. + +## 11. Documentation deliverables + +Required docs before GA: + +- Windows Installation Guide, +- Standard WebUI on Windows (scope and entrypoint) Guide, +- LAN Access and Security Guide, +- GPU Setup and Troubleshooting, +- Upgrade and Backup/Restore Guide, +- Service Mode Operations Guide, +- Known limitations and compatibility matrix (including SingUI out-of-scope for this local build). + +## 12. Immediate next actions + +1. Pick packaging stack: PyInstaller vs Nuitka and WiX vs NSIS. +2. Stand up a Windows CI runner for reproducible x64 artifacts. +3. Implement Phase 0 audit checklist and assign owners. +4. Build a minimal Phase 1 Windows runnable prototype and measure startup/runtime behavior. +5. Define LAN-mode auth requirement before opening firewall by default. diff --git a/docs/plans/2026-07-09_StemEngine_Image_Size_Analysis_And_Reduction_Plan.md b/docs/plans/2026-07-09_StemEngine_Image_Size_Analysis_And_Reduction_Plan.md new file mode 100644 index 0000000..843da1b --- /dev/null +++ b/docs/plans/2026-07-09_StemEngine_Image_Size_Analysis_And_Reduction_Plan.md @@ -0,0 +1,154 @@ +# StemEngine Image Size Analysis And Reduction Plan (2026-07-09) + +## Scope + +- Target image: `stemengine:latest` +- Context: `/home/translator/docker/appdata/stemengine` +- Goal: explain why image size is reported around 30 GB and define a practical shrink plan. + +## Agent Guidance Used + +- `docs/agents/agency-agents/adapted/software-architect.md` +- `docs/agents/agency-agents/adapted/performance-benchmarker.md` +- `docs/agents/agency-agents/adapted/technical-writer.md` +- `docs/agents/agency-overrides/release-gate.md` + +Applied interpretation: + +- Prefer incremental, migration-safe changes. +- Prioritize measurable size reductions with validation gates. +- Keep this plan evidence-based and implementation-ready. + +## Baseline Evidence + +Collected from local Docker host: + +- `docker image ls` / `docker system df -v` report `stemengine:latest` at about `30.3GB`. +- `docker image inspect stemengine:latest --format '{{.Size}}'` returned `10890691183` bytes (~10.9 GB payload accounting). +- `docker system df -v` also shows `Build cache usage: 18.04GB`. + +Interpretation: + +- You have two size vectors: + - Image accounting shown by Docker UI/CLI (`~30.3GB`). + - Content payload size from image metadata (`~10.9GB`). +- Independent of accounting differences, the image stack is objectively heavyweight and the host also carries large build cache. + +## Root Cause Analysis + +Primary contributors are visible in image history and Dockerfile design. + +1. Heavy ML dependency layers in a single runtime image. + - `RUN python3.11 -m uv pip install --system --no-cache -r /app/backend/requirements.txt` -> `8.61GB` + - `RUN ... install torch==2.6.0 torchaudio==2.6.0 torchvision==0.21.0 --index-url .../cu124` -> `5.2GB` + +2. Large GPU runtime base + additional system packages. + - Base: `nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04` + - Additional apt layer in Dockerfile contributes `1.96GB`. + - Current image includes cuDNN runtime base and also installs `libcudnn8` for compatibility. + +3. Dev tooling shipped in production backend image. + - Node.js is installed in backend image for frontend watch workflow (`start.sh`). + - Build toolchain (`build-essential`, `python3.11-dev`, `git`) is installed in runtime stage. + +4. Monolithic "all capabilities in one image" architecture. + - Main backend image carries separation + transcription + optional feature stacks together. + - `requirements.txt` includes broad stacks (WhisperX, audio-separator[gpu], transformers, etc.). + +5. Host-level Docker build cache adds major extra storage pressure. + - Build cache currently around `18GB`, independent from running container writable layers. + +## Plan To Reduce Size + +## Phase 0: Measurement And Guardrails (No behavior change) + +1. Add a size baseline script and capture per-build metrics: + - `docker image inspect` bytes + - `docker history` top layers + - `docker system df -v` snapshot +2. Define acceptance target bands: + - `stemengine-runtime-core`: <= 8 GB + - `stemengine-runtime-full`: <= 14 GB +3. Keep functional parity tests for: + - separation endpoints + - transcription endpoints + - startup health + +## Phase 1: Low-Risk Structural Cuts + +1. Split dev and production Dockerfiles. + - Keep current live-reload behavior in `Dockerfile.dev`. + - Create `Dockerfile.runtime` without Node.js/watch flow. + +2. Remove dev-only packages from runtime stage. + - Move `build-essential`, `python3.11-dev`, `git` to builder stage only. + - Keep only runtime libs in final image. + +3. Stop frontend watch responsibilities in backend runtime image. + - Continue serving WebUI via existing separate web image (`stemengine-webui`) in production. + +4. Expand `.dockerignore` to shrink build context and cache churn. + - Add: `.git`, `tmp`, `jobs`, `uploads`, `imports`, `projects`, `stemengine-models`, `opt`, and local cache folders. + +Expected impact: moderate immediate reduction and better cache efficiency. + +## Phase 2: Dependency Profile Split + +1. Split Python requirements by feature profile: + - `requirements-core.txt` (API + essential separation path) + - `requirements-separation-gpu.txt` + - `requirements-transcription.txt` + +2. Build separate image variants: + - `stemengine-core` + - `stemengine-full` + - optional sidecar-based transcription image + +3. Keep sidecars for optional heavy subsystems (already aligned by architecture): + - Omnizart sidecar + - Parakeet sidecar + - consider WhisperX sidecar to decouple ASR weight from core separation runtime + +Expected impact: largest controllable drop for users who do not need the full stack at all times. + +## Phase 3: CUDA/cuDNN Rationalization + +1. Validate if dual cuDNN footprint can be reduced by workload split: + - keep separation runtime on one compatible CUDA/cuDNN stack + - isolate components requiring alternate ABI in dedicated image/sidecar + +2. Keep `onnxruntime-gpu==1.20.1` compatibility constraint intact unless a validated migration path exists. + +Expected impact: moderate to high, but requires compatibility testing. + +## Operational Cleanup Plan + +1. Add scheduled cache housekeeping guidance: + - `docker builder prune` for stale cache + - policy for retaining only active build caches +2. Document difference between: + - image footprint + - container writable layer size + - volume/model-cache usage + - build cache usage + +## Proposed Execution Order + +1. Implement Phase 1 (fastest, lowest risk). +2. Measure size delta and run regression tests. +3. Implement Phase 2 profile split. +4. Implement/validate Phase 3 only after Phase 2 stabilizes. + +## Validation Checklist + +- Build succeeds for each target image profile. +- API health endpoint passes. +- At least one GPU separation job succeeds. +- At least one transcription job succeeds (when transcription profile enabled). +- Reported image size and host disk usage deltas are recorded. + +## Success Criteria + +- Clear production path that does not require dev/watch tooling in backend image. +- Core runtime path substantially below current reported size. +- Full runtime path materially reduced and documented with measured deltas. \ No newline at end of file diff --git a/docs/plans/2026-07-09_StemEngine_PSARC_SlopPak_And_PitchBend_Integration_Plan.md b/docs/plans/2026-07-09_StemEngine_PSARC_SlopPak_And_PitchBend_Integration_Plan.md new file mode 100644 index 0000000..6d32e7a --- /dev/null +++ b/docs/plans/2026-07-09_StemEngine_PSARC_SlopPak_And_PitchBend_Integration_Plan.md @@ -0,0 +1,488 @@ +# StemEngine Plan: PSARC to sloppak + Pitch Bend Integration (2026-07-09) + +## Objective + +Plan the implementation of: + +1. PSARC to sloppak conversion in StemEngine with both: + - discrete single-item conversion + - batch conversion +2. Integration of higher-efficiency queue and orchestration patterns observed in Slopsmith engineup. +3. Automatic pitch bend detection for MIDI conversion in both basic-pitch and Omnizart flows. +4. Pitch bend playback support in StemEngine's MIDI player. +5. A lightweight in-app MIDI editor that persists user changes. + +Source reference set analyzed: + +- `E:/Docker/Slopsmith/engineup/slopsmith-plugin-sloppak-converter` +- `E:/Docker/Slopsmith/engineup/slopsmith-plugin-midi` +- `E:/Docker/Slopsmith/engineup/slopsmith-plugin-notedetect` +- StemEngine backend/frontend MIDI and transcription codepaths. + +## Agent Guidance Applied + +- `docs/agents/agency-agents/adapted/software-architect.md` +- `docs/agents/agency-agents/adapted/performance-benchmarker.md` +- `docs/agents/agency-agents/adapted/technical-writer.md` +- `docs/agents/agency-overrides/release-gate.md` + +Interpretation used for this plan: + +- Keep migration incremental and reversible. +- Preserve existing APIs/flows unless extension is required. +- Define measurable validation gates before closure. + +## Current State (StemEngine) + +1. No existing PSARC or sloppak conversion support in backend or frontend. +2. MIDI transcription exists: + - basic-pitch in-process (`transcriber_service.py`) + - Omnizart sidecar/subprocess (`omnizart_service.py`) +3. basic-pitch presets already include `multiple_pitch_bends` for guitar/bass modes in config, but no explicit auto bend policy orchestration across all modes. +4. Frontend MIDI player currently schedules note events only (Tone.Part + triggerAttackRelease) and does not parse/play pitch bend events. +5. No in-app MIDI editing workflow currently exists for note/bend edits and save-back. + +## External Capability Map (Slopsmith engineup) + +From `slopsmith-plugin-sloppak-converter`: + +1. Queue-backed conversion jobs with explicit lifecycle states. +2. Discrete enqueue + bulk enqueue API split. +3. Operational controls: pause/resume, cancel queued, retry failed, clear finished. +4. Live WebSocket queue/job updates with snapshot on connect. +5. Queue persistence to disk and resume behavior after restart. +6. Robust path validation and traversal guards. +7. Thread-offloaded sync filesystem checks for large bulk operations. +8. Result dedupe/skip reasons (`already_queued`, `already_running`, `already_converted`, etc.). +9. Optional stem splitting and sidecar-friendly architecture. + +From `slopsmith-plugin-notedetect` and plugin patterns: + +1. Strong use of capability gating and fail-soft behavior. +2. Practical support for multiple detection modes with deterministic fallback. +3. Real-time event/state reporting suitable for user feedback. + +## Architecture Rationale (3-6 bullets) + +1. Reuse StemEngine's existing job manager and job storage model instead of introducing a separate queue subsystem. +2. Introduce converter functionality as a new backend service module with a dedicated router, keeping project/transcription APIs stable. +3. Extend MIDI metadata and frontend player types to include bend envelopes without breaking note-only playback. +4. Add feature flags and capability probes for conversion dependencies (vgmstream/ffmpeg/optional split engine) and bend extraction. +5. Keep heavy conversion and bend extraction work in background jobs to avoid blocking request handlers. +6. Keep MIDI editing lightweight and deterministic (event editing only, no full DAW scope). + +## Proposed Implementation Plan + +## Phase 1: Converter Foundations (Backend) + +1. Add new service module set: + - `backend/services/psarc_sloppak_service.py` + - `backend/services/psarc_sloppak_queue_service.py` (or integrated with job_manager wrappers) +2. Add new router: + - `backend/routers/sloppak_converter.py` +3. Add API endpoints (v1): + - `POST /api/sloppak/enqueue` (single/discrete) + - `POST /api/sloppak/enqueue-bulk` (batch) + - `GET /api/sloppak/jobs` + - `POST /api/sloppak/jobs/cancel-queued` + - `POST /api/sloppak/jobs/retry-failed` + - `POST /api/sloppak/jobs/clear-finished` + - `POST /api/sloppak/pause` + - `POST /api/sloppak/resume` +4. Input safety and dedupe: + - enforce relative paths under configured root + - reject path traversal and absolute paths + - return explicit skip reasons for duplicates and missing files +5. Add conversion result schema to job files metadata: + - output path + - generated stem list + - output size + - elapsed time + +## Phase 2: Converter UX + Batch Operations (Frontend) + +1. Add Conversions screen/page with queue table. +2. Add discrete convert action in relevant file/project rows. +3. Add batch selection + enqueue flow. +4. Add queue control actions: + - pause/resume + - cancel queued + - retry failed + - clear finished +5. Add polling first, then optional WebSocket live updates if needed for UX smoothness. + +## Phase 3: Efficiency Integrations from engineup + +Port efficiency patterns that are superior to current baseline: + +1. Persistence of converter queue state across restart. +2. Thread-offloaded preflight checks for large batch enqueue operations. +3. Skip-reason taxonomy and idempotent enqueue semantics. +4. Job snapshot + update stream for low-latency UI updates. +5. Optional dependency capability flag in jobs snapshot, so UI can degrade gracefully. + +Note: + +- StemEngine already has strong background execution primitives in `job_manager.py`; this phase is primarily about adapter behavior and UX semantics, not replacing core executors. + +## Phase 4: Automatic Pitch Bend Detection in MIDI Conversion + +### 4A. basic-pitch path + +1. Add explicit bend policy option layer per mode: + - `auto` (default) + - `force_on` + - `force_off` +2. Define auto policy: + - guitar/bass modes: enabled + - monophonic vocal strict: disabled by default unless confidence conditions are met + - polyphonic/melody: adaptive based on confidence/segment stability +3. Persist bend extraction diagnostics in job metadata: + - bend_event_count + - bend_confidence_summary + +### 4B. Omnizart path + +1. Add post-process bend derivation from Omnizart output where native bends are absent: + - for `omnizart-vocal-contour`, derive pitch wheel events from F0 contour to nearest active note context + - for other modes, keep opt-in derivation path with confidence gating +2. Add mode-aware bend derivation guardrails: + - avoid over-bending on dense polyphonic sections + - cap bend range and smoothing windows +3. Store derivation metadata in job result for transparency/debugging. + +## Phase 5: MIDI Player Pitch Bend Playback + +1. Extend frontend MIDI types in `ProjectDetail.tsx` to include track/channel bend events. +2. Add bend scheduling pipeline: + - parse pitch bend events from `@tonejs/midi` tracks + - schedule bend curves in sync with notes +3. Implement synth-side bend application strategy: + - minimum viable: per-stem channel bend stream applied to active synth voice group + - improved: per-track/per-channel synth partitioning for reduced bend cross-talk +4. Add UI toggle: + - Pitch Bend Playback On/Off (default on) + - optional bend intensity scaling for audition + +## Phase 6: Lightweight MIDI Editor + Save + +1. Add MIDI editor panel in project detail view: + - selectable track/channel + - piano-roll lane with basic note blocks + - bend lane for pitch wheel curve points +2. Minimum edit operations (v1): + - add/remove/move/resize note events + - velocity adjust + - add/move/remove bend points + - quantize selection (coarse presets only) +3. Persistence model: + - keep edits as an internal normalized event list while editing + - save action writes updated `.mid` artifact via backend endpoint + - preserve original source MIDI and write an edited sibling output for rollback safety +4. New API endpoints (v1): + - `POST /api/midi/edit/sessions` (open edit session for source midi) + - `PATCH /api/midi/edit/sessions/{session_id}` (apply edit ops) + - `POST /api/midi/edit/sessions/{session_id}/save` (materialize edited midi) + - `DELETE /api/midi/edit/sessions/{session_id}` (discard edit session) +5. Save metadata: + - editor version + - source midi path/hash + - edit timestamp + - optional edit summary (counts of note/bend ops) + +## Data Model and API Additions + +1. Job metadata extensions: + - `conversion_kind` (`psarc_to_sloppak`) + - `skip_reason` + - `bend_stats` +2. MIDI editor metadata extensions: + - `edited_midi_path` + - `source_midi_path` + - `midi_edit_summary` + - `midi_editor_version` +3. Optional app settings additions: + - converter roots and dependency flags + - bend policy defaults + - editor snap/quantize defaults +4. Response contracts should remain additive to avoid breaking current clients. + +## Validation Plan + +## Converter Validation + +1. Single PSARC conversion success path. +2. Batch conversion with mixed valid/invalid/already-converted entries and reason-code verification. +3. Pause/resume behavior and cancellation semantics. +4. Restart recovery of queued/in-progress converter jobs. +5. Path traversal and absolute-path rejection tests. + +## Pitch Bend Validation + +1. basic-pitch guitar/bass outputs include pitch bend events when expected. +2. Omnizart contour-derived bends generated with confidence thresholds. +3. Frontend MIDI playback audibly follows bends and remains synchronized. +4. No regression in note-only playback when bends absent. + +## MIDI Editor Validation + +1. User can perform note edits and save an updated midi artifact. +2. User can perform bend edits and hear them during playback. +3. Save operation preserves original midi and writes edited sibling output. +4. Reloading project displays edited midi artifact and updated playback behavior. +5. Session discard path leaves source midi unchanged. + +## Required checks before closure + +1. Backend compile checks for touched modules. +2. Frontend typecheck for touched UI/player files. +3. Evidence captured in job output samples and manual playback verification notes. +4. Edited midi regression sample checked in test fixtures (or documented local artifact path). + +## Non-Goals (for first delivery) + +1. Full Slopsmith plugin runtime compatibility layer inside StemEngine. +2. Perfect per-note MPE-grade bend rendering for dense polyphonic MIDI on first pass. +3. Replacing existing StemEngine job manager architecture. +4. Building a full DAW timeline with automation lanes beyond notes + pitch bend. + +## Risks and Mitigations + +1. Dependency complexity for PSARC conversion stack. + - Mitigation: explicit capability probing + fail-soft UI messaging. +2. Bend overfitting or artifacts in polyphonic material. + - Mitigation: conservative default auto policy + user toggle. +3. Playback CPU overhead from bend scheduling. + - Mitigation: throttle/simplify bend control points and offer UI disable toggle. +4. Invalid MIDI serialization after complex edit sequences. + - Mitigation: backend-side schema validation and round-trip parse checks before save success. + +## Proposed Delivery Slices + +1. Slice A: Converter backend + discrete endpoint + minimal frontend trigger. +2. Slice B: Batch enqueue + queue controls + reason codes. +3. Slice C: basic-pitch auto bend policy + metadata. +4. Slice D: Omnizart bend derivation + safeguards. +5. Slice E: MIDI player bend playback + UI controls. +6. Slice F: lightweight MIDI editor UI + edit session endpoints. +7. Slice G: save/discard workflow + artifact persistence + regression validation. + +This sequencing keeps value incremental while containing regression risk. + +## Implementation Update (2026-07-09) + +Status update for requested execution order: + +1. Slice F implemented (lightweight MIDI editor v1). +2. Slice A implemented (converter backend + discrete enqueue). +3. Slice B implemented (batch enqueue + queue controls + reasoned skip behavior). + +### Implemented artifacts + +Backend: + +- `backend/routers/midi_editor.py` +- `backend/services/midi_editor_service.py` +- `backend/routers/sloppak_converter.py` +- `backend/services/sloppak_converter_service.py` +- `backend/main.py` (router registration) + +Frontend: + +- `frontend/src/components/MidiLightEditor.tsx` +- `frontend/src/components/SloppakConverterPanel.tsx` +- `frontend/src/pages/ProjectDetail.tsx` (integration points) +- `frontend/src/api/client.ts` (new API client methods/types) +- `frontend/src/types.ts` (new job_type values) + +### Notes on Slice F scope delivered + +- Session-based MIDI edit lifecycle is live: + - open session from an existing transcribe job MIDI file + - apply note/bend edit operations + - save to a new sibling MIDI artifact (source remains unchanged) + - discard session +- Frontend editor supports lightweight note and bend edits, quantize, and save/download. + +### Notes on Slices A/B scope delivered + +- Converter queue API is live for: + - single enqueue + - bulk enqueue + - pause/resume + - cancel queued + - retry failed + - clear finished +- Frontend queue panel is integrated in Project Detail with controls and download links. +- Current conversion implementation writes a valid `.sloppak` zip package containing: + - `manifest.json` + - original `.psarc` payload under `source/` + +This satisfies A/B operational flow and queue semantics; deeper Rocksmith-aware extraction/normalization can be layered as a follow-on enhancement. + +### Validation evidence + +- Backend compile check: + +### Operationalization follow-up (2026-07-09) + +To make deep extraction deployable without code edits: + +1. Added runtime capability endpoint: + - `GET /api/sloppak/capabilities` + - reports unpacker availability and converter root readiness. +2. Added environment wiring/documentation: + - `stemengine.yml`: `SLOPPAK_SOURCE_ROOT`, `SLOPPAK_OUTPUT_ROOT`, `SLOPPAK_PSARC_UNPACK_CMD` + - `.env.example`: includes concrete command-template example + - `README.md`: setup + verification instructions + +Validation evidence: + +- `python3 -m compileall backend/routers/sloppak_converter.py backend/services/sloppak_converter_service.py` + - `python3 -m compileall backend/main.py backend/routers/sloppak_converter.py backend/routers/midi_editor.py backend/services/sloppak_converter_service.py backend/services/midi_editor_service.py` +- Frontend typecheck: + - `cd frontend && npm run -s typecheck` + +## Follow-up Implementation Update (2026-07-09, later) + +Requested enhancement implemented: + +- Added deeper Rocksmith-aware extraction/normalization pass to converter packaging logic. + +Implemented in: + +- `backend/services/sloppak_converter_service.py` + +Behavior added: + +1. Optional PSARC unpack pipeline with graceful fallback: + - preferred: `patcher.unpack_psarc` (when available) + - fallback: env command template via `SLOPPAK_PSARC_UNPACK_CMD` +2. Normalized metadata extraction from unpacked payload: + - song fields: title/artist/album/year + - arrangements: id/name/tuning/capo (deduplicated) + - WEM inventory list and count +3. Lyrics extraction from Rocksmith vocals XML (``), when present. +4. Enriched `.sloppak` package contents: + - `manifest.json` now includes normalized `metadata`, `rocksmith.unpack`, `rocksmith.arrangements`, and `rocksmith.audio` sections. + - `metadata/rocksmith-summary.json` added to package. + - `lyrics.json` added when lyrics were extractable. + +Compatibility notes: + +- Conversion remains successful even when unpack tools are unavailable. +- In no-unpacker mode, package still includes source PSARC baseline content and reports unpack status in manifest. + +Validation evidence: + +- `python3 -m compileall backend/services/sloppak_converter_service.py` + +## Follow-up Implementation Update (2026-07-09, later) — Slices C, D, and E + +Requested next slices implemented: + +- Slice C: basic-pitch auto bend policy + bend diagnostics metadata. +- Slice D: Omnizart bend derivation with guardrails + metadata transparency. +- Slice E: frontend MIDI player pitch bend playback + UI controls. + +### Slice C (basic-pitch) delivered + +Implemented in: + +- `backend/services/transcriber_service.py` +- `backend/routers/transcribe.py` +- `backend/routers/projects.py` +- `backend/services/project_reprocess_service.py` + +Behavior added: + +1. New bend policy channel for transcription jobs: + - `auto` (default) + - `force_on` + - `force_off` +2. Auto-policy orchestration: + - guitar/bass -> bends enabled + - monophonic -> bends disabled + - polyphonic/melody -> adaptive decision based on confidence summary + note stability +3. Bend diagnostics persisted to job result metadata under `result.bend_stats`: + - `bend_event_count` + - `bend_confidence_summary` + - `segment_stability` + - policy decision fields + +### Slice D (Omnizart) delivered + +Implemented in: + +- `backend/services/omnizart_service.py` + +Behavior added: + +1. Omnizart path now receives and honors bend policy (`auto|force_on|force_off`). +2. Post-process derivation path when native bends are absent: + - `auto`: derive for `vocal-contour` + - `force_on`: derive for all Omnizart modes + - `force_off`: never derive +3. Guardrails to avoid over-bending: + - dense polyphonic detection gate + - capped semitone delta and MIDI bend value range + - transition-only derivation (no uncontrolled continuous drift) +4. Result metadata transparency in `result.bend_stats`: + - native vs derived bend counts + - final bend count + - derivation applied/reason + +### Slice E (frontend playback) delivered + +Implemented in: + +- `frontend/src/pages/ProjectDetail.tsx` + +Behavior added: + +1. MIDI parser/playback now reads `pitchBends` from `@tonejs/midi` track data. +2. Pitch bend events are scheduled into playback transport per stem. +3. New user controls in MIDI playback panel: + - Bend playback toggle (On/Off, default On) + - Bend intensity scaling slider (0-200%) +4. Playback status now reports bend event counts alongside note counts. + +Validation evidence: + +- Backend compile: + - `python3 -m compileall backend/services/transcriber_service.py backend/services/omnizart_service.py backend/routers/transcribe.py backend/routers/projects.py backend/services/project_reprocess_service.py` +- Frontend typecheck: + - `cd frontend && npm run -s typecheck` + +## Remaining Work Checklist (as of 2026-07-09) + +Completion status: implemented and closed. + +- [x] Dedicated conversions page added (in addition to Project Detail panel): + - `frontend/src/pages/Conversions.tsx` + - route: `/conversions` + - sidebar navigation entry added + +- [x] Slice G closure evidence bundle captured: + - Save/discard/source-immutability regression executed in runtime container (`python3.11`): + - source: `/app/media/jobs/midi_regression_evidence/regression_source.mid` + - edited output: `/app/media/jobs/midi_regression_evidence/regression_source.edited.mid` + - `source_unchanged_after_save=true` + - `source_unchanged_after_discard=true` + +- [x] Phase 3 efficiency backlog delivered: + - queue persistence/recovery on restart via `ensure_queue_hydrated()` + startup hook in `backend/main.py` + - thread-offloaded batch preflight checks in `enqueue_bulk` (`ThreadPoolExecutor` path) + - low-latency snapshot/update stream: + - `GET /api/sloppak/snapshot` + - `GET /api/sloppak/updates` (SSE) + - frontend panel subscribes via `EventSource` and refetches on snapshot events + +- [x] Validation-plan evidence captured in this plan doc: + - converter hydration/snapshot runtime checks (container): + - `ensure_queue_hydrated() -> {'resubmitted': 0, 'requeued_running': 0}` + - snapshot contains paused/jobs/capabilities payload + - update stream runtime check (container): + - first chunks emitted: `retry: 1500`, `event: snapshot` + - frontend validation: + - `cd frontend && npm run -s typecheck` \ No newline at end of file diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 6345f32..9a894b9 100755 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -5,6 +5,7 @@ import { isFeatureEnabled, MOBILE_KARAOKE_BOX_V1 } from './utils/featureFlags' import MobileKaraokeShell from './components/singui/MobileKaraokeShell' import Separator from './pages/Separator' import ProjectDetail from './pages/ProjectDetail' +import ConversionsPage from './pages/Conversions' import SettingsPage from './pages/Settings' import { ThemeProvider } from './context/ThemeContext' import { QueueProvider } from './context/QueueContext' @@ -24,6 +25,7 @@ import { PLAYLISTS_ROUTE, SING_ROUTE, SETTINGS_ROUTE, + CONVERSIONS_ROUTE, VISUALIZER_LAB_ROUTE, KARAOKE_ROUTE, KARAOKE_TV_ROUTE, @@ -226,6 +228,7 @@ function WebUiShell() { } /> } /> + } /> } /> } /> diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 1fed9cd..b63e7a8 100755 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -229,10 +229,11 @@ export const submitSeparate = ( export const getTranscribeModes = () => request(`${BASE}/transcribe/modes`) -export const submitTranscribe = (file: File, mode: string) => { +export const submitTranscribe = (file: File, mode: string, bendPolicy = 'auto') => { const fd = new FormData() fd.append('file', file) fd.append('mode', mode) + fd.append('bend_policy', bendPolicy) return request<{ job_id: string }>(`${BASE}/transcribe`, { method: 'POST', body: fd }) } @@ -348,10 +349,11 @@ export const submitLrcAlignExisting = ( // ---------- Quick-action helpers (from existing project/job on server) ---------- -export const submitTranscribeFromProject = (projectName: string, mode = 'music') => { +export const submitTranscribeFromProject = (projectName: string, mode = 'music', bendPolicy = 'auto') => { const fd = new FormData() fd.append('project_name', projectName) fd.append('mode', mode) + fd.append('bend_policy', bendPolicy) return request<{ job_id: string; stem: string }>(`${BASE}/transcribe/from-project`, { method: 'POST', body: fd }) } @@ -600,11 +602,11 @@ export const getProjectGloss = (id: string, targetLang: string, sourceLang?: str export const deleteProject = (id: string) => request<{ detail: string }>(`${BASE}/projects/${id}`, { method: 'DELETE' }) -export const transcribeProject = (id: string, stemName: string, mode: string) => +export const transcribeProject = (id: string, stemName: string, mode: string, bendPolicy = 'auto') => request<{ job_id: string }>(`${BASE}/projects/${id}/transcribe`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ stem_name: stemName, mode }), + body: JSON.stringify({ stem_name: stemName, mode, bend_policy: bendPolicy }), }) export const abcProject = (id: string, stemName: string, songTitle?: string) => @@ -1462,3 +1464,146 @@ export const deletePixabayCacheEntry = (key: string) => /** Clear the entire Pixabay video cache. */ export const clearPixabayCache = () => request<{ deleted: string }>(`${BASE}/pixabay/cache`, { method: 'DELETE' }) + +// ── Sloppak converter ─────────────────────────────────────────────────────── + +export interface SloppakJobListResponse { + paused: boolean + jobs: Job[] +} + +export interface SloppakEnqueueResult { + ok?: boolean + skipped?: boolean + filename?: string + reason?: string + job_id?: string + output_path?: string +} + +export interface SloppakBulkEnqueueResult { + enqueued: SloppakEnqueueResult[] + skipped: SloppakEnqueueResult[] + summary: { + requested: number + enqueued: number + skipped: number + } +} + +export const listSloppakJobs = () => + request(`${BASE}/sloppak/jobs`) + +export const getSloppakSnapshot = () => + request }>(`${BASE}/sloppak/snapshot`) + +export const hydrateSloppakQueue = () => + request<{ resubmitted: number; requeued_running: number }>(`${BASE}/sloppak/hydrate`, { method: 'POST' }) + +export const enqueueSloppak = (filename: string, reconvert = false) => + request(`${BASE}/sloppak/enqueue`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ filename, reconvert }), + }) + +export const enqueueSloppakBulk = (filenames: string[], reconvert = false) => + request(`${BASE}/sloppak/enqueue-bulk`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ filenames, reconvert }), + }) + +export const pauseSloppakQueue = () => + request<{ paused: boolean }>(`${BASE}/sloppak/pause`, { method: 'POST' }) + +export const resumeSloppakQueue = () => + request<{ paused: boolean }>(`${BASE}/sloppak/resume`, { method: 'POST' }) + +export const cancelQueuedSloppak = () => + request<{ cancelled: number }>(`${BASE}/sloppak/jobs/cancel-queued`, { method: 'POST' }) + +export const retryFailedSloppak = () => + request<{ retried: number; skipped: number }>(`${BASE}/sloppak/jobs/retry-failed`, { method: 'POST' }) + +export const clearFinishedSloppak = () => + request<{ deleted: number }>(`${BASE}/sloppak/jobs/clear-finished`, { method: 'POST' }) + +// ── MIDI light editor ─────────────────────────────────────────────────────── + +export interface MidiEditorNote { + id: string + pitch: number + start: number + duration: number + velocity: number +} + +export interface MidiEditorBend { + id: string + time: number + value: number +} + +export interface MidiEditorTrack { + index: number + name: string + program: number + is_drum: boolean + notes: MidiEditorNote[] + bends: MidiEditorBend[] +} + +export interface MidiEditorSession { + id: string + created_at: string + source_job_id: string + source_filename: string + source_path: string + editor_version: string + tracks: MidiEditorTrack[] + last_saved_path?: string + last_saved_job_id?: string +} + +export interface MidiEditOp { + op: string + track_index: number + note_id?: string + bend_id?: string + pitch?: number + start?: number + duration?: number + velocity?: number + time?: number + value?: number + step?: number +} + +export const createMidiEditSession = (jobId: string, filename?: string) => + request(`${BASE}/midi/edit/sessions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ job_id: jobId, filename }), + }) + +export const getMidiEditSession = (sessionId: string) => + request(`${BASE}/midi/edit/sessions/${encodeURIComponent(sessionId)}`) + +export const patchMidiEditSession = (sessionId: string, ops: MidiEditOp[]) => + request(`${BASE}/midi/edit/sessions/${encodeURIComponent(sessionId)}`, { + method: 'PATCH', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ ops }), + }) + +export const saveMidiEditSession = (sessionId: string) => + request<{ job_id: string; filename: string; path: string; summary: { track_count: number; note_count: number; bend_count: number } }>( + `${BASE}/midi/edit/sessions/${encodeURIComponent(sessionId)}/save`, + { method: 'POST' } + ) + +export const discardMidiEditSession = (sessionId: string) => + request<{ discarded: boolean }>(`${BASE}/midi/edit/sessions/${encodeURIComponent(sessionId)}`, { + method: 'DELETE', + }) diff --git a/frontend/src/components/MidiLightEditor.tsx b/frontend/src/components/MidiLightEditor.tsx new file mode 100644 index 0000000..bd5df42 --- /dev/null +++ b/frontend/src/components/MidiLightEditor.tsx @@ -0,0 +1,282 @@ +import { useMemo, useState } from 'react' +import { Download, Loader2, Save, Trash2 } from 'lucide-react' +import { + createMidiEditSession, + discardMidiEditSession, + patchMidiEditSession, + saveMidiEditSession, + type MidiEditOp, + type MidiEditorSession, +} from '../api/client' +import { downloadUrl } from '../api/client' + +interface StemRef { + label: string + jobId: string + filename: string +} + +export default function MidiLightEditor({ stem }: { stem: StemRef }) { + const [open, setOpen] = useState(false) + const [busy, setBusy] = useState(false) + const [err, setErr] = useState(null) + const [info, setInfo] = useState(null) + const [session, setSession] = useState(null) + const [trackIndex, setTrackIndex] = useState(0) + const [quantize, setQuantize] = useState(0.25) + const [savedJobId, setSavedJobId] = useState(null) + const [savedFilename, setSavedFilename] = useState(null) + + const activeTrack = useMemo(() => { + if (!session) return null + return session.tracks[trackIndex] ?? null + }, [session, trackIndex]) + + async function openEditor() { + setBusy(true) + setErr(null) + setInfo(null) + try { + const created = await createMidiEditSession(stem.jobId, stem.filename) + setSession(created) + setTrackIndex(0) + setOpen(true) + setSavedJobId(null) + setSavedFilename(null) + } catch (e) { + setErr(String(e)) + } finally { + setBusy(false) + } + } + + async function applyOps(ops: MidiEditOp[]) { + if (!session) return + setBusy(true) + setErr(null) + try { + const updated = await patchMidiEditSession(session.id, ops) + setSession(updated) + } catch (e) { + setErr(String(e)) + } finally { + setBusy(false) + } + } + + async function handleSave() { + if (!session) return + setBusy(true) + setErr(null) + setInfo(null) + try { + const res = await saveMidiEditSession(session.id) + setSavedJobId(res.job_id) + setSavedFilename(res.filename) + setInfo(`Saved MIDI (${res.summary.note_count} notes, ${res.summary.bend_count} bends).`) + } catch (e) { + setErr(String(e)) + } finally { + setBusy(false) + } + } + + async function closeEditor(discard: boolean) { + if (discard && session) { + try { + await discardMidiEditSession(session.id) + } catch (e) { + setErr(String(e)) + } + } + setOpen(false) + setSession(null) + setTrackIndex(0) + } + + if (!open) { + return ( + + ) + } + + return ( +
+
+ MIDI Editor · {stem.label} + + + + + + + +
+ + {savedJobId && savedFilename && ( + + + Download {savedFilename} + + )} + + {info &&

{info}

} + {err &&

{err}

} + + {activeTrack && ( + <> +
+

Notes (first 16)

+ {activeTrack.notes.slice(0, 16).map((note) => ( +
+ { + const pitch = Number(e.target.value) + if (!Number.isFinite(pitch) || pitch === note.pitch) return + void applyOps([{ op: 'update_note', track_index: trackIndex, note_id: note.id, pitch }]) + }} + /> + { + const start = Number(e.target.value) + if (!Number.isFinite(start) || start === note.start) return + void applyOps([{ op: 'update_note', track_index: trackIndex, note_id: note.id, start }]) + }} + /> + { + const duration = Number(e.target.value) + if (!Number.isFinite(duration) || duration === note.duration) return + void applyOps([{ op: 'update_note', track_index: trackIndex, note_id: note.id, duration }]) + }} + /> + { + const velocity = Number(e.target.value) + if (!Number.isFinite(velocity) || velocity === note.velocity) return + void applyOps([{ op: 'update_note', track_index: trackIndex, note_id: note.id, velocity }]) + }} + /> + + pitch/start/dur/vel +
+ ))} + +
+ +
+

Pitch Bends (first 12)

+ {activeTrack.bends.slice(0, 12).map((bend) => ( +
+ { + const time = Number(e.target.value) + if (!Number.isFinite(time) || time === bend.time) return + void applyOps([{ op: 'update_bend', track_index: trackIndex, bend_id: bend.id, time }]) + }} + /> + { + const value = Number(e.target.value) + if (!Number.isFinite(value) || value === bend.value) return + void applyOps([{ op: 'update_bend', track_index: trackIndex, bend_id: bend.id, value }]) + }} + /> + +
+ ))} + +
+ + )} +
+ ) +} diff --git a/frontend/src/components/Sidebar.tsx b/frontend/src/components/Sidebar.tsx index 9711386..58493d3 100755 --- a/frontend/src/components/Sidebar.tsx +++ b/frontend/src/components/Sidebar.tsx @@ -1,11 +1,11 @@ import React, { useState, useRef, useMemo, useEffect } from 'react' import { useQuery, useQueryClient } from '@tanstack/react-query' -import { Link, useNavigate } from 'react-router-dom' +import { Link, useLocation, useNavigate } from 'react-router-dom' import { ChevronLeft, ChevronRight, Loader2, CheckCircle, XCircle, Clock, AlertCircle, X, Music2, Mic2, FileCode, Workflow, FolderOpen, Type, ChevronDown, ChevronUp, Radio, Search, - Download, CheckCircle2, Trash2, Zap, + Download, CheckCircle2, Trash2, Zap, FileArchive, } from 'lucide-react' import { listJobs, cancelJob, getJob, @@ -15,7 +15,7 @@ import { } from '../api/client' import type { Job, Project, SubsonicGroupedSearchResults, SubsonicSong, ImportedFile } from '../types' import AutoPipelineDialog from './AutoPipelineDialog' -import { albumRoute, artistRoute, PLAYLISTS_ROUTE, projectRoute, SING_SEARCH_ROUTE, trackRoute } from '../routes' +import { albumRoute, artistRoute, CONVERSIONS_ROUTE, PLAYLISTS_ROUTE, projectRoute, SING_SEARCH_ROUTE, trackRoute } from '../routes' import { readSidebarProjectSearch, writeSidebarProjectSearch } from '../utils/uiState' import { publishProjectDeleted } from '../utils/crossTab' import { usePollingGate } from '../hooks/usePollingGate' @@ -872,6 +872,7 @@ interface Props { } export default function Sidebar({ isOpen, onToggle, onDeleteProject, activeProjectId, onProjectNavigate }: Props) { + const location = useLocation() const queryClient = useQueryClient() const [searchQuery, setSearchQuery] = useState(() => readSidebarProjectSearch()) const canPollSidebar = usePollingGate(isOpen) @@ -993,6 +994,19 @@ export default function Sidebar({ isOpen, onToggle, onDeleteProject, activeProje )} + + + Conversions + + {/* Search box for projects */}
diff --git a/frontend/src/components/SloppakConverterPanel.tsx b/frontend/src/components/SloppakConverterPanel.tsx new file mode 100644 index 0000000..fc7d9e4 --- /dev/null +++ b/frontend/src/components/SloppakConverterPanel.tsx @@ -0,0 +1,241 @@ +import { useEffect, useMemo, useState } from 'react' +import { useQuery } from '@tanstack/react-query' +import { Download, Loader2, Pause, Play, RefreshCcw, Trash2 } from 'lucide-react' +import { + cancelQueuedSloppak, + clearFinishedSloppak, + enqueueSloppak, + enqueueSloppakBulk, + getSloppakSnapshot, + pauseSloppakQueue, + resumeSloppakQueue, + retryFailedSloppak, + downloadUrl, +} from '../api/client' +import type { Job } from '../types' + +const ACTIVE = new Set(['queued', 'running']) + +function sourceFromJob(job: Job): string { + const result = job.result + if (!result) return '' + const candidate = (result as Record).source_filename + return typeof candidate === 'string' ? candidate : '' +} + +export default function SloppakConverterPanel() { + const [singlePath, setSinglePath] = useState('') + const [batchText, setBatchText] = useState('') + const [busy, setBusy] = useState(false) + const [msg, setMsg] = useState(null) + const [err, setErr] = useState(null) + + const { data, refetch } = useQuery({ + queryKey: ['sloppak-jobs'], + queryFn: getSloppakSnapshot, + refetchInterval: (query) => { + const payload = query.state.data + if (!payload?.jobs) return 4000 + return payload.jobs.some((j) => ACTIVE.has(j.status)) ? 2000 : 5000 + }, + }) + + useEffect(() => { + if (typeof window === 'undefined' || typeof EventSource === 'undefined') { + return + } + + const source = new EventSource('/api/sloppak/updates') + const onSnapshot = () => { void refetch() } + source.addEventListener('snapshot', onSnapshot) + source.onmessage = () => { void refetch() } + source.onerror = () => { + // Passive fallback: query polling remains active when SSE drops. + } + return () => { + source.removeEventListener('snapshot', onSnapshot) + source.close() + } + }, [refetch]) + + const jobs = data?.jobs ?? [] + const paused = Boolean(data?.paused) + const caps = (data?.capabilities ?? {}) as Record + const unpacker = (caps.unpacker ?? {}) as Record + const unpackReady = Boolean(unpacker.available) + const unpackDetail = typeof unpacker.detail === 'string' ? unpacker.detail : '' + + const stats = useMemo(() => { + const counts = { queued: 0, running: 0, done: 0, error: 0, cancelled: 0 } + jobs.forEach((job) => { + if (job.status in counts) { + counts[job.status as keyof typeof counts] += 1 + } + }) + return counts + }, [jobs]) + + async function runAction(fn: () => Promise, success: string) { + setBusy(true) + setErr(null) + setMsg(null) + try { + await fn() + setMsg(success) + await refetch() + } catch (e) { + setErr(String(e)) + } finally { + setBusy(false) + } + } + + async function handleEnqueueSingle() { + if (!singlePath.trim()) return + await runAction(async () => { + const res = await enqueueSloppak(singlePath.trim()) + if (!res.ok && res.reason) { + throw new Error(`Skipped: ${res.reason}`) + } + setSinglePath('') + }, 'Enqueued conversion.') + } + + async function handleEnqueueBatch() { + const filenames = batchText + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + if (!filenames.length) return + + await runAction(async () => { + const res = await enqueueSloppakBulk(filenames) + setMsg(`Batch enqueue: ${res.summary.enqueued} enqueued, ${res.summary.skipped} skipped.`) + setBatchText('') + }, 'Batch enqueue completed.') + } + + return ( +
+
+

PSARC → sloppak Converter

+ +
+ +
+
+
+

Single Convert

+
+ setSinglePath(e.target.value)} + placeholder="relative/path/song.psarc" + className="flex-1 rounded-lg border ui-border ui-surface-2 px-2.5 py-1.5 text-xs ui-text" + /> + +
+
+ +
+

Batch Convert

+