mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 21:17:56 +09:00
Compare commits
258
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7eb082d382 | ||
|
|
1af7ab0bde | ||
|
|
7ba40af80d | ||
|
|
f434e3c93d | ||
|
|
4b6523dbd8 | ||
|
|
d67828b78f | ||
|
|
db55205127 | ||
|
|
01ff0eb042 | ||
|
|
438ea6a836 | ||
|
|
07f48b6213 | ||
|
|
5630cfd253 | ||
|
|
f543ea6efa | ||
|
|
5a2c77978b | ||
|
|
0f7ec8f71e | ||
|
|
dbb6f3b751 | ||
|
|
dd287f8c92 | ||
|
|
07c459258a | ||
|
|
3522fdd223 | ||
|
|
441f99edeb | ||
|
|
2e145dd927 | ||
|
|
985f5c7718 | ||
|
|
2b5ef28789 | ||
|
|
76940691b1 | ||
|
|
4b346dd369 | ||
|
|
7480950a1b | ||
|
|
64ada25e9c | ||
|
|
de6322dd87 | ||
|
|
49b472429d | ||
|
|
d57282c770 | ||
|
|
a41948fe37 | ||
|
|
ffd4e9e377 | ||
|
|
ef9c4fc243 | ||
|
|
1f030333ae | ||
|
|
707df6af72 | ||
|
|
c94c553069 | ||
|
|
d757b0c860 | ||
|
|
c632e68b5b | ||
|
|
9376d53fed | ||
|
|
18ee2df5a2 | ||
|
|
68b8cd25a4 | ||
|
|
cd91b06eb9 | ||
|
|
779d1e0e3e | ||
|
|
4e5ac5be12 | ||
|
|
973427a12b | ||
|
|
112919b55f | ||
|
|
6dc3962797 | ||
|
|
d35b8c4db0 | ||
|
|
fcbbb9a562 | ||
|
|
4bde7839ce | ||
|
|
2b8194c9ff | ||
|
|
4bbfe485c2 | ||
|
|
5bdc0cd675 | ||
|
|
4e085376a6 | ||
|
|
52b63624f0 | ||
|
|
ad6209306e | ||
|
|
849eeeb95e | ||
|
|
2135c2fb0b | ||
|
|
f944769117 | ||
|
|
11677e474d | ||
|
|
b7893f44e9 | ||
|
|
2917178fb7 | ||
|
|
d058f150b1 | ||
|
|
c0ae2122e5 | ||
|
|
bc7abf6274 | ||
|
|
a86390c402 | ||
|
|
f01a28ef75 | ||
|
|
8456aac142 | ||
|
|
ea68398ba3 | ||
|
|
6c915741ba | ||
|
|
3456bee932 | ||
|
|
a03147323f | ||
|
|
6294383f51 | ||
|
|
80ab599896 | ||
|
|
4f8f15d88c | ||
|
|
d1c9fd0d0e | ||
|
|
8a014b89a8 | ||
|
|
70fa23a7ce | ||
|
|
0f5f0a85c0 | ||
|
|
9702b18203 | ||
|
|
304f9f120c | ||
|
|
2904315af3 | ||
|
|
a2f3a4709a | ||
|
|
5fc208acef | ||
|
|
16a100efb2 | ||
|
|
c823bd9426 | ||
|
|
af82886b99 | ||
|
|
84e7b7b77a | ||
|
|
2e396aaa0b | ||
|
|
4d712f3c15 | ||
|
|
44b732019b | ||
|
|
aa1c4303af | ||
|
|
8a4aeb7811 | ||
|
|
3fe20ef206 | ||
|
|
7c701f2fb4 | ||
|
|
1076cb7350 | ||
|
|
acb92a652a | ||
|
|
fdd6032551 | ||
|
|
500af61530 | ||
|
|
874022eb9b | ||
|
|
086d263c24 | ||
|
|
e72690e1c9 | ||
|
|
8a99556b32 | ||
|
|
c769118e80 | ||
|
|
6e9d9a14ed | ||
|
|
e6b72a828f | ||
|
|
fd4783fa08 | ||
|
|
7e3619b4a6 | ||
|
|
a96b20d443 | ||
|
|
ee3ae19ac0 | ||
|
|
7af7967422 | ||
|
|
01d8ed1ea5 | ||
|
|
48a7cc603c | ||
|
|
fa1fed414b | ||
|
|
72385e3a77 | ||
|
|
68ce7ea8ea | ||
|
|
23b0c1e8b6 | ||
|
|
d113369217 | ||
|
|
7c64f2344d | ||
|
|
79e6c60a61 | ||
|
|
6e3c8d32a3 | ||
|
|
91908c3a9b | ||
|
|
ea61d06d18 | ||
|
|
2b18c4570f | ||
|
|
71ad45d6f2 | ||
|
|
d83d279fd7 | ||
|
|
b8b4cd20fe | ||
|
|
8ca2cbc260 | ||
|
|
e8b3d3c906 | ||
|
|
98a71c1b8b | ||
|
|
df26bfe191 | ||
|
|
d24df939d8 | ||
|
|
bb96e9fe6a | ||
|
|
4dda662450 | ||
|
|
c402a2cb64 | ||
|
|
d071c4834f | ||
|
|
b993bb497b | ||
|
|
45a94ef7c7 | ||
|
|
9a6767aee8 | ||
|
|
729d3ec420 | ||
|
|
4fe9a98698 | ||
|
|
60de3f9108 | ||
|
|
58ccc4324e | ||
|
|
0b266decb3 | ||
|
|
917b61d06b | ||
|
|
8cc00ddffb | ||
|
|
191d73bb63 | ||
|
|
9b77362fd0 | ||
|
|
011712e437 | ||
|
|
089889e625 | ||
|
|
1e5a6604a1 | ||
|
|
3528b90be6 | ||
|
|
7aef8906cc | ||
|
|
fc729647d8 | ||
|
|
7d527298a7 | ||
|
|
600701bcfa | ||
|
|
ec840b2e69 | ||
|
|
ad0cb72353 | ||
|
|
09d22f51dd | ||
|
|
b90cf9f36e | ||
|
|
cfdeb04202 | ||
|
|
b2568101a9 | ||
|
|
af3a9cd9ae | ||
|
|
2c59e1b588 | ||
|
|
ce521ea5f4 | ||
|
|
14baac1920 | ||
|
|
33e8d1600f | ||
|
|
feb2d89b3e | ||
|
|
4b7d7d2ffc | ||
|
|
b7bb1d08f4 | ||
|
|
b1608ad2ec | ||
|
|
8015c9aad6 | ||
|
|
c06626169d | ||
|
|
a61b74b355 | ||
|
|
d7a867ff09 | ||
|
|
4fdf4a07dc | ||
|
|
6caeece63f | ||
|
|
a285f34096 | ||
|
|
b38594fb74 | ||
|
|
6ae07225af | ||
|
|
43cac62477 | ||
|
|
350861440e | ||
|
|
c00cdeb808 | ||
|
|
8e9df9c765 | ||
|
|
0f67ba6bb0 | ||
|
|
7cbde5acca | ||
|
|
9faa62bdb0 | ||
|
|
bc7c1304df | ||
|
|
455b5a03c6 | ||
|
|
aaf8bf5e40 | ||
|
|
42c7d2537e | ||
|
|
e3fea67a5c | ||
|
|
84b153cf24 | ||
|
|
4a7be0ede7 | ||
|
|
7966930188 | ||
|
|
5955a76565 | ||
|
|
c36a200885 | ||
|
|
4bd5819592 | ||
|
|
ece8abe4b6 | ||
|
|
a802386e7d | ||
|
|
e9f18118ca | ||
|
|
9ae13ed8b9 | ||
|
|
b9cc45ab83 | ||
|
|
349ac90ba6 | ||
|
|
edc720d1e6 | ||
|
|
7c81822fc0 | ||
|
|
6d9e015833 | ||
|
|
49c9a82034 | ||
|
|
8407990b53 | ||
|
|
d349778ec3 | ||
|
|
fd428febc7 | ||
|
|
dab88f0868 | ||
|
|
5cbffc5f25 | ||
|
|
2b4ea1dfa3 | ||
|
|
8af4a2911f | ||
|
|
343241a82f | ||
|
|
c05857f304 | ||
|
|
c0816f6b31 | ||
|
|
93090504d9 | ||
|
|
e885edb2ca | ||
|
|
42b8ce0977 | ||
|
|
5ac6eaba80 | ||
|
|
b549c35515 | ||
|
|
9c9d0b5b27 | ||
|
|
3776f4d803 | ||
|
|
7c8c57cd26 | ||
|
|
abc36e28cf | ||
|
|
2308b553c8 | ||
|
|
c9d2e90dbb | ||
|
|
67fa0b6c71 | ||
|
|
a08fa5cc5b | ||
|
|
3b523ec98c | ||
|
|
0c4925f219 | ||
|
|
68d2f36bd5 | ||
|
|
6de0d6d883 | ||
|
|
e3b71cfc1c | ||
|
|
6f0760235b | ||
|
|
c9ed433bf4 | ||
|
|
625feba75a | ||
|
|
f0c02b15c4 | ||
|
|
0f2c6da2b9 | ||
|
|
9497c5b260 | ||
|
|
2cf216d4c2 | ||
|
|
fc4caf6382 | ||
|
|
af61c12e13 | ||
|
|
f51e619798 | ||
|
|
761b9d5104 | ||
|
|
139a868dbe | ||
|
|
cf1fc0aa09 | ||
|
|
e4042932f7 | ||
|
|
b4ef812c60 | ||
|
|
2b6e34395d | ||
|
|
48875ebeff | ||
|
|
45f4c6a741 | ||
|
|
614d1cab12 | ||
|
|
80105fe315 | ||
|
|
bd093fb7a7 | ||
|
|
b523be752f | ||
|
|
b74d7027c5 |
No files matched your search
@@ -0,0 +1,101 @@
|
||||
---
|
||||
description: Perform a non-destructive cross-artifact consistency and quality analysis across spec.md, plan.md, and tasks.md after task generation.
|
||||
---
|
||||
|
||||
The user input to you can be provided directly by the agent or as a command argument - you **MUST** consider it before proceeding with the prompt (if not empty).
|
||||
|
||||
User input:
|
||||
|
||||
$ARGUMENTS
|
||||
|
||||
Goal: Identify inconsistencies, duplications, ambiguities, and underspecified items across the three core artifacts (`spec.md`, `plan.md`, `tasks.md`) before implementation. This command MUST run only after `/tasks` has successfully produced a complete `tasks.md`.
|
||||
|
||||
STRICTLY READ-ONLY: Do **not** modify any files. Output a structured analysis report. Offer an optional remediation plan (user must explicitly approve before any follow-up editing commands would be invoked manually).
|
||||
|
||||
Constitution Authority: The project constitution (`.specify/memory/constitution.md`) is **non-negotiable** within this analysis scope. Constitution conflicts are automatically CRITICAL and require adjustment of the spec, plan, or tasks—not dilution, reinterpretation, or silent ignoring of the principle. If a principle itself needs to change, that must occur in a separate, explicit constitution update outside `/analyze`.
|
||||
|
||||
Execution steps:
|
||||
|
||||
1. Run `.specify/scripts/bash/check-prerequisites.sh --json --require-tasks --include-tasks` once from repo root and parse JSON for FEATURE_DIR and AVAILABLE_DOCS. Derive absolute paths:
|
||||
- SPEC = FEATURE_DIR/spec.md
|
||||
- PLAN = FEATURE_DIR/plan.md
|
||||
- TASKS = FEATURE_DIR/tasks.md
|
||||
Abort with an error message if any required file is missing (instruct the user to run missing prerequisite command).
|
||||
|
||||
2. Load artifacts:
|
||||
- Parse spec.md sections: Overview/Context, Functional Requirements, Non-Functional Requirements, User Stories, Edge Cases (if present).
|
||||
- Parse plan.md: Architecture/stack choices, Data Model references, Phases, Technical constraints.
|
||||
- Parse tasks.md: Task IDs, descriptions, phase grouping, parallel markers [P], referenced file paths.
|
||||
- Load constitution `.specify/memory/constitution.md` for principle validation.
|
||||
|
||||
3. Build internal semantic models:
|
||||
- Requirements inventory: Each functional + non-functional requirement with a stable key (derive slug based on imperative phrase; e.g., "User can upload file" -> `user-can-upload-file`).
|
||||
- User story/action inventory.
|
||||
- Task coverage mapping: Map each task to one or more requirements or stories (inference by keyword / explicit reference patterns like IDs or key phrases).
|
||||
- Constitution rule set: Extract principle names and any MUST/SHOULD normative statements.
|
||||
|
||||
4. Detection passes:
|
||||
A. Duplication detection:
|
||||
- Identify near-duplicate requirements. Mark lower-quality phrasing for consolidation.
|
||||
B. Ambiguity detection:
|
||||
- Flag vague adjectives (fast, scalable, secure, intuitive, robust) lacking measurable criteria.
|
||||
- Flag unresolved placeholders (TODO, TKTK, ???, <placeholder>, etc.).
|
||||
C. Underspecification:
|
||||
- Requirements with verbs but missing object or measurable outcome.
|
||||
- User stories missing acceptance criteria alignment.
|
||||
- Tasks referencing files or components not defined in spec/plan.
|
||||
D. Constitution alignment:
|
||||
- Any requirement or plan element conflicting with a MUST principle.
|
||||
- Missing mandated sections or quality gates from constitution.
|
||||
E. Coverage gaps:
|
||||
- Requirements with zero associated tasks.
|
||||
- Tasks with no mapped requirement/story.
|
||||
- Non-functional requirements not reflected in tasks (e.g., performance, security).
|
||||
F. Inconsistency:
|
||||
- Terminology drift (same concept named differently across files).
|
||||
- Data entities referenced in plan but absent in spec (or vice versa).
|
||||
- Task ordering contradictions (e.g., integration tasks before foundational setup tasks without dependency note).
|
||||
- Conflicting requirements (e.g., one requires to use Next.js while other says to use Vue as the framework).
|
||||
|
||||
5. Severity assignment heuristic:
|
||||
- CRITICAL: Violates constitution MUST, missing core spec artifact, or requirement with zero coverage that blocks baseline functionality.
|
||||
- HIGH: Duplicate or conflicting requirement, ambiguous security/performance attribute, untestable acceptance criterion.
|
||||
- MEDIUM: Terminology drift, missing non-functional task coverage, underspecified edge case.
|
||||
- LOW: Style/wording improvements, minor redundancy not affecting execution order.
|
||||
|
||||
6. Produce a Markdown report (no file writes) with sections:
|
||||
|
||||
### Specification Analysis Report
|
||||
| ID | Category | Severity | Location(s) | Summary | Recommendation |
|
||||
|----|----------|----------|-------------|---------|----------------|
|
||||
| A1 | Duplication | HIGH | spec.md:L120-134 | Two similar requirements ... | Merge phrasing; keep clearer version |
|
||||
(Add one row per finding; generate stable IDs prefixed by category initial.)
|
||||
|
||||
Additional subsections:
|
||||
- Coverage Summary Table:
|
||||
| Requirement Key | Has Task? | Task IDs | Notes |
|
||||
- Constitution Alignment Issues (if any)
|
||||
- Unmapped Tasks (if any)
|
||||
- Metrics:
|
||||
* Total Requirements
|
||||
* Total Tasks
|
||||
* Coverage % (requirements with >=1 task)
|
||||
* Ambiguity Count
|
||||
* Duplication Count
|
||||
* Critical Issues Count
|
||||
|
||||
7. At end of report, output a concise Next Actions block:
|
||||
- If CRITICAL issues exist: Recommend resolving before `/implement`.
|
||||
- If only LOW/MEDIUM: User may proceed, but provide improvement suggestions.
|
||||
- Provide explicit command suggestions: e.g., "Run /specify with refinement", "Run /plan to adjust architecture", "Manually edit tasks.md to add coverage for 'performance-metrics'".
|
||||
|
||||
8. Ask the user: "Would you like me to suggest concrete remediation edits for the top N issues?" (Do NOT apply them automatically.)
|
||||
|
||||
Behavior rules:
|
||||
- NEVER modify files.
|
||||
- NEVER hallucinate missing sections—if absent, report them.
|
||||
- KEEP findings deterministic: if rerun without changes, produce consistent IDs and counts.
|
||||
- LIMIT total findings in the main table to 50; aggregate remainder in a summarized overflow note.
|
||||
- If zero issues found, emit a success report with coverage statistics and proceed recommendation.
|
||||
|
||||
Context: $ARGUMENTS
|
||||
@@ -0,0 +1,158 @@
|
||||
---
|
||||
description: Identify underspecified areas in the current feature spec by asking up to 5 highly targeted clarification questions and encoding answers back into the spec.
|
||||
---
|
||||
|
||||
The user input to you can be provided directly by the agent or as a command argument - you **MUST** consider it before proceeding with the prompt (if not empty).
|
||||
|
||||
User input:
|
||||
|
||||
$ARGUMENTS
|
||||
|
||||
Goal: Detect and reduce ambiguity or missing decision points in the active feature specification and record the clarifications directly in the spec file.
|
||||
|
||||
Note: This clarification workflow is expected to run (and be completed) BEFORE invoking `/plan`. If the user explicitly states they are skipping clarification (e.g., exploratory spike), you may proceed, but must warn that downstream rework risk increases.
|
||||
|
||||
Execution steps:
|
||||
|
||||
1. Run `.specify/scripts/bash/check-prerequisites.sh --json --paths-only` from repo root **once** (combined `--json --paths-only` mode / `-Json -PathsOnly`). Parse minimal JSON payload fields:
|
||||
- `FEATURE_DIR`
|
||||
- `FEATURE_SPEC`
|
||||
- (Optionally capture `IMPL_PLAN`, `TASKS` for future chained flows.)
|
||||
- If JSON parsing fails, abort and instruct user to re-run `/specify` or verify feature branch environment.
|
||||
|
||||
2. Load the current spec file. Perform a structured ambiguity & coverage scan using this taxonomy. For each category, mark status: Clear / Partial / Missing. Produce an internal coverage map used for prioritization (do not output raw map unless no questions will be asked).
|
||||
|
||||
Functional Scope & Behavior:
|
||||
- Core user goals & success criteria
|
||||
- Explicit out-of-scope declarations
|
||||
- User roles / personas differentiation
|
||||
|
||||
Domain & Data Model:
|
||||
- Entities, attributes, relationships
|
||||
- Identity & uniqueness rules
|
||||
- Lifecycle/state transitions
|
||||
- Data volume / scale assumptions
|
||||
|
||||
Interaction & UX Flow:
|
||||
- Critical user journeys / sequences
|
||||
- Error/empty/loading states
|
||||
- Accessibility or localization notes
|
||||
|
||||
Non-Functional Quality Attributes:
|
||||
- Performance (latency, throughput targets)
|
||||
- Scalability (horizontal/vertical, limits)
|
||||
- Reliability & availability (uptime, recovery expectations)
|
||||
- Observability (logging, metrics, tracing signals)
|
||||
- Security & privacy (authN/Z, data protection, threat assumptions)
|
||||
- Compliance / regulatory constraints (if any)
|
||||
|
||||
Integration & External Dependencies:
|
||||
- External services/APIs and failure modes
|
||||
- Data import/export formats
|
||||
- Protocol/versioning assumptions
|
||||
|
||||
Edge Cases & Failure Handling:
|
||||
- Negative scenarios
|
||||
- Rate limiting / throttling
|
||||
- Conflict resolution (e.g., concurrent edits)
|
||||
|
||||
Constraints & Tradeoffs:
|
||||
- Technical constraints (language, storage, hosting)
|
||||
- Explicit tradeoffs or rejected alternatives
|
||||
|
||||
Terminology & Consistency:
|
||||
- Canonical glossary terms
|
||||
- Avoided synonyms / deprecated terms
|
||||
|
||||
Completion Signals:
|
||||
- Acceptance criteria testability
|
||||
- Measurable Definition of Done style indicators
|
||||
|
||||
Misc / Placeholders:
|
||||
- TODO markers / unresolved decisions
|
||||
- Ambiguous adjectives ("robust", "intuitive") lacking quantification
|
||||
|
||||
For each category with Partial or Missing status, add a candidate question opportunity unless:
|
||||
- Clarification would not materially change implementation or validation strategy
|
||||
- Information is better deferred to planning phase (note internally)
|
||||
|
||||
3. Generate (internally) a prioritized queue of candidate clarification questions (maximum 5). Do NOT output them all at once. Apply these constraints:
|
||||
- Maximum of 5 total questions across the whole session.
|
||||
- Each question must be answerable with EITHER:
|
||||
* A short multiple‑choice selection (2–5 distinct, mutually exclusive options), OR
|
||||
* A one-word / short‑phrase answer (explicitly constrain: "Answer in <=5 words").
|
||||
- Only include questions whose answers materially impact architecture, data modeling, task decomposition, test design, UX behavior, operational readiness, or compliance validation.
|
||||
- Ensure category coverage balance: attempt to cover the highest impact unresolved categories first; avoid asking two low-impact questions when a single high-impact area (e.g., security posture) is unresolved.
|
||||
- Exclude questions already answered, trivial stylistic preferences, or plan-level execution details (unless blocking correctness).
|
||||
- Favor clarifications that reduce downstream rework risk or prevent misaligned acceptance tests.
|
||||
- If more than 5 categories remain unresolved, select the top 5 by (Impact * Uncertainty) heuristic.
|
||||
|
||||
4. Sequential questioning loop (interactive):
|
||||
- Present EXACTLY ONE question at a time.
|
||||
- For multiple‑choice questions render options as a Markdown table:
|
||||
|
||||
| Option | Description |
|
||||
|--------|-------------|
|
||||
| A | <Option A description> |
|
||||
| B | <Option B description> |
|
||||
| C | <Option C description> | (add D/E as needed up to 5)
|
||||
| Short | Provide a different short answer (<=5 words) | (Include only if free-form alternative is appropriate)
|
||||
|
||||
- For short‑answer style (no meaningful discrete options), output a single line after the question: `Format: Short answer (<=5 words)`.
|
||||
- After the user answers:
|
||||
* Validate the answer maps to one option or fits the <=5 word constraint.
|
||||
* If ambiguous, ask for a quick disambiguation (count still belongs to same question; do not advance).
|
||||
* Once satisfactory, record it in working memory (do not yet write to disk) and move to the next queued question.
|
||||
- Stop asking further questions when:
|
||||
* All critical ambiguities resolved early (remaining queued items become unnecessary), OR
|
||||
* User signals completion ("done", "good", "no more"), OR
|
||||
* You reach 5 asked questions.
|
||||
- Never reveal future queued questions in advance.
|
||||
- If no valid questions exist at start, immediately report no critical ambiguities.
|
||||
|
||||
5. Integration after EACH accepted answer (incremental update approach):
|
||||
- Maintain in-memory representation of the spec (loaded once at start) plus the raw file contents.
|
||||
- For the first integrated answer in this session:
|
||||
* Ensure a `## Clarifications` section exists (create it just after the highest-level contextual/overview section per the spec template if missing).
|
||||
* Under it, create (if not present) a `### Session YYYY-MM-DD` subheading for today.
|
||||
- Append a bullet line immediately after acceptance: `- Q: <question> → A: <final answer>`.
|
||||
- Then immediately apply the clarification to the most appropriate section(s):
|
||||
* Functional ambiguity → Update or add a bullet in Functional Requirements.
|
||||
* User interaction / actor distinction → Update User Stories or Actors subsection (if present) with clarified role, constraint, or scenario.
|
||||
* Data shape / entities → Update Data Model (add fields, types, relationships) preserving ordering; note added constraints succinctly.
|
||||
* Non-functional constraint → Add/modify measurable criteria in Non-Functional / Quality Attributes section (convert vague adjective to metric or explicit target).
|
||||
* Edge case / negative flow → Add a new bullet under Edge Cases / Error Handling (or create such subsection if template provides placeholder for it).
|
||||
* Terminology conflict → Normalize term across spec; retain original only if necessary by adding `(formerly referred to as "X")` once.
|
||||
- If the clarification invalidates an earlier ambiguous statement, replace that statement instead of duplicating; leave no obsolete contradictory text.
|
||||
- Save the spec file AFTER each integration to minimize risk of context loss (atomic overwrite).
|
||||
- Preserve formatting: do not reorder unrelated sections; keep heading hierarchy intact.
|
||||
- Keep each inserted clarification minimal and testable (avoid narrative drift).
|
||||
|
||||
6. Validation (performed after EACH write plus final pass):
|
||||
- Clarifications session contains exactly one bullet per accepted answer (no duplicates).
|
||||
- Total asked (accepted) questions ≤ 5.
|
||||
- Updated sections contain no lingering vague placeholders the new answer was meant to resolve.
|
||||
- No contradictory earlier statement remains (scan for now-invalid alternative choices removed).
|
||||
- Markdown structure valid; only allowed new headings: `## Clarifications`, `### Session YYYY-MM-DD`.
|
||||
- Terminology consistency: same canonical term used across all updated sections.
|
||||
|
||||
7. Write the updated spec back to `FEATURE_SPEC`.
|
||||
|
||||
8. Report completion (after questioning loop ends or early termination):
|
||||
- Number of questions asked & answered.
|
||||
- Path to updated spec.
|
||||
- Sections touched (list names).
|
||||
- Coverage summary table listing each taxonomy category with Status: Resolved (was Partial/Missing and addressed), Deferred (exceeds question quota or better suited for planning), Clear (already sufficient), Outstanding (still Partial/Missing but low impact).
|
||||
- If any Outstanding or Deferred remain, recommend whether to proceed to `/plan` or run `/clarify` again later post-plan.
|
||||
- Suggested next command.
|
||||
|
||||
Behavior rules:
|
||||
- If no meaningful ambiguities found (or all potential questions would be low-impact), respond: "No critical ambiguities detected worth formal clarification." and suggest proceeding.
|
||||
- If spec file missing, instruct user to run `/specify` first (do not create a new spec here).
|
||||
- Never exceed 5 total asked questions (clarification retries for a single question do not count as new questions).
|
||||
- Avoid speculative tech stack questions unless the absence blocks functional clarity.
|
||||
- Respect user early termination signals ("stop", "done", "proceed").
|
||||
- If no questions asked due to full coverage, output a compact coverage summary (all categories Clear) then suggest advancing.
|
||||
- If quota reached with unresolved high-impact categories remaining, explicitly flag them under Deferred with rationale.
|
||||
|
||||
Context for prioritization: $ARGUMENTS
|
||||
@@ -0,0 +1,73 @@
|
||||
---
|
||||
description: Create or update the project constitution from interactive or provided principle inputs, ensuring all dependent templates stay in sync.
|
||||
---
|
||||
|
||||
The user input to you can be provided directly by the agent or as a command argument - you **MUST** consider it before proceeding with the prompt (if not empty).
|
||||
|
||||
User input:
|
||||
|
||||
$ARGUMENTS
|
||||
|
||||
You are updating the project constitution at `.specify/memory/constitution.md`. This file is a TEMPLATE containing placeholder tokens in square brackets (e.g. `[PROJECT_NAME]`, `[PRINCIPLE_1_NAME]`). Your job is to (a) collect/derive concrete values, (b) fill the template precisely, and (c) propagate any amendments across dependent artifacts.
|
||||
|
||||
Follow this execution flow:
|
||||
|
||||
1. Load the existing constitution template at `.specify/memory/constitution.md`.
|
||||
- Identify every placeholder token of the form `[ALL_CAPS_IDENTIFIER]`.
|
||||
**IMPORTANT**: The user might require less or more principles than the ones used in the template. If a number is specified, respect that - follow the general template. You will update the doc accordingly.
|
||||
|
||||
2. Collect/derive values for placeholders:
|
||||
- If user input (conversation) supplies a value, use it.
|
||||
- Otherwise infer from existing repo context (README, docs, prior constitution versions if embedded).
|
||||
- For governance dates: `RATIFICATION_DATE` is the original adoption date (if unknown ask or mark TODO), `LAST_AMENDED_DATE` is today if changes are made, otherwise keep previous.
|
||||
- `CONSTITUTION_VERSION` must increment according to semantic versioning rules:
|
||||
* MAJOR: Backward incompatible governance/principle removals or redefinitions.
|
||||
* MINOR: New principle/section added or materially expanded guidance.
|
||||
* PATCH: Clarifications, wording, typo fixes, non-semantic refinements.
|
||||
- If version bump type ambiguous, propose reasoning before finalizing.
|
||||
|
||||
3. Draft the updated constitution content:
|
||||
- Replace every placeholder with concrete text (no bracketed tokens left except intentionally retained template slots that the project has chosen not to define yet—explicitly justify any left).
|
||||
- Preserve heading hierarchy and comments can be removed once replaced unless they still add clarifying guidance.
|
||||
- Ensure each Principle section: succinct name line, paragraph (or bullet list) capturing non‑negotiable rules, explicit rationale if not obvious.
|
||||
- Ensure Governance section lists amendment procedure, versioning policy, and compliance review expectations.
|
||||
|
||||
4. Consistency propagation checklist (convert prior checklist into active validations):
|
||||
- Read `.specify/templates/plan-template.md` and ensure any "Constitution Check" or rules align with updated principles.
|
||||
- Read `.specify/templates/spec-template.md` for scope/requirements alignment—update if constitution adds/removes mandatory sections or constraints.
|
||||
- Read `.specify/templates/tasks-template.md` and ensure task categorization reflects new or removed principle-driven task types (e.g., observability, versioning, testing discipline).
|
||||
- Read each command file in `.specify/templates/commands/*.md` (including this one) to verify no outdated references (agent-specific names like CLAUDE only) remain when generic guidance is required.
|
||||
- Read any runtime guidance docs (e.g., `README.md`, `docs/quickstart.md`, or agent-specific guidance files if present). Update references to principles changed.
|
||||
|
||||
5. Produce a Sync Impact Report (prepend as an HTML comment at top of the constitution file after update):
|
||||
- Version change: old → new
|
||||
- List of modified principles (old title → new title if renamed)
|
||||
- Added sections
|
||||
- Removed sections
|
||||
- Templates requiring updates (✅ updated / ⚠ pending) with file paths
|
||||
- Follow-up TODOs if any placeholders intentionally deferred.
|
||||
|
||||
6. Validation before final output:
|
||||
- No remaining unexplained bracket tokens.
|
||||
- Version line matches report.
|
||||
- Dates ISO format YYYY-MM-DD.
|
||||
- Principles are declarative, testable, and free of vague language ("should" → replace with MUST/SHOULD rationale where appropriate).
|
||||
|
||||
7. Write the completed constitution back to `.specify/memory/constitution.md` (overwrite).
|
||||
|
||||
8. Output a final summary to the user with:
|
||||
- New version and bump rationale.
|
||||
- Any files flagged for manual follow-up.
|
||||
- Suggested commit message (e.g., `docs: amend constitution to vX.Y.Z (principle additions + governance update)`).
|
||||
|
||||
Formatting & Style Requirements:
|
||||
- Use Markdown headings exactly as in the template (do not demote/promote levels).
|
||||
- Wrap long rationale lines to keep readability (<100 chars ideally) but do not hard enforce with awkward breaks.
|
||||
- Keep a single blank line between sections.
|
||||
- Avoid trailing whitespace.
|
||||
|
||||
If the user supplies partial updates (e.g., only one principle revision), still perform validation and version decision steps.
|
||||
|
||||
If critical info missing (e.g., ratification date truly unknown), insert `TODO(<FIELD_NAME>): explanation` and include in the Sync Impact Report under deferred items.
|
||||
|
||||
Do not create a new template; always operate on the existing `.specify/memory/constitution.md` file.
|
||||
@@ -0,0 +1,56 @@
|
||||
---
|
||||
description: Execute the implementation plan by processing and executing all tasks defined in tasks.md
|
||||
---
|
||||
|
||||
The user input can be provided directly by the agent or as a command argument - you **MUST** consider it before proceeding with the prompt (if not empty).
|
||||
|
||||
User input:
|
||||
|
||||
$ARGUMENTS
|
||||
|
||||
1. Run `.specify/scripts/bash/check-prerequisites.sh --json --require-tasks --include-tasks` from repo root and parse FEATURE_DIR and AVAILABLE_DOCS list. All paths must be absolute.
|
||||
|
||||
2. Load and analyze the implementation context:
|
||||
- **REQUIRED**: Read tasks.md for the complete task list and execution plan
|
||||
- **REQUIRED**: Read plan.md for tech stack, architecture, and file structure
|
||||
- **IF EXISTS**: Read data-model.md for entities and relationships
|
||||
- **IF EXISTS**: Read contracts/ for API specifications and test requirements
|
||||
- **IF EXISTS**: Read research.md for technical decisions and constraints
|
||||
- **IF EXISTS**: Read quickstart.md for integration scenarios
|
||||
|
||||
3. Parse tasks.md structure and extract:
|
||||
- **Task phases**: Setup, Tests, Core, Integration, Polish
|
||||
- **Task dependencies**: Sequential vs parallel execution rules
|
||||
- **Task details**: ID, description, file paths, parallel markers [P]
|
||||
- **Execution flow**: Order and dependency requirements
|
||||
|
||||
4. Execute implementation following the task plan:
|
||||
- **Phase-by-phase execution**: Complete each phase before moving to the next
|
||||
- **Respect dependencies**: Run sequential tasks in order, parallel tasks [P] can run together
|
||||
- **Follow TDD approach**: Execute test tasks before their corresponding implementation tasks
|
||||
- **File-based coordination**: Tasks affecting the same files must run sequentially
|
||||
- **Validation checkpoints**: Verify each phase completion before proceeding
|
||||
|
||||
5. Implementation execution rules:
|
||||
- **Setup first**: Initialize project structure, dependencies, configuration
|
||||
- **Tests before code**: If you need to write tests for contracts, entities, and integration scenarios
|
||||
- **Core development**: Implement models, services, CLI commands, endpoints
|
||||
- **Integration work**: Database connections, middleware, logging, external services
|
||||
- **Polish and validation**: Unit tests, performance optimization, documentation
|
||||
|
||||
6. Progress tracking and error handling:
|
||||
- Report progress after each completed task
|
||||
- Halt execution if any non-parallel task fails
|
||||
- For parallel tasks [P], continue with successful tasks, report failed ones
|
||||
- Provide clear error messages with context for debugging
|
||||
- Suggest next steps if implementation cannot proceed
|
||||
- **IMPORTANT** For completed tasks, make sure to mark the task off as [X] in the tasks file.
|
||||
|
||||
7. Completion validation:
|
||||
- Verify all required tasks are completed
|
||||
- Check that implemented features match the original specification
|
||||
- Validate that tests pass and coverage meets requirements
|
||||
- Confirm the implementation follows the technical plan
|
||||
- Report final status with summary of completed work
|
||||
|
||||
Note: This command assumes a complete task breakdown exists in tasks.md. If tasks are incomplete or missing, suggest running `/tasks` first to regenerate the task list.
|
||||
@@ -0,0 +1,43 @@
|
||||
---
|
||||
description: Execute the implementation planning workflow using the plan template to generate design artifacts.
|
||||
---
|
||||
|
||||
The user input to you can be provided directly by the agent or as a command argument - you **MUST** consider it before proceeding with the prompt (if not empty).
|
||||
|
||||
User input:
|
||||
|
||||
$ARGUMENTS
|
||||
|
||||
Given the implementation details provided as an argument, do this:
|
||||
|
||||
1. Run `.specify/scripts/bash/setup-plan.sh --json` from the repo root and parse JSON for FEATURE_SPEC, IMPL_PLAN, SPECS_DIR, BRANCH. All future file paths must be absolute.
|
||||
- BEFORE proceeding, inspect FEATURE_SPEC for a `## Clarifications` section with at least one `Session` subheading. If missing or clearly ambiguous areas remain (vague adjectives, unresolved critical choices), PAUSE and instruct the user to run `/clarify` first to reduce rework. Only continue if: (a) Clarifications exist OR (b) an explicit user override is provided (e.g., "proceed without clarification"). Do not attempt to fabricate clarifications yourself.
|
||||
2. Read and analyze the feature specification to understand:
|
||||
- The feature requirements and user stories
|
||||
- Functional and non-functional requirements
|
||||
- Success criteria and acceptance criteria
|
||||
- Any technical constraints or dependencies mentioned
|
||||
|
||||
3. Read the constitution at `.specify/memory/constitution.md` to understand constitutional requirements.
|
||||
|
||||
4. Execute the implementation plan template:
|
||||
- Load `.specify/templates/plan-template.md` (already copied to IMPL_PLAN path)
|
||||
- Set Input path to FEATURE_SPEC
|
||||
- Run the Execution Flow (main) function steps 1-9
|
||||
- The template is self-contained and executable
|
||||
- Follow error handling and gate checks as specified
|
||||
- Let the template guide artifact generation in $SPECS_DIR:
|
||||
* Phase 0 generates research.md
|
||||
* Phase 1 generates data-model.md, contracts/, quickstart.md
|
||||
* Phase 2 generates tasks.md
|
||||
- Incorporate user-provided details from arguments into Technical Context: $ARGUMENTS
|
||||
- Update Progress Tracking as you complete each phase
|
||||
|
||||
5. Verify execution completed:
|
||||
- Check Progress Tracking shows all phases complete
|
||||
- Ensure all required artifacts were generated
|
||||
- Confirm no ERROR states in execution
|
||||
|
||||
6. Report results with branch name, file paths, and generated artifacts.
|
||||
|
||||
Use absolute paths with the repository root for all file operations to avoid path issues.
|
||||
@@ -0,0 +1,21 @@
|
||||
---
|
||||
description: Create or update the feature specification from a natural language feature description.
|
||||
---
|
||||
|
||||
The user input to you can be provided directly by the agent or as a command argument - you **MUST** consider it before proceeding with the prompt (if not empty).
|
||||
|
||||
User input:
|
||||
|
||||
$ARGUMENTS
|
||||
|
||||
The text the user typed after `/specify` in the triggering message **is** the feature description. Assume you always have it available in this conversation even if `$ARGUMENTS` appears literally below. Do not ask the user to repeat it unless they provided an empty command.
|
||||
|
||||
Given that feature description, do this:
|
||||
|
||||
1. Run the script `.specify/scripts/bash/create-new-feature.sh --json "$ARGUMENTS"` from repo root and parse its JSON output for BRANCH_NAME and SPEC_FILE. All file paths must be absolute.
|
||||
**IMPORTANT** You must only ever run this script once. The JSON is provided in the terminal as output - always refer to it to get the actual content you're looking for.
|
||||
2. Load `.specify/templates/spec-template.md` to understand required sections.
|
||||
3. Write the specification to SPEC_FILE using the template structure, replacing placeholders with concrete details derived from the feature description (arguments) while preserving section order and headings.
|
||||
4. Report completion with branch name, spec file path, and readiness for the next phase.
|
||||
|
||||
Note: The script creates and checks out the new branch and initializes the spec file before writing.
|
||||
@@ -0,0 +1,62 @@
|
||||
---
|
||||
description: Generate an actionable, dependency-ordered tasks.md for the feature based on available design artifacts.
|
||||
---
|
||||
|
||||
The user input to you can be provided directly by the agent or as a command argument - you **MUST** consider it before proceeding with the prompt (if not empty).
|
||||
|
||||
User input:
|
||||
|
||||
$ARGUMENTS
|
||||
|
||||
1. Run `.specify/scripts/bash/check-prerequisites.sh --json` from repo root and parse FEATURE_DIR and AVAILABLE_DOCS list. All paths must be absolute.
|
||||
2. Load and analyze available design documents:
|
||||
- Always read plan.md for tech stack and libraries
|
||||
- IF EXISTS: Read data-model.md for entities
|
||||
- IF EXISTS: Read contracts/ for API endpoints
|
||||
- IF EXISTS: Read research.md for technical decisions
|
||||
- IF EXISTS: Read quickstart.md for test scenarios
|
||||
|
||||
Note: Not all projects have all documents. For example:
|
||||
- CLI tools might not have contracts/
|
||||
- Simple libraries might not need data-model.md
|
||||
- Generate tasks based on what's available
|
||||
|
||||
3. Generate tasks following the template:
|
||||
- Use `.specify/templates/tasks-template.md` as the base
|
||||
- Replace example tasks with actual tasks based on:
|
||||
* **Setup tasks**: Project init, dependencies, linting
|
||||
* **Test tasks [P]**: One per contract, one per integration scenario
|
||||
* **Core tasks**: One per entity, service, CLI command, endpoint
|
||||
* **Integration tasks**: DB connections, middleware, logging
|
||||
* **Polish tasks [P]**: Unit tests, performance, docs
|
||||
|
||||
4. Task generation rules:
|
||||
- Each contract file → contract test task marked [P]
|
||||
- Each entity in data-model → model creation task marked [P]
|
||||
- Each endpoint → implementation task (not parallel if shared files)
|
||||
- Each user story → integration test marked [P]
|
||||
- Different files = can be parallel [P]
|
||||
- Same file = sequential (no [P])
|
||||
|
||||
5. Order tasks by dependencies:
|
||||
- Setup before everything
|
||||
- Tests before implementation (TDD)
|
||||
- Models before services
|
||||
- Services before endpoints
|
||||
- Core before integration
|
||||
- Everything before polish
|
||||
|
||||
6. Include parallel execution examples:
|
||||
- Group [P] tasks that can run together
|
||||
- Show actual Task agent commands
|
||||
|
||||
7. Create FEATURE_DIR/tasks.md with:
|
||||
- Correct feature name from implementation plan
|
||||
- Numbered tasks (T001, T002, etc.)
|
||||
- Clear file paths for each task
|
||||
- Dependency notes
|
||||
- Parallel execution guidance
|
||||
|
||||
Context for task generation: $ARGUMENTS
|
||||
|
||||
The tasks.md should be immediately executable - each task must be specific enough that an LLM can complete it without additional context.
|
||||
@@ -0,0 +1,368 @@
|
||||
---
|
||||
name: Anchor Sequence Optimization
|
||||
overview: Replace the current multiprocessing-based anchor sequence algorithm with an efficient single-threaded implementation using hash-based n-gram indexing, greedy longest-first matching, and simplified scoring to achieve sub-30-second performance in both local and cloud environments.
|
||||
todos:
|
||||
- id: create-golden-fixtures
|
||||
content: Create script to generate 10 golden test fixtures from representative song data (varying lengths/complexity), run current algorithm to capture expected anchor outputs
|
||||
status: completed
|
||||
- id: create-golden-test
|
||||
content: Create test_anchor_sequence_golden.py with regression tests that verify anchor finder output matches fixtures, ensure tests pass with current implementation before any changes
|
||||
status: completed
|
||||
- id: remove-multiprocessing
|
||||
content: Remove multiprocessing.Pool from find_anchors() and _remove_overlapping_sequences(), replace with single-threaded loops
|
||||
status: completed
|
||||
- id: build-ngram-index
|
||||
content: Add _build_ngram_index() method to create hash-based lookup table for O(1) n-gram matching
|
||||
status: completed
|
||||
- id: greedy-skip
|
||||
content: Add covered_positions set tracking and skip logic to avoid processing overlapping n-grams
|
||||
status: completed
|
||||
- id: simple-scoring
|
||||
content: Add _simple_score_anchor() with rule-based heuristics, make spaCy scoring optional via use_nlp_scoring flag
|
||||
status: completed
|
||||
- id: optimize-overlap
|
||||
content: Replace O(n^2) overlap removal with O(n log n) sorted single-pass algorithm
|
||||
status: completed
|
||||
- id: add-perf-tests
|
||||
content: Add performance benchmark test asserting <30 second completion for all fixture sizes
|
||||
status: pending
|
||||
---
|
||||
|
||||
# Anchor Sequence Algorithm Optimization
|
||||
|
||||
## Problem Analysis
|
||||
|
||||
The current `AnchorSequenceFinder` has critical performance issues:
|
||||
|
||||
### Root Causes of Hanging
|
||||
|
||||
1. **Multiprocessing Pool with fork()**: Python's `Pool()` uses `fork()` on Linux, which causes deadlocks in Cloud Run containers
|
||||
2. **spaCy model loading in forked processes**: The `PhraseAnalyzer` loads spaCy's `en_core_web_sm` model, which can hang when forked
|
||||
3. **Pool creation blocking**: The hang occurs before job submission, so timeout checks never trigger
|
||||
|
||||
### Algorithmic Inefficiencies
|
||||
|
||||
1. **O(n^2) n-gram matching**: Linear search through references for each n-gram
|
||||
2. **Redundant work**: Processes all n-gram lengths (3 to ~30) even though longer matches are preferred
|
||||
3. **O(n^2) overlap removal**: Each anchor checked against all previous anchors
|
||||
4. **Expensive phrase scoring**: spaCy NLP called twice per anchor (phrase + context)
|
||||
|
||||
## Proposed Solution: Hash-Based Greedy Algorithm
|
||||
|
||||
### Core Algorithm Changes
|
||||
|
||||
**1. Hash-Based N-gram Index (O(1) lookup instead of O(n))**
|
||||
|
||||
- Pre-build a dictionary mapping n-grams to their positions in each reference
|
||||
- Use tuple of cleaned words as key for O(1) lookup
|
||||
- Build index once during preprocessing
|
||||
|
||||
**2. Greedy Longest-First Matching**
|
||||
|
||||
- Process n-gram lengths from longest to shortest (current behavior)
|
||||
- BUT: Skip positions already covered by longer anchors immediately
|
||||
- Use a "covered positions" set to avoid redundant work
|
||||
|
||||
**3. Single-Threaded Processing**
|
||||
|
||||
- Remove all `multiprocessing.Pool` usage
|
||||
- Use efficient algorithms instead of parallelism
|
||||
- Works identically in local and cloud environments
|
||||
|
||||
**4. Simplified Phrase Scoring (Optional)**
|
||||
|
||||
- Make spaCy-based scoring optional via configuration flag
|
||||
- Default to rule-based heuristic scoring:
|
||||
- Prefer sequences starting/ending at word boundaries
|
||||
- Prefer sequences matching more reference sources
|
||||
- Prefer longer sequences
|
||||
- Option to enable full NLP scoring for higher-quality results offline
|
||||
|
||||
**5. O(n log n) Overlap Removal**
|
||||
|
||||
- Sort anchors by position
|
||||
- Single pass with interval tracking using sorted list
|
||||
- Skip anchors that overlap with any already-selected anchor
|
||||
|
||||
### New Data Structures
|
||||
|
||||
```python
|
||||
# N-gram index: maps (word1, word2, ...) -> {source: [positions]}
|
||||
ngram_index: Dict[Tuple[str, ...], Dict[str, List[int]]]
|
||||
|
||||
# Covered positions in transcription (for greedy skip)
|
||||
covered_trans_positions: Set[int]
|
||||
```
|
||||
|
||||
### Performance Targets
|
||||
|
||||
- **Target**: Complete in under 30 seconds for any song
|
||||
- **Typical song**: 200-400 words, 3 reference sources
|
||||
- **Expected performance**: 1-5 seconds for typical inputs
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
### Execution Order (Critical)
|
||||
|
||||
1. **Phase 0**: Create golden test fixtures and verify tests pass with current code
|
||||
2. **Phase 1-5**: Implement optimizations incrementally, running golden tests after each phase
|
||||
3. **Final**: Add performance benchmarks
|
||||
|
||||
### Files to Create/Modify
|
||||
|
||||
- `lyrics_transcriber_temp/scripts/generate_anchor_fixtures.py` (new)
|
||||
- `lyrics_transcriber_temp/tests/fixtures/anchor_golden/*.json` (new)
|
||||
- `lyrics_transcriber_temp/tests/unit/correction/test_anchor_sequence_golden.py` (new)
|
||||
- `lyrics_transcriber_temp/lyrics_transcriber/correction/anchor_sequence.py` (modify)
|
||||
|
||||
### Key Changes
|
||||
|
||||
**Phase 0: Golden Test Setup** (MUST complete and pass before any algorithm changes)
|
||||
|
||||
- Create fixture generation script with 10 representative test cases
|
||||
- Run script to generate fixtures using current implementation
|
||||
- Create golden test file that loads fixtures and verifies outputs
|
||||
- Verify all golden tests pass with current code
|
||||
|
||||
**Phase 1: Remove Multiprocessing**
|
||||
|
||||
- Replace `Pool().apply_async()` with direct function calls
|
||||
- Remove all pool creation, cleanup, and timeout handling
|
||||
- Single-threaded loop through n-gram lengths
|
||||
|
||||
**Phase 2: Add Hash-Based Index**
|
||||
|
||||
- New method `_build_ngram_index()` that builds lookup table during preprocessing
|
||||
- Modify `_find_matching_sources()` to use O(1) index lookup
|
||||
- Index built once, shared across all n-gram length iterations
|
||||
|
||||
**Phase 3: Greedy Position Skipping**
|
||||
|
||||
- Track covered transcription positions in a set
|
||||
- Skip n-grams that start in already-covered positions
|
||||
- Reduces redundant work significantly for songs with good matches
|
||||
|
||||
**Phase 4: Simplify Scoring**
|
||||
|
||||
- Add `use_nlp_scoring: bool = False` parameter to constructor
|
||||
- New method `_simple_score_anchor()` using rule-based heuristics
|
||||
- Keep spaCy scoring available for offline/high-quality mode
|
||||
|
||||
**Phase 5: Optimize Overlap Removal**
|
||||
|
||||
- Sort anchors by (start_position, -length)
|
||||
- Single pass: keep anchor if no overlap with last kept anchor
|
||||
- O(n log n) total instead of O(n^2)
|
||||
|
||||
## API Compatibility
|
||||
|
||||
The public interface remains unchanged:
|
||||
|
||||
- `find_anchors(transcribed, references, transcription_result) -> List[ScoredAnchor]`
|
||||
- `find_gaps(transcribed, anchors, references, transcription_result) -> List[GapSequence]`
|
||||
|
||||
New optional constructor parameters:
|
||||
|
||||
- `use_nlp_scoring: bool = False` - Enable spaCy-based phrase scoring (slower but higher quality)
|
||||
- Timeout parameter retained but less critical with efficient algorithm
|
||||
|
||||
## Fallback Strategy
|
||||
|
||||
If performance is still insufficient for edge cases:
|
||||
|
||||
1. **Early termination**: Stop after finding sufficient coverage (e.g., 80% of words anchored)
|
||||
2. **Adaptive n-gram range**: Start with medium lengths (8-15), expand only if needed
|
||||
3. **Sampling**: For very long songs, sample n-gram positions instead of exhaustive search
|
||||
|
||||
## Phase 0: Golden Test Setup (BEFORE any algorithm changes)
|
||||
|
||||
### Goal
|
||||
|
||||
Create regression tests using REAL song data from local cache to verify the optimized algorithm produces equivalent results.
|
||||
|
||||
### Cache File Structure (Understanding)
|
||||
|
||||
The local cache at `~/lyrics-transcriber-cache/` contains:
|
||||
|
||||
- **Reference lyrics**: `{provider}_{hash}_converted.json` where provider is spotify/genius/lrclib/musixmatch
|
||||
- Hash is based on (artist, title) lookup - same song shares same hash across providers
|
||||
- `_raw.json` contains metadata (track name, artist)
|
||||
- `_converted.json` contains LyricsData structure with segments/words
|
||||
- **Transcriptions**: `audioshake_{hash}_converted.json`
|
||||
- Hash is based on audio file content (different from reference hash)
|
||||
- Contains TranscriptionData structure
|
||||
- **Anchor results**: `anchors_{hash}.json`
|
||||
- Hash is computed from combined inputs (transcription + all references)
|
||||
|
||||
### Manual Song Selection
|
||||
|
||||
Identify 10 songs from cache with good coverage. Search raw files for track names:
|
||||
|
||||
### Finding Song Mappings
|
||||
|
||||
To link audioshake transcriptions to reference lyrics, search cache files:
|
||||
|
||||
```bash
|
||||
# Find songs by searching raw files for track names
|
||||
grep -h "trackName" ~/lyrics-transcriber-cache/spotify_*_raw.json | head -20
|
||||
|
||||
# Find audioshake files for a known song (e.g., search for filename patterns)
|
||||
ls ~/lyrics-transcriber-cache/audioshake_*_raw.json | head -20
|
||||
|
||||
# Search converted files for specific lyrics text
|
||||
grep -l "Napoleon did surrender" ~/lyrics-transcriber-cache/*_converted.json
|
||||
```
|
||||
|
||||
**Target song list** (reference_hash to be paired with audioshake_hash):
|
||||
|
||||
1. ABBA - Waterloo (refs: 3365f6a1e42388d84bcbfc0ac6f8aa6c) - short, repetitive chorus
|
||||
2. The Format - Time Bomb (refs: 3a3802ec28c4ab6689909b7ef784bb6f) - medium length
|
||||
3. Rancid - Time Bomb (refs: d02d20b854cbc6b890f274bd6c278752) - fast punk lyrics
|
||||
4. Minutemen - Storm in My House (refs: bc625704b26b307d8680ae5235b74994) - short
|
||||
5. + 6 more songs to be identified during fixture generation by exploring cache
|
||||
|
||||
### Fixture Generation Script
|
||||
|
||||
Create `lyrics_transcriber_temp/scripts/generate_anchor_fixtures.py`:
|
||||
|
||||
```python
|
||||
# Input: List of (name, audioshake_hash, reference_hash) tuples
|
||||
# For each:
|
||||
# 1. Load audioshake_{hash}_converted.json as transcription
|
||||
# 2. Load {spotify,genius,lrclib}_{hash}_converted.json as references
|
||||
# 3. Run current AnchorSequenceFinder
|
||||
# 4. Save complete fixture with inputs + outputs
|
||||
```
|
||||
|
||||
The script will:
|
||||
|
||||
1. Accept a config file listing the 10 song mappings
|
||||
2. Load real LyricsData from cache files
|
||||
3. Run the CURRENT AnchorSequenceFinder implementation
|
||||
4. Save fixtures that capture the real behavior
|
||||
|
||||
### Fixture Format
|
||||
|
||||
```
|
||||
lyrics_transcriber_temp/tests/fixtures/anchor_golden/
|
||||
- waterloo_abba.json
|
||||
- time_bomb_format.json
|
||||
- time_bomb_rancid.json
|
||||
- ...
|
||||
```
|
||||
|
||||
Each fixture contains REAL data:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "waterloo_abba",
|
||||
"description": "ABBA - Waterloo, short song with repetitive chorus",
|
||||
"source_hashes": {
|
||||
"audioshake": "1654b7c0755fad7f23bc888308f274de",
|
||||
"references": "3365f6a1e42388d84bcbfc0ac6f8aa6c"
|
||||
},
|
||||
"transcription": {
|
||||
"segments": [...], // Full LyricsData from audioshake cache
|
||||
"word_count": 150
|
||||
},
|
||||
"references": {
|
||||
"spotify": { "segments": [...] },
|
||||
"genius": { "segments": [...] },
|
||||
"lrclib": { "segments": [...] }
|
||||
},
|
||||
"expected_anchors": [
|
||||
{
|
||||
"text": "My my at Waterloo Napoleon did surrender",
|
||||
"transcription_position": 0,
|
||||
"length": 7,
|
||||
"reference_positions": {"spotify": 0, "genius": 0, "lrclib": 0},
|
||||
"confidence": 1.0
|
||||
},
|
||||
// ... all anchors from current implementation
|
||||
],
|
||||
"expected_anchor_count": 25,
|
||||
"expected_word_coverage": 142,
|
||||
"expected_coverage_percent": 94.6
|
||||
}
|
||||
```
|
||||
|
||||
### Golden Test
|
||||
|
||||
Create `lyrics_transcriber_temp/tests/unit/correction/test_anchor_sequence_golden.py`:
|
||||
|
||||
```python
|
||||
import pytest
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
FIXTURES_DIR = Path(__file__).parent.parent.parent / "fixtures" / "anchor_golden"
|
||||
|
||||
@pytest.fixture(params=list(FIXTURES_DIR.glob("*.json")))
|
||||
def golden_fixture(request):
|
||||
with open(request.param) as f:
|
||||
return json.load(f)
|
||||
|
||||
def test_anchor_count_matches(golden_fixture, anchor_finder):
|
||||
"""Anchor count should be within 10% of expected."""
|
||||
anchors = run_anchor_finder(golden_fixture)
|
||||
expected = golden_fixture["expected_anchor_count"]
|
||||
assert abs(len(anchors) - expected) <= expected * 0.1
|
||||
|
||||
def test_key_anchors_match(golden_fixture, anchor_finder):
|
||||
"""First, last, and longest anchors should match exactly."""
|
||||
anchors = run_anchor_finder(golden_fixture)
|
||||
expected = golden_fixture["expected_anchors"]
|
||||
# Compare first anchor
|
||||
assert anchors[0].text == expected[0]["text"]
|
||||
# Compare positions match
|
||||
...
|
||||
|
||||
def test_word_coverage(golden_fixture, anchor_finder):
|
||||
"""Total word coverage should be within 5% of expected."""
|
||||
anchors = run_anchor_finder(golden_fixture)
|
||||
coverage = calculate_coverage(anchors, golden_fixture)
|
||||
expected = golden_fixture["expected_coverage_percent"]
|
||||
assert abs(coverage - expected) <= 5.0
|
||||
|
||||
def test_no_invalid_positions(golden_fixture, anchor_finder):
|
||||
"""All anchor positions should be valid."""
|
||||
anchors = run_anchor_finder(golden_fixture)
|
||||
word_count = golden_fixture["transcription"]["word_count"]
|
||||
for anchor in anchors:
|
||||
assert 0 <= anchor.transcription_position < word_count
|
||||
```
|
||||
|
||||
### Validation Steps
|
||||
|
||||
1. **Generate fixtures from current implementation**:
|
||||
```bash
|
||||
cd lyrics_transcriber_temp
|
||||
python scripts/generate_anchor_fixtures.py
|
||||
```
|
||||
|
||||
2. **Verify fixtures were created**:
|
||||
```bash
|
||||
ls tests/fixtures/anchor_golden/
|
||||
# Should show 10 .json files
|
||||
```
|
||||
|
||||
3. **Run golden tests with CURRENT implementation**:
|
||||
```bash
|
||||
pytest tests/unit/correction/test_anchor_sequence_golden.py -v
|
||||
```
|
||||
|
||||
4. **All tests MUST pass before any algorithm changes**
|
||||
|
||||
5. **After each optimization phase, re-run golden tests**:
|
||||
```bash
|
||||
pytest tests/unit/correction/test_anchor_sequence_golden.py -v
|
||||
# Must still pass!
|
||||
```
|
||||
|
||||
|
||||
## Testing (Post-Optimization)
|
||||
|
||||
- Golden tests verify correctness (same outputs as before)
|
||||
- Existing unit tests verify API compatibility
|
||||
- Performance benchmark test with 30-second timeout assertion
|
||||
- Test with various song lengths via golden fixtures
|
||||
@@ -0,0 +1,21 @@
|
||||
---
|
||||
description: Git branching and PR workflow
|
||||
alwaysApply: true
|
||||
---
|
||||
|
||||
# Branching Workflow
|
||||
|
||||
**Never commit directly to `main`.** Always work on a feature branch.
|
||||
|
||||
Create descriptive branch names like `feature/add-discord-notifications`, `fix/audio-sync-issue`, or `refactor/cleanup-api-routes`.
|
||||
|
||||
# Creating a Pull Request
|
||||
|
||||
Once all changes are complete, tests pass, and commits are pushed, create a PR with:
|
||||
|
||||
1. **Summary** - Brief overview of the changes
|
||||
2. **Changes Made** - List of specific modifications
|
||||
3. **Testing** - How changes were tested (unit tests, manual testing)
|
||||
4. **Related Issues** - Reference any GitHub issues (e.g., "Fixes #123")
|
||||
|
||||
Only merge after review and CI checks pass.
|
||||
@@ -0,0 +1,14 @@
|
||||
# Infrastructure Changes
|
||||
|
||||
All Google Cloud infrastructure (IAM, Cloud Storage, Cloud Run, Secret Manager, Artifact Registry, etc.) must be managed via Pulumi in the `infrastructure/` folder.
|
||||
|
||||
## Rules
|
||||
|
||||
- **Reading/querying** with `gcloud` CLI is fine
|
||||
- **Creating or modifying** resources must go through IaC — never use `gcloud` or the Console to make changes
|
||||
|
||||
## Workflow
|
||||
|
||||
1. Make changes in `infrastructure/`
|
||||
2. Run `pulumi preview` and verify the output
|
||||
3. Run `pulumi up` to apply
|
||||
@@ -0,0 +1,27 @@
|
||||
---
|
||||
description: Version bumping rule for commits
|
||||
alwaysApply: true
|
||||
---
|
||||
|
||||
# Version Management
|
||||
|
||||
When committing and pushing changes, always bump the patch version in `pyproject.toml`.
|
||||
|
||||
The version is located at `tool.poetry.version` in `pyproject.toml`. For example, if the current version is `0.70.3`, bump it to `0.70.4`.
|
||||
|
||||
This ensures every commit pushed to the repository has a unique version number for tracking and deployment purposes.
|
||||
|
||||
# Testing Requirements
|
||||
|
||||
Before committing and pushing changes, ensure all tests pass:
|
||||
|
||||
```bash
|
||||
python -m pytest tests/ backend/tests/ -v -x --ignore=tests/integration
|
||||
```
|
||||
|
||||
Ensure all code touched is thoroughly covered by unit tests, we have a 70% coverage minimum in CI:
|
||||
```bash
|
||||
poetry run pytest tests/unit/ -v --cov=karaoke_gen --cov-report=xml --cov-report=term-missing --cov-fail-under=70
|
||||
```
|
||||
|
||||
Do not commit or push code that fails tests. Fix any test failures before proceeding.
|
||||
@@ -0,0 +1,73 @@
|
||||
# .gcloudignore - Exclude files from Cloud Build uploads
|
||||
# This dramatically reduces upload time (from 218MB to ~5-10MB)
|
||||
|
||||
# Git
|
||||
.git/
|
||||
.gitignore
|
||||
.gitmodules
|
||||
|
||||
# Node.js / Frontend
|
||||
node_modules/
|
||||
frontend-react/node_modules/
|
||||
frontend-react/dist/
|
||||
frontend-react/.vite/
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
.venv/
|
||||
venv/
|
||||
ENV/
|
||||
env/
|
||||
*.egg-info/
|
||||
.eggs/
|
||||
dist/
|
||||
build/
|
||||
*.egg
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
.DS_Store
|
||||
|
||||
# Testing
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
coverage.xml
|
||||
*.cover
|
||||
|
||||
# Documentation (not needed for Docker build)
|
||||
docs/
|
||||
|
||||
# Local development files
|
||||
lyrics_transcriber_local/
|
||||
*.local
|
||||
.env
|
||||
.env.*
|
||||
|
||||
# Large test files
|
||||
tests/data/*.mp3
|
||||
tests/data/*.wav
|
||||
tests/data/*.mp4
|
||||
tests/data/*.mkv
|
||||
|
||||
# Infrastructure (not needed for Docker build)
|
||||
infrastructure/
|
||||
|
||||
# Misc
|
||||
*.bak
|
||||
*.tmp
|
||||
Thumbs.db
|
||||
@@ -0,0 +1,128 @@
|
||||
# GitHub Actions CI/CD
|
||||
|
||||
This directory contains GitHub Actions workflows for automated testing and deployment.
|
||||
|
||||
## Workflows
|
||||
|
||||
### `test.yml` - Automated Testing & Deployment (CI/CD)
|
||||
|
||||
**Triggers:**
|
||||
- Push to `main` or `replace-modal-with-google-cloud` branches
|
||||
- Pull requests to `main`
|
||||
|
||||
**Jobs:**
|
||||
|
||||
#### Testing Phase (runs in parallel)
|
||||
|
||||
1. **Unit Tests** (~30s)
|
||||
- Runs 62 unit tests with mocked dependencies
|
||||
- Fast validation of business logic
|
||||
- Uses Python 3.12 with pip caching
|
||||
|
||||
2. **Emulator Integration Tests** (~2min)
|
||||
- Runs 11 integration tests against local GCP emulators
|
||||
- Tests real Firestore and GCS interactions
|
||||
- Requires:
|
||||
- Java 21 (for Firestore emulator)
|
||||
- Google Cloud SDK (for Firestore emulator)
|
||||
- Docker (for GCS emulator via fake-gcs-server)
|
||||
- Uses the same `scripts/run-emulator-tests.sh` as local development
|
||||
|
||||
3. **Code Quality** (~20s)
|
||||
- Basic Python syntax validation
|
||||
- Import checks
|
||||
- Optional: flake8, black, isort
|
||||
|
||||
#### Deployment Phase (runs after all tests pass)
|
||||
|
||||
4. **Deploy to Cloud Run** (~3-4min)
|
||||
- **Only runs on push to `replace-modal-with-google-cloud` branch** (not on PRs)
|
||||
- Builds Docker image and pushes to Artifact Registry
|
||||
- Deploys to Cloud Run with proper configuration
|
||||
- Verifies deployment health
|
||||
- **Requires GitHub secrets:**
|
||||
- `GCP_SA_KEY` - Service account key JSON
|
||||
- `ADMIN_TOKENS` - Admin authentication tokens
|
||||
|
||||
**Total Runtime:**
|
||||
- **Testing only** (PRs): ~3 minutes
|
||||
- **Testing + Deployment** (push to branch): ~7 minutes
|
||||
|
||||
## Local vs CI
|
||||
|
||||
The CI runs the exact same tests as local development:
|
||||
|
||||
```bash
|
||||
# Local development
|
||||
./scripts/run-tests.sh # Unit tests (same as CI job 1)
|
||||
./scripts/run-emulator-tests.sh # Emulator tests (same as CI job 2)
|
||||
python backend/validate.py # Validation (same as CI job 3)
|
||||
|
||||
# CI automatically runs all three on every push
|
||||
```
|
||||
|
||||
## Viewing Results
|
||||
|
||||
1. Go to the [Actions tab](../../actions) in GitHub
|
||||
2. Click on the latest workflow run
|
||||
3. View job logs and test results
|
||||
4. Download test artifacts (coverage reports) if needed
|
||||
|
||||
## Adding New Tests
|
||||
|
||||
Tests are automatically discovered by pytest. To add new tests:
|
||||
|
||||
1. **Unit tests:** Add to `backend/tests/test_*.py`
|
||||
2. **Emulator tests:** Add to `backend/tests/emulator/test_*.py`
|
||||
3. **Cloud tests:** Add to `backend/tests/test_api_integration.py` (runs manually)
|
||||
|
||||
The CI will automatically pick them up!
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Emulator Tests Fail in CI
|
||||
|
||||
**Common issues:**
|
||||
- Emulator startup timeout → Increase wait time in `scripts/start-emulators.sh`
|
||||
- Docker not available → Check GitHub Actions runner compatibility
|
||||
- Port conflicts → Emulators use ports 8080 (Firestore) and 4443 (GCS)
|
||||
|
||||
### CI is Slow
|
||||
|
||||
**Optimizations:**
|
||||
- ✅ Python pip caching enabled
|
||||
- ✅ Jobs run in parallel
|
||||
- ✅ Docker image pre-pulled
|
||||
- Future: Cache gcloud SDK installation
|
||||
|
||||
### Tests Pass Locally But Fail in CI
|
||||
|
||||
**Debug steps:**
|
||||
1. Check Python version matches (3.12)
|
||||
2. Check environment variables are set correctly
|
||||
3. Review CI logs for missing dependencies
|
||||
4. Run tests in a clean venv locally
|
||||
|
||||
## Setting Up Continuous Deployment
|
||||
|
||||
The CD pipeline is configured but requires GitHub secrets to be set. See the [CD Setup Guide](../docs/03-deployment/CD-SETUP.md) for detailed instructions.
|
||||
|
||||
**Quick setup:**
|
||||
|
||||
1. Create GCP service account with deployment permissions
|
||||
2. Download service account key JSON
|
||||
3. Add GitHub secrets:
|
||||
- `GCP_SA_KEY` - Full service account key JSON content
|
||||
- `ADMIN_TOKENS` - Comma-separated admin tokens
|
||||
|
||||
Once configured, every push to `replace-modal-with-google-cloud` will automatically deploy to Cloud Run after all tests pass.
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
- [ ] Add test coverage reporting (coveralls, codecov)
|
||||
- [x] ✅ Add deployment workflow (deploy to Cloud Run after tests pass)
|
||||
- [ ] Add frontend tests (when frontend is built)
|
||||
- [ ] Add performance benchmarking
|
||||
- [ ] Add security scanning (dependabot, snyk)
|
||||
- [ ] Migrate to Workload Identity Federation (more secure than service account keys)
|
||||
|
||||
@@ -0,0 +1,539 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: "ci-${{ github.ref }}"
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ============================================
|
||||
# PACKAGE TESTS (karaoke-gen Python CLI)
|
||||
# ============================================
|
||||
|
||||
package-unit-tests:
|
||||
name: "Package - Unit Tests (70% coverage)"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python 3.13
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Install Poetry
|
||||
uses: snok/install-poetry@v1
|
||||
with:
|
||||
virtualenvs-create: true
|
||||
virtualenvs-in-project: true
|
||||
|
||||
- name: Install dependencies
|
||||
run: poetry install --with dev
|
||||
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ffmpeg
|
||||
|
||||
- name: Install spaCy language models
|
||||
run: poetry run python -m spacy download en_core_web_sm
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
poetry run pytest tests/unit/ -v --cov=karaoke_gen --cov-report=xml --cov-report=term-missing --cov-fail-under=70
|
||||
|
||||
- name: Upload coverage reports
|
||||
uses: codecov/codecov-action@v5
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
files: ./coverage.xml
|
||||
flags: unittests
|
||||
name: codecov-umbrella
|
||||
|
||||
package-integration-tests:
|
||||
name: "Package - Integration Tests"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python 3.13
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Install Poetry
|
||||
uses: snok/install-poetry@v1
|
||||
with:
|
||||
virtualenvs-create: true
|
||||
virtualenvs-in-project: true
|
||||
|
||||
- name: Install dependencies
|
||||
run: poetry install --with dev
|
||||
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ffmpeg
|
||||
|
||||
- name: Install spaCy language models
|
||||
run: poetry run python -m spacy download en_core_web_sm
|
||||
|
||||
- name: Run integration tests
|
||||
run: poetry run pytest tests/integration/ -v
|
||||
|
||||
package-build:
|
||||
name: "Package - Build & Install"
|
||||
runs-on: ubuntu-latest
|
||||
needs: [package-unit-tests, package-integration-tests]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python 3.13
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Install Poetry
|
||||
uses: snok/install-poetry@v1
|
||||
with:
|
||||
virtualenvs-create: true
|
||||
virtualenvs-in-project: true
|
||||
|
||||
- name: Build package
|
||||
run: poetry build
|
||||
|
||||
- name: Free disk space
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
haskell: true
|
||||
dotnet: false
|
||||
large-packages: false
|
||||
docker-images: false
|
||||
swap-storage: false
|
||||
|
||||
- name: Test package installation
|
||||
run: |
|
||||
pip install dist/*.whl
|
||||
python -m karaoke_gen.utils.gen_cli --help
|
||||
|
||||
- name: Upload build artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: package-dist
|
||||
path: dist/
|
||||
retention-days: 1
|
||||
|
||||
# ============================================
|
||||
# BACKEND TESTS (Cloud Run API service)
|
||||
# ============================================
|
||||
|
||||
backend-unit-tests:
|
||||
name: "Backend - Unit Tests"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: false
|
||||
|
||||
- name: Set up Python 3.13
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Install Poetry
|
||||
uses: snok/install-poetry@v1
|
||||
with:
|
||||
virtualenvs-create: true
|
||||
virtualenvs-in-project: true
|
||||
|
||||
- name: Install dependencies
|
||||
run: poetry install
|
||||
|
||||
- name: Run unit tests
|
||||
env:
|
||||
GOOGLE_CLOUD_PROJECT: test-project
|
||||
GCS_BUCKET_NAME: test-bucket
|
||||
FIRESTORE_COLLECTION: jobs
|
||||
ENVIRONMENT: test
|
||||
run: |
|
||||
poetry run pytest \
|
||||
backend/tests/test_models.py \
|
||||
backend/tests/test_job_manager.py \
|
||||
backend/tests/test_file_upload.py \
|
||||
backend/tests/test_services.py \
|
||||
-v --tb=short
|
||||
|
||||
backend-emulator-tests:
|
||||
name: "Backend - Emulator Integration Tests"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Free disk space
|
||||
run: |
|
||||
sudo rm -rf /usr/share/dotnet
|
||||
sudo rm -rf /opt/ghc
|
||||
sudo rm -rf /usr/local/share/boost
|
||||
df -h
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: false
|
||||
|
||||
- name: Set up Python 3.13
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Install Poetry
|
||||
uses: snok/install-poetry@v1
|
||||
with:
|
||||
virtualenvs-create: true
|
||||
virtualenvs-in-project: true
|
||||
|
||||
- name: Set up Java 21 (required by Firestore emulator)
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
distribution: "temurin"
|
||||
java-version: "21"
|
||||
|
||||
- name: Set up Cloud SDK
|
||||
uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
version: "latest"
|
||||
|
||||
- name: Install Firestore emulator
|
||||
run: gcloud components install beta cloud-firestore-emulator --quiet
|
||||
|
||||
- name: Install dependencies
|
||||
run: poetry install
|
||||
|
||||
- name: Start Docker for GCS emulator
|
||||
run: docker pull fsouza/fake-gcs-server:latest
|
||||
|
||||
- name: Run emulator integration tests
|
||||
run: |
|
||||
chmod +x scripts/*.sh
|
||||
./scripts/run-emulator-tests.sh
|
||||
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: backend-test-results
|
||||
path: backend/htmlcov/
|
||||
retention-days: 7
|
||||
|
||||
backend-lint:
|
||||
name: "Backend - Code Quality"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: false
|
||||
|
||||
- name: Set up Python 3.13
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Install Poetry
|
||||
uses: snok/install-poetry@v1
|
||||
with:
|
||||
virtualenvs-create: true
|
||||
virtualenvs-in-project: true
|
||||
|
||||
- name: Install dependencies
|
||||
run: poetry install
|
||||
|
||||
- name: Run validation (syntax check only)
|
||||
run: |
|
||||
poetry run python -m py_compile backend/main.py backend/config.py
|
||||
echo "✅ Basic syntax validation passed"
|
||||
|
||||
# ============================================
|
||||
# DEPLOYMENTS (main branch only)
|
||||
# ============================================
|
||||
|
||||
package-publish:
|
||||
name: "Deploy - Publish to PyPI"
|
||||
runs-on: ubuntu-latest
|
||||
needs: [package-build]
|
||||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python 3.13
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Install Poetry
|
||||
uses: snok/install-poetry@v1
|
||||
|
||||
- name: Download build artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: package-dist
|
||||
path: dist/
|
||||
|
||||
- name: Extract version
|
||||
id: extract_version
|
||||
run: |
|
||||
VERSION=$(poetry version --short)
|
||||
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
||||
echo "📋 Extracted version: $VERSION"
|
||||
|
||||
- name: Check if tag exists
|
||||
id: check_tag
|
||||
run: |
|
||||
VERSION="${{ steps.extract_version.outputs.version }}"
|
||||
if git ls-remote --tags origin | grep -q "refs/tags/v$VERSION$"; then
|
||||
echo "tag_exists=true" >> $GITHUB_OUTPUT
|
||||
echo "🏷️ Tag v$VERSION already exists, skipping release"
|
||||
else
|
||||
echo "tag_exists=false" >> $GITHUB_OUTPUT
|
||||
echo "✨ Tag v$VERSION does not exist, proceeding with release"
|
||||
fi
|
||||
|
||||
- name: Publish to PyPI
|
||||
if: steps.check_tag.outputs.tag_exists == 'false'
|
||||
run: |
|
||||
echo "📦 Publishing the following files to PyPI:"
|
||||
ls -la dist/
|
||||
echo "🚀 Publishing to PyPI..."
|
||||
poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }}
|
||||
poetry publish
|
||||
|
||||
- name: Create Git Tag
|
||||
if: steps.check_tag.outputs.tag_exists == 'false'
|
||||
run: |
|
||||
git config user.name github-actions
|
||||
git config user.email github-actions@github.com
|
||||
git tag -a "v${{ steps.extract_version.outputs.version }}" -m "Release v${{ steps.extract_version.outputs.version }}"
|
||||
git push https://${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} "v${{ steps.extract_version.outputs.version }}"
|
||||
|
||||
- name: Create GitHub Release
|
||||
if: steps.check_tag.outputs.tag_exists == 'false'
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
tag_name: "v${{ steps.extract_version.outputs.version }}"
|
||||
name: "v${{ steps.extract_version.outputs.version }}"
|
||||
body: |
|
||||
## Changes in v${{ steps.extract_version.outputs.version }}
|
||||
|
||||
This release was automatically created from the main branch.
|
||||
|
||||
### Installation
|
||||
```bash
|
||||
pip install karaoke-gen==${{ steps.extract_version.outputs.version }}
|
||||
```
|
||||
|
||||
### Assets
|
||||
- Python package published to PyPI
|
||||
- Frontend deployed to GitHub Pages at gen.nomadkaraoke.com
|
||||
|
||||
For detailed changes, see the commit history.
|
||||
draft: false
|
||||
prerelease: false
|
||||
files: |
|
||||
dist/*.whl
|
||||
dist/*.tar.gz
|
||||
|
||||
deploy-frontend:
|
||||
name: "Deploy - Frontend (GitHub Pages)"
|
||||
runs-on: ubuntu-latest
|
||||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Pages
|
||||
uses: actions/configure-pages@v4
|
||||
|
||||
- name: Prepare frontend files
|
||||
run: |
|
||||
echo "📁 Preparing frontend files for deployment..."
|
||||
mkdir -p _site
|
||||
cp -r frontend/* _site/
|
||||
echo "✅ Frontend files copied to _site/"
|
||||
ls -la _site/
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
path: "_site"
|
||||
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
|
||||
deploy-backend:
|
||||
name: "Deploy - Backend (Cloud Run)"
|
||||
runs-on: ubuntu-latest
|
||||
needs: [backend-unit-tests, backend-emulator-tests, backend-lint]
|
||||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: false
|
||||
|
||||
- name: Authenticate to Google Cloud (Workload Identity)
|
||||
id: auth
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
|
||||
|
||||
- name: Set up Cloud SDK
|
||||
uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
version: "latest"
|
||||
|
||||
- name: Check if base image needs rebuild
|
||||
id: check-base
|
||||
run: |
|
||||
echo "🔍 Checking if base image needs rebuild..."
|
||||
|
||||
CURRENT_HASH=$(sha256sum poetry.lock | cut -d' ' -f1)
|
||||
echo "Current poetry.lock hash: $CURRENT_HASH"
|
||||
|
||||
gcloud auth configure-docker us-central1-docker.pkg.dev --quiet
|
||||
|
||||
BASE_IMAGE="us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend-base:latest"
|
||||
EXISTING_HASH=""
|
||||
|
||||
if docker pull $BASE_IMAGE 2>/dev/null; then
|
||||
EXISTING_HASH=$(docker inspect $BASE_IMAGE --format='{{index .Config.Labels "poetry.lock.hash"}}' 2>/dev/null || echo "")
|
||||
echo "Existing base image hash: $EXISTING_HASH"
|
||||
else
|
||||
echo "⚠️ No existing base image found"
|
||||
fi
|
||||
|
||||
if [ "$CURRENT_HASH" != "$EXISTING_HASH" ]; then
|
||||
echo "🔄 Base image needs rebuild (poetry.lock changed)"
|
||||
echo "needs_rebuild=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "✅ Base image is up to date"
|
||||
echo "needs_rebuild=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Rebuild base image (if needed)
|
||||
if: steps.check-base.outputs.needs_rebuild == 'true'
|
||||
run: |
|
||||
echo "🏗️ Rebuilding base image with updated dependencies..."
|
||||
|
||||
BASE_BUILD_ID=$(gcloud builds submit \
|
||||
--config=cloudbuild-base.yaml \
|
||||
--project=nomadkaraoke \
|
||||
--format='value(id)' \
|
||||
--async \
|
||||
.)
|
||||
|
||||
echo "Base Build ID: $BASE_BUILD_ID"
|
||||
echo "Build URL: https://console.cloud.google.com/cloud-build/builds/$BASE_BUILD_ID?project=nomadkaraoke"
|
||||
|
||||
echo "Waiting for base image build to complete..."
|
||||
while true; do
|
||||
BUILD_STATUS=$(gcloud builds describe $BASE_BUILD_ID --project=nomadkaraoke --format='value(status)')
|
||||
echo "Base build status: $BUILD_STATUS"
|
||||
|
||||
if [ "$BUILD_STATUS" = "SUCCESS" ]; then
|
||||
echo "✅ Base image build completed successfully"
|
||||
break
|
||||
elif [ "$BUILD_STATUS" = "FAILURE" ] || [ "$BUILD_STATUS" = "CANCELLED" ] || [ "$BUILD_STATUS" = "TIMEOUT" ]; then
|
||||
echo "❌ Base image build failed with status: $BUILD_STATUS"
|
||||
echo ""
|
||||
echo "==================== BUILD LOGS ===================="
|
||||
gcloud logging read "resource.type=build AND resource.labels.build_id=$BASE_BUILD_ID" \
|
||||
--project=nomadkaraoke \
|
||||
--format="value(textPayload)" \
|
||||
--limit=200 \
|
||||
--order=asc || echo "Failed to retrieve logs from Cloud Logging"
|
||||
echo "===================================================="
|
||||
gcloud builds log $BASE_BUILD_ID --project=nomadkaraoke 2>&1 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sleep 30
|
||||
done
|
||||
|
||||
- name: Trigger Cloud Build
|
||||
run: |
|
||||
echo "🚀 Triggering Cloud Build for deployment..."
|
||||
echo "Commit: ${{ github.sha }}"
|
||||
|
||||
BUILD_ID=$(gcloud builds submit \
|
||||
--config=cloudbuild.yaml \
|
||||
--project=nomadkaraoke \
|
||||
--format='value(id)' \
|
||||
--async \
|
||||
.)
|
||||
|
||||
echo "Build ID: $BUILD_ID"
|
||||
echo "Build URL: https://console.cloud.google.com/cloud-build/builds/$BUILD_ID?project=nomadkaraoke"
|
||||
|
||||
echo "Waiting for build to complete..."
|
||||
while true; do
|
||||
BUILD_STATUS=$(gcloud builds describe $BUILD_ID --project=nomadkaraoke --format='value(status)')
|
||||
echo "Build status: $BUILD_STATUS"
|
||||
|
||||
if [ "$BUILD_STATUS" = "SUCCESS" ]; then
|
||||
echo "✅ Build completed successfully"
|
||||
break
|
||||
elif [ "$BUILD_STATUS" = "FAILURE" ] || [ "$BUILD_STATUS" = "CANCELLED" ] || [ "$BUILD_STATUS" = "TIMEOUT" ]; then
|
||||
echo "❌ Build failed with status: $BUILD_STATUS"
|
||||
echo ""
|
||||
echo "==================== BUILD LOGS ===================="
|
||||
gcloud logging read "resource.type=build AND resource.labels.build_id=$BUILD_ID" \
|
||||
--project=nomadkaraoke \
|
||||
--format="value(textPayload)" \
|
||||
--limit=200 \
|
||||
--order=asc || echo "Failed to retrieve logs from Cloud Logging"
|
||||
echo "===================================================="
|
||||
echo ""
|
||||
gcloud builds log $BUILD_ID --project=nomadkaraoke 2>&1 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sleep 30
|
||||
done
|
||||
|
||||
- name: Verify deployment
|
||||
run: |
|
||||
SERVICE_URL=$(gcloud run services describe karaoke-backend \
|
||||
--region us-central1 \
|
||||
--format 'value(status.url)')
|
||||
|
||||
echo "Service URL: $SERVICE_URL"
|
||||
sleep 10
|
||||
curl -f "$SERVICE_URL/api/health" || exit 1
|
||||
echo "✅ Deployment verified - service is healthy"
|
||||
|
||||
- name: Summary
|
||||
if: success()
|
||||
run: |
|
||||
echo "🎉 Deployment successful!"
|
||||
echo "Commit: ${{ github.sha }}"
|
||||
echo "Branch: ${{ github.ref_name }}"
|
||||
echo "Run: ${{ github.run_number }}"
|
||||
@@ -1,132 +0,0 @@
|
||||
name: Test and Publish
|
||||
|
||||
on: [push, pull_request, workflow_dispatch]
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ['3.13']
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install Poetry
|
||||
uses: snok/install-poetry@v1
|
||||
with:
|
||||
virtualenvs-create: true
|
||||
virtualenvs-in-project: true
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
poetry install --with dev
|
||||
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ffmpeg
|
||||
|
||||
- name: Install spaCy language models
|
||||
run: |
|
||||
poetry run python -m spacy download en_core_web_sm
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
poetry run pytest tests/unit/ -v --cov=karaoke_gen --cov-report=xml --cov-report=term-missing --cov-fail-under=70
|
||||
|
||||
- name: Upload coverage reports
|
||||
uses: codecov/codecov-action@v5
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
files: ./coverage.xml
|
||||
flags: unittests
|
||||
name: codecov-umbrella
|
||||
|
||||
- name: Run integration tests
|
||||
run: |
|
||||
poetry run pytest tests/integration/ -v
|
||||
|
||||
- name: Build package
|
||||
run: |
|
||||
poetry build
|
||||
|
||||
- name: Free Disk Space (GHA Ubuntu Runner)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
# this might remove tools that are actually needed,
|
||||
# if set to "true" but frees about 6 GB
|
||||
tool-cache: false
|
||||
|
||||
# all of these default to true, but feel free to set to
|
||||
# "false" if necessary for your workflow
|
||||
android: true
|
||||
haskell: true
|
||||
dotnet: false
|
||||
large-packages: false
|
||||
docker-images: false
|
||||
swap-storage: false
|
||||
|
||||
- name: Test package installation
|
||||
run: |
|
||||
pip install dist/*.whl
|
||||
python -m karaoke_gen.utils.gen_cli --help
|
||||
|
||||
- if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||
name: Publish to PyPI
|
||||
run: |
|
||||
echo "📦 Publishing the following files to PyPI:"
|
||||
ls -la dist/
|
||||
echo "🚀 Publishing to PyPI..."
|
||||
poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }}
|
||||
poetry publish
|
||||
|
||||
- if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||
name: Extract version
|
||||
id: extract_version
|
||||
run: |
|
||||
VERSION=$(poetry version --short)
|
||||
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
||||
echo "📋 Extracted version: $VERSION"
|
||||
|
||||
- if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||
name: Create Git Tag
|
||||
run: |
|
||||
git config user.name github-actions
|
||||
git config user.email github-actions@github.com
|
||||
git tag -a "v${{ steps.extract_version.outputs.version }}" -m "Release v${{ steps.extract_version.outputs.version }}"
|
||||
git push origin "v${{ steps.extract_version.outputs.version }}"
|
||||
|
||||
- if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||
name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
tag_name: "v${{ steps.extract_version.outputs.version }}"
|
||||
name: "v${{ steps.extract_version.outputs.version }}"
|
||||
body: |
|
||||
## Changes in v${{ steps.extract_version.outputs.version }}
|
||||
|
||||
This release was automatically created from the main branch.
|
||||
|
||||
### Installation
|
||||
```bash
|
||||
pip install karaoke-gen==${{ steps.extract_version.outputs.version }}
|
||||
```
|
||||
|
||||
### Assets
|
||||
- Python package published to PyPI
|
||||
- Frontend assets included in the package
|
||||
|
||||
For detailed changes, see the commit history.
|
||||
draft: false
|
||||
prerelease: false
|
||||
files: |
|
||||
dist/*.whl
|
||||
dist/*.tar.gz
|
||||
|
||||
|
||||
+50
-4
@@ -5,11 +5,57 @@
|
||||
*.mp3
|
||||
|
||||
# Andrew functional adds
|
||||
audio/
|
||||
lyrics/
|
||||
karaoke/
|
||||
/audio/
|
||||
/lyrics/
|
||||
/karaoke/
|
||||
tests/outputs
|
||||
input
|
||||
/input
|
||||
/output/
|
||||
|
||||
# Next.js / React Frontend (./frontend)
|
||||
node_modules/
|
||||
frontend/node_modules/
|
||||
frontend/.next/
|
||||
frontend/out/
|
||||
frontend/.swc/
|
||||
frontend/.turbo/
|
||||
frontend/build/
|
||||
frontend/dist/
|
||||
|
||||
# Next.js specific
|
||||
frontend/*.tsbuildinfo
|
||||
frontend/next-env.d.ts
|
||||
|
||||
# Environment files (local overrides)
|
||||
frontend/.env*.local
|
||||
|
||||
# Vercel
|
||||
frontend/.vercel/
|
||||
|
||||
# Package manager logs
|
||||
frontend/npm-debug.log*
|
||||
frontend/yarn-debug.log*
|
||||
frontend/yarn-error.log*
|
||||
frontend/pnpm-debug.log*
|
||||
frontend/.pnpm-debug.log*
|
||||
|
||||
# Testing
|
||||
frontend/coverage/
|
||||
|
||||
# TypeScript
|
||||
frontend/*.tsbuildinfo
|
||||
|
||||
# ESLint
|
||||
frontend/.eslintcache
|
||||
|
||||
# Storybook
|
||||
frontend/storybook-static/
|
||||
|
||||
# PWA files
|
||||
frontend/**/public/sw.js
|
||||
frontend/**/public/workbox-*.js
|
||||
frontend/**/public/swe-worker-*.js
|
||||
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
# [PROJECT_NAME] Constitution
|
||||
<!-- Example: Spec Constitution, TaskFlow Constitution, etc. -->
|
||||
|
||||
## Core Principles
|
||||
|
||||
### [PRINCIPLE_1_NAME]
|
||||
<!-- Example: I. Library-First -->
|
||||
[PRINCIPLE_1_DESCRIPTION]
|
||||
<!-- Example: Every feature starts as a standalone library; Libraries must be self-contained, independently testable, documented; Clear purpose required - no organizational-only libraries -->
|
||||
|
||||
### [PRINCIPLE_2_NAME]
|
||||
<!-- Example: II. CLI Interface -->
|
||||
[PRINCIPLE_2_DESCRIPTION]
|
||||
<!-- Example: Every library exposes functionality via CLI; Text in/out protocol: stdin/args → stdout, errors → stderr; Support JSON + human-readable formats -->
|
||||
|
||||
### [PRINCIPLE_3_NAME]
|
||||
<!-- Example: III. Test-First (NON-NEGOTIABLE) -->
|
||||
[PRINCIPLE_3_DESCRIPTION]
|
||||
<!-- Example: TDD mandatory: Tests written → User approved → Tests fail → Then implement; Red-Green-Refactor cycle strictly enforced -->
|
||||
|
||||
### [PRINCIPLE_4_NAME]
|
||||
<!-- Example: IV. Integration Testing -->
|
||||
[PRINCIPLE_4_DESCRIPTION]
|
||||
<!-- Example: Focus areas requiring integration tests: New library contract tests, Contract changes, Inter-service communication, Shared schemas -->
|
||||
|
||||
### [PRINCIPLE_5_NAME]
|
||||
<!-- Example: V. Observability, VI. Versioning & Breaking Changes, VII. Simplicity -->
|
||||
[PRINCIPLE_5_DESCRIPTION]
|
||||
<!-- Example: Text I/O ensures debuggability; Structured logging required; Or: MAJOR.MINOR.BUILD format; Or: Start simple, YAGNI principles -->
|
||||
|
||||
## [SECTION_2_NAME]
|
||||
<!-- Example: Additional Constraints, Security Requirements, Performance Standards, etc. -->
|
||||
|
||||
[SECTION_2_CONTENT]
|
||||
<!-- Example: Technology stack requirements, compliance standards, deployment policies, etc. -->
|
||||
|
||||
## [SECTION_3_NAME]
|
||||
<!-- Example: Development Workflow, Review Process, Quality Gates, etc. -->
|
||||
|
||||
[SECTION_3_CONTENT]
|
||||
<!-- Example: Code review requirements, testing gates, deployment approval process, etc. -->
|
||||
|
||||
## Governance
|
||||
<!-- Example: Constitution supersedes all other practices; Amendments require documentation, approval, migration plan -->
|
||||
|
||||
[GOVERNANCE_RULES]
|
||||
<!-- Example: All PRs/reviews must verify compliance; Complexity must be justified; Use [GUIDANCE_FILE] for runtime development guidance -->
|
||||
|
||||
**Version**: [CONSTITUTION_VERSION] | **Ratified**: [RATIFICATION_DATE] | **Last Amended**: [LAST_AMENDED_DATE]
|
||||
<!-- Example: Version: 2.1.1 | Ratified: 2025-06-13 | Last Amended: 2025-07-16 -->
|
||||
Executable
+166
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Consolidated prerequisite checking script
|
||||
#
|
||||
# This script provides unified prerequisite checking for Spec-Driven Development workflow.
|
||||
# It replaces the functionality previously spread across multiple scripts.
|
||||
#
|
||||
# Usage: ./check-prerequisites.sh [OPTIONS]
|
||||
#
|
||||
# OPTIONS:
|
||||
# --json Output in JSON format
|
||||
# --require-tasks Require tasks.md to exist (for implementation phase)
|
||||
# --include-tasks Include tasks.md in AVAILABLE_DOCS list
|
||||
# --paths-only Only output path variables (no validation)
|
||||
# --help, -h Show help message
|
||||
#
|
||||
# OUTPUTS:
|
||||
# JSON mode: {"FEATURE_DIR":"...", "AVAILABLE_DOCS":["..."]}
|
||||
# Text mode: FEATURE_DIR:... \n AVAILABLE_DOCS: \n ✓/✗ file.md
|
||||
# Paths only: REPO_ROOT: ... \n BRANCH: ... \n FEATURE_DIR: ... etc.
|
||||
|
||||
set -e
|
||||
|
||||
# Parse command line arguments
|
||||
JSON_MODE=false
|
||||
REQUIRE_TASKS=false
|
||||
INCLUDE_TASKS=false
|
||||
PATHS_ONLY=false
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--json)
|
||||
JSON_MODE=true
|
||||
;;
|
||||
--require-tasks)
|
||||
REQUIRE_TASKS=true
|
||||
;;
|
||||
--include-tasks)
|
||||
INCLUDE_TASKS=true
|
||||
;;
|
||||
--paths-only)
|
||||
PATHS_ONLY=true
|
||||
;;
|
||||
--help|-h)
|
||||
cat << 'EOF'
|
||||
Usage: check-prerequisites.sh [OPTIONS]
|
||||
|
||||
Consolidated prerequisite checking for Spec-Driven Development workflow.
|
||||
|
||||
OPTIONS:
|
||||
--json Output in JSON format
|
||||
--require-tasks Require tasks.md to exist (for implementation phase)
|
||||
--include-tasks Include tasks.md in AVAILABLE_DOCS list
|
||||
--paths-only Only output path variables (no prerequisite validation)
|
||||
--help, -h Show this help message
|
||||
|
||||
EXAMPLES:
|
||||
# Check task prerequisites (plan.md required)
|
||||
./check-prerequisites.sh --json
|
||||
|
||||
# Check implementation prerequisites (plan.md + tasks.md required)
|
||||
./check-prerequisites.sh --json --require-tasks --include-tasks
|
||||
|
||||
# Get feature paths only (no validation)
|
||||
./check-prerequisites.sh --paths-only
|
||||
|
||||
EOF
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: Unknown option '$arg'. Use --help for usage information." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Source common functions
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/common.sh"
|
||||
|
||||
# Get feature paths and validate branch
|
||||
eval $(get_feature_paths)
|
||||
check_feature_branch "$CURRENT_BRANCH" "$HAS_GIT" || exit 1
|
||||
|
||||
# If paths-only mode, output paths and exit (support JSON + paths-only combined)
|
||||
if $PATHS_ONLY; then
|
||||
if $JSON_MODE; then
|
||||
# Minimal JSON paths payload (no validation performed)
|
||||
printf '{"REPO_ROOT":"%s","BRANCH":"%s","FEATURE_DIR":"%s","FEATURE_SPEC":"%s","IMPL_PLAN":"%s","TASKS":"%s"}\n' \
|
||||
"$REPO_ROOT" "$CURRENT_BRANCH" "$FEATURE_DIR" "$FEATURE_SPEC" "$IMPL_PLAN" "$TASKS"
|
||||
else
|
||||
echo "REPO_ROOT: $REPO_ROOT"
|
||||
echo "BRANCH: $CURRENT_BRANCH"
|
||||
echo "FEATURE_DIR: $FEATURE_DIR"
|
||||
echo "FEATURE_SPEC: $FEATURE_SPEC"
|
||||
echo "IMPL_PLAN: $IMPL_PLAN"
|
||||
echo "TASKS: $TASKS"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Validate required directories and files
|
||||
if [[ ! -d "$FEATURE_DIR" ]]; then
|
||||
echo "ERROR: Feature directory not found: $FEATURE_DIR" >&2
|
||||
echo "Run /specify first to create the feature structure." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "$IMPL_PLAN" ]]; then
|
||||
echo "ERROR: plan.md not found in $FEATURE_DIR" >&2
|
||||
echo "Run /plan first to create the implementation plan." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check for tasks.md if required
|
||||
if $REQUIRE_TASKS && [[ ! -f "$TASKS" ]]; then
|
||||
echo "ERROR: tasks.md not found in $FEATURE_DIR" >&2
|
||||
echo "Run /tasks first to create the task list." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Build list of available documents
|
||||
docs=()
|
||||
|
||||
# Always check these optional docs
|
||||
[[ -f "$RESEARCH" ]] && docs+=("research.md")
|
||||
[[ -f "$DATA_MODEL" ]] && docs+=("data-model.md")
|
||||
|
||||
# Check contracts directory (only if it exists and has files)
|
||||
if [[ -d "$CONTRACTS_DIR" ]] && [[ -n "$(ls -A "$CONTRACTS_DIR" 2>/dev/null)" ]]; then
|
||||
docs+=("contracts/")
|
||||
fi
|
||||
|
||||
[[ -f "$QUICKSTART" ]] && docs+=("quickstart.md")
|
||||
|
||||
# Include tasks.md if requested and it exists
|
||||
if $INCLUDE_TASKS && [[ -f "$TASKS" ]]; then
|
||||
docs+=("tasks.md")
|
||||
fi
|
||||
|
||||
# Output results
|
||||
if $JSON_MODE; then
|
||||
# Build JSON array of documents
|
||||
if [[ ${#docs[@]} -eq 0 ]]; then
|
||||
json_docs="[]"
|
||||
else
|
||||
json_docs=$(printf '"%s",' "${docs[@]}")
|
||||
json_docs="[${json_docs%,}]"
|
||||
fi
|
||||
|
||||
printf '{"FEATURE_DIR":"%s","AVAILABLE_DOCS":%s}\n' "$FEATURE_DIR" "$json_docs"
|
||||
else
|
||||
# Text output
|
||||
echo "FEATURE_DIR:$FEATURE_DIR"
|
||||
echo "AVAILABLE_DOCS:"
|
||||
|
||||
# Show status of each potential document
|
||||
check_file "$RESEARCH" "research.md"
|
||||
check_file "$DATA_MODEL" "data-model.md"
|
||||
check_dir "$CONTRACTS_DIR" "contracts/"
|
||||
check_file "$QUICKSTART" "quickstart.md"
|
||||
|
||||
if $INCLUDE_TASKS; then
|
||||
check_file "$TASKS" "tasks.md"
|
||||
fi
|
||||
fi
|
||||
Executable
+113
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env bash
|
||||
# Common functions and variables for all scripts
|
||||
|
||||
# Get repository root, with fallback for non-git repositories
|
||||
get_repo_root() {
|
||||
if git rev-parse --show-toplevel >/dev/null 2>&1; then
|
||||
git rev-parse --show-toplevel
|
||||
else
|
||||
# Fall back to script location for non-git repos
|
||||
local script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
(cd "$script_dir/../../.." && pwd)
|
||||
fi
|
||||
}
|
||||
|
||||
# Get current branch, with fallback for non-git repositories
|
||||
get_current_branch() {
|
||||
# First check if SPECIFY_FEATURE environment variable is set
|
||||
if [[ -n "${SPECIFY_FEATURE:-}" ]]; then
|
||||
echo "$SPECIFY_FEATURE"
|
||||
return
|
||||
fi
|
||||
|
||||
# Then check git if available
|
||||
if git rev-parse --abbrev-ref HEAD >/dev/null 2>&1; then
|
||||
git rev-parse --abbrev-ref HEAD
|
||||
return
|
||||
fi
|
||||
|
||||
# For non-git repos, try to find the latest feature directory
|
||||
local repo_root=$(get_repo_root)
|
||||
local specs_dir="$repo_root/specs"
|
||||
|
||||
if [[ -d "$specs_dir" ]]; then
|
||||
local latest_feature=""
|
||||
local highest=0
|
||||
|
||||
for dir in "$specs_dir"/*; do
|
||||
if [[ -d "$dir" ]]; then
|
||||
local dirname=$(basename "$dir")
|
||||
if [[ "$dirname" =~ ^([0-9]{3})- ]]; then
|
||||
local number=${BASH_REMATCH[1]}
|
||||
number=$((10#$number))
|
||||
if [[ "$number" -gt "$highest" ]]; then
|
||||
highest=$number
|
||||
latest_feature=$dirname
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -n "$latest_feature" ]]; then
|
||||
echo "$latest_feature"
|
||||
return
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "main" # Final fallback
|
||||
}
|
||||
|
||||
# Check if we have git available
|
||||
has_git() {
|
||||
git rev-parse --show-toplevel >/dev/null 2>&1
|
||||
}
|
||||
|
||||
check_feature_branch() {
|
||||
local branch="$1"
|
||||
local has_git_repo="$2"
|
||||
|
||||
# For non-git repos, we can't enforce branch naming but still provide output
|
||||
if [[ "$has_git_repo" != "true" ]]; then
|
||||
echo "[specify] Warning: Git repository not detected; skipped branch validation" >&2
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ ! "$branch" =~ ^[0-9]{3}- ]]; then
|
||||
echo "ERROR: Not on a feature branch. Current branch: $branch" >&2
|
||||
echo "Feature branches should be named like: 001-feature-name" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
get_feature_dir() { echo "$1/specs/$2"; }
|
||||
|
||||
get_feature_paths() {
|
||||
local repo_root=$(get_repo_root)
|
||||
local current_branch=$(get_current_branch)
|
||||
local has_git_repo="false"
|
||||
|
||||
if has_git; then
|
||||
has_git_repo="true"
|
||||
fi
|
||||
|
||||
local feature_dir=$(get_feature_dir "$repo_root" "$current_branch")
|
||||
|
||||
cat <<EOF
|
||||
REPO_ROOT='$repo_root'
|
||||
CURRENT_BRANCH='$current_branch'
|
||||
HAS_GIT='$has_git_repo'
|
||||
FEATURE_DIR='$feature_dir'
|
||||
FEATURE_SPEC='$feature_dir/spec.md'
|
||||
IMPL_PLAN='$feature_dir/plan.md'
|
||||
TASKS='$feature_dir/tasks.md'
|
||||
RESEARCH='$feature_dir/research.md'
|
||||
DATA_MODEL='$feature_dir/data-model.md'
|
||||
QUICKSTART='$feature_dir/quickstart.md'
|
||||
CONTRACTS_DIR='$feature_dir/contracts'
|
||||
EOF
|
||||
}
|
||||
|
||||
check_file() { [[ -f "$1" ]] && echo " ✓ $2" || echo " ✗ $2"; }
|
||||
check_dir() { [[ -d "$1" && -n $(ls -A "$1" 2>/dev/null) ]] && echo " ✓ $2" || echo " ✗ $2"; }
|
||||
Executable
+97
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
JSON_MODE=false
|
||||
ARGS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--json) JSON_MODE=true ;;
|
||||
--help|-h) echo "Usage: $0 [--json] <feature_description>"; exit 0 ;;
|
||||
*) ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
FEATURE_DESCRIPTION="${ARGS[*]}"
|
||||
if [ -z "$FEATURE_DESCRIPTION" ]; then
|
||||
echo "Usage: $0 [--json] <feature_description>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Function to find the repository root by searching for existing project markers
|
||||
find_repo_root() {
|
||||
local dir="$1"
|
||||
while [ "$dir" != "/" ]; do
|
||||
if [ -d "$dir/.git" ] || [ -d "$dir/.specify" ]; then
|
||||
echo "$dir"
|
||||
return 0
|
||||
fi
|
||||
dir="$(dirname "$dir")"
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Resolve repository root. Prefer git information when available, but fall back
|
||||
# to searching for repository markers so the workflow still functions in repositories that
|
||||
# were initialised with --no-git.
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
if git rev-parse --show-toplevel >/dev/null 2>&1; then
|
||||
REPO_ROOT=$(git rev-parse --show-toplevel)
|
||||
HAS_GIT=true
|
||||
else
|
||||
REPO_ROOT="$(find_repo_root "$SCRIPT_DIR")"
|
||||
if [ -z "$REPO_ROOT" ]; then
|
||||
echo "Error: Could not determine repository root. Please run this script from within the repository." >&2
|
||||
exit 1
|
||||
fi
|
||||
HAS_GIT=false
|
||||
fi
|
||||
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
SPECS_DIR="$REPO_ROOT/specs"
|
||||
mkdir -p "$SPECS_DIR"
|
||||
|
||||
HIGHEST=0
|
||||
if [ -d "$SPECS_DIR" ]; then
|
||||
for dir in "$SPECS_DIR"/*; do
|
||||
[ -d "$dir" ] || continue
|
||||
dirname=$(basename "$dir")
|
||||
number=$(echo "$dirname" | grep -o '^[0-9]\+' || echo "0")
|
||||
number=$((10#$number))
|
||||
if [ "$number" -gt "$HIGHEST" ]; then HIGHEST=$number; fi
|
||||
done
|
||||
fi
|
||||
|
||||
NEXT=$((HIGHEST + 1))
|
||||
FEATURE_NUM=$(printf "%03d" "$NEXT")
|
||||
|
||||
BRANCH_NAME=$(echo "$FEATURE_DESCRIPTION" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/-\+/-/g' | sed 's/^-//' | sed 's/-$//')
|
||||
WORDS=$(echo "$BRANCH_NAME" | tr '-' '\n' | grep -v '^$' | head -3 | tr '\n' '-' | sed 's/-$//')
|
||||
BRANCH_NAME="${FEATURE_NUM}-${WORDS}"
|
||||
|
||||
if [ "$HAS_GIT" = true ]; then
|
||||
git checkout -b "$BRANCH_NAME"
|
||||
else
|
||||
>&2 echo "[specify] Warning: Git repository not detected; skipped branch creation for $BRANCH_NAME"
|
||||
fi
|
||||
|
||||
FEATURE_DIR="$SPECS_DIR/$BRANCH_NAME"
|
||||
mkdir -p "$FEATURE_DIR"
|
||||
|
||||
TEMPLATE="$REPO_ROOT/.specify/templates/spec-template.md"
|
||||
SPEC_FILE="$FEATURE_DIR/spec.md"
|
||||
if [ -f "$TEMPLATE" ]; then cp "$TEMPLATE" "$SPEC_FILE"; else touch "$SPEC_FILE"; fi
|
||||
|
||||
# Set the SPECIFY_FEATURE environment variable for the current session
|
||||
export SPECIFY_FEATURE="$BRANCH_NAME"
|
||||
|
||||
if $JSON_MODE; then
|
||||
printf '{"BRANCH_NAME":"%s","SPEC_FILE":"%s","FEATURE_NUM":"%s"}\n' "$BRANCH_NAME" "$SPEC_FILE" "$FEATURE_NUM"
|
||||
else
|
||||
echo "BRANCH_NAME: $BRANCH_NAME"
|
||||
echo "SPEC_FILE: $SPEC_FILE"
|
||||
echo "FEATURE_NUM: $FEATURE_NUM"
|
||||
echo "SPECIFY_FEATURE environment variable set to: $BRANCH_NAME"
|
||||
fi
|
||||
Executable
+60
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
# Parse command line arguments
|
||||
JSON_MODE=false
|
||||
ARGS=()
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--json)
|
||||
JSON_MODE=true
|
||||
;;
|
||||
--help|-h)
|
||||
echo "Usage: $0 [--json]"
|
||||
echo " --json Output results in JSON format"
|
||||
echo " --help Show this help message"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$arg")
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Get script directory and load common functions
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/common.sh"
|
||||
|
||||
# Get all paths and variables from common functions
|
||||
eval $(get_feature_paths)
|
||||
|
||||
# Check if we're on a proper feature branch (only for git repos)
|
||||
check_feature_branch "$CURRENT_BRANCH" "$HAS_GIT" || exit 1
|
||||
|
||||
# Ensure the feature directory exists
|
||||
mkdir -p "$FEATURE_DIR"
|
||||
|
||||
# Copy plan template if it exists
|
||||
TEMPLATE="$REPO_ROOT/.specify/templates/plan-template.md"
|
||||
if [[ -f "$TEMPLATE" ]]; then
|
||||
cp "$TEMPLATE" "$IMPL_PLAN"
|
||||
echo "Copied plan template to $IMPL_PLAN"
|
||||
else
|
||||
echo "Warning: Plan template not found at $TEMPLATE"
|
||||
# Create a basic plan file if template doesn't exist
|
||||
touch "$IMPL_PLAN"
|
||||
fi
|
||||
|
||||
# Output results
|
||||
if $JSON_MODE; then
|
||||
printf '{"FEATURE_SPEC":"%s","IMPL_PLAN":"%s","SPECS_DIR":"%s","BRANCH":"%s","HAS_GIT":"%s"}\n' \
|
||||
"$FEATURE_SPEC" "$IMPL_PLAN" "$FEATURE_DIR" "$CURRENT_BRANCH" "$HAS_GIT"
|
||||
else
|
||||
echo "FEATURE_SPEC: $FEATURE_SPEC"
|
||||
echo "IMPL_PLAN: $IMPL_PLAN"
|
||||
echo "SPECS_DIR: $FEATURE_DIR"
|
||||
echo "BRANCH: $CURRENT_BRANCH"
|
||||
echo "HAS_GIT: $HAS_GIT"
|
||||
fi
|
||||
+719
@@ -0,0 +1,719 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Update agent context files with information from plan.md
|
||||
#
|
||||
# This script maintains AI agent context files by parsing feature specifications
|
||||
# and updating agent-specific configuration files with project information.
|
||||
#
|
||||
# MAIN FUNCTIONS:
|
||||
# 1. Environment Validation
|
||||
# - Verifies git repository structure and branch information
|
||||
# - Checks for required plan.md files and templates
|
||||
# - Validates file permissions and accessibility
|
||||
#
|
||||
# 2. Plan Data Extraction
|
||||
# - Parses plan.md files to extract project metadata
|
||||
# - Identifies language/version, frameworks, databases, and project types
|
||||
# - Handles missing or incomplete specification data gracefully
|
||||
#
|
||||
# 3. Agent File Management
|
||||
# - Creates new agent context files from templates when needed
|
||||
# - Updates existing agent files with new project information
|
||||
# - Preserves manual additions and custom configurations
|
||||
# - Supports multiple AI agent formats and directory structures
|
||||
#
|
||||
# 4. Content Generation
|
||||
# - Generates language-specific build/test commands
|
||||
# - Creates appropriate project directory structures
|
||||
# - Updates technology stacks and recent changes sections
|
||||
# - Maintains consistent formatting and timestamps
|
||||
#
|
||||
# 5. Multi-Agent Support
|
||||
# - Handles agent-specific file paths and naming conventions
|
||||
# - Supports: Claude, Gemini, Copilot, Cursor, Qwen, opencode, Codex, Windsurf
|
||||
# - Can update single agents or all existing agent files
|
||||
# - Creates default Claude file if no agent files exist
|
||||
#
|
||||
# Usage: ./update-agent-context.sh [agent_type]
|
||||
# Agent types: claude|gemini|copilot|cursor|qwen|opencode|codex|windsurf
|
||||
# Leave empty to update all existing agent files
|
||||
|
||||
set -e
|
||||
|
||||
# Enable strict error handling
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
#==============================================================================
|
||||
# Configuration and Global Variables
|
||||
#==============================================================================
|
||||
|
||||
# Get script directory and load common functions
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/common.sh"
|
||||
|
||||
# Get all paths and variables from common functions
|
||||
eval $(get_feature_paths)
|
||||
|
||||
NEW_PLAN="$IMPL_PLAN" # Alias for compatibility with existing code
|
||||
AGENT_TYPE="${1:-}"
|
||||
|
||||
# Agent-specific file paths
|
||||
CLAUDE_FILE="$REPO_ROOT/CLAUDE.md"
|
||||
GEMINI_FILE="$REPO_ROOT/GEMINI.md"
|
||||
COPILOT_FILE="$REPO_ROOT/.github/copilot-instructions.md"
|
||||
CURSOR_FILE="$REPO_ROOT/.cursor/rules/specify-rules.mdc"
|
||||
QWEN_FILE="$REPO_ROOT/QWEN.md"
|
||||
AGENTS_FILE="$REPO_ROOT/AGENTS.md"
|
||||
WINDSURF_FILE="$REPO_ROOT/.windsurf/rules/specify-rules.md"
|
||||
KILOCODE_FILE="$REPO_ROOT/.kilocode/rules/specify-rules.md"
|
||||
AUGGIE_FILE="$REPO_ROOT/.augment/rules/specify-rules.md"
|
||||
ROO_FILE="$REPO_ROOT/.roo/rules/specify-rules.md"
|
||||
|
||||
# Template file
|
||||
TEMPLATE_FILE="$REPO_ROOT/.specify/templates/agent-file-template.md"
|
||||
|
||||
# Global variables for parsed plan data
|
||||
NEW_LANG=""
|
||||
NEW_FRAMEWORK=""
|
||||
NEW_DB=""
|
||||
NEW_PROJECT_TYPE=""
|
||||
|
||||
#==============================================================================
|
||||
# Utility Functions
|
||||
#==============================================================================
|
||||
|
||||
log_info() {
|
||||
echo "INFO: $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo "✓ $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo "ERROR: $1" >&2
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo "WARNING: $1" >&2
|
||||
}
|
||||
|
||||
# Cleanup function for temporary files
|
||||
cleanup() {
|
||||
local exit_code=$?
|
||||
rm -f /tmp/agent_update_*_$$
|
||||
rm -f /tmp/manual_additions_$$
|
||||
exit $exit_code
|
||||
}
|
||||
|
||||
# Set up cleanup trap
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
#==============================================================================
|
||||
# Validation Functions
|
||||
#==============================================================================
|
||||
|
||||
validate_environment() {
|
||||
# Check if we have a current branch/feature (git or non-git)
|
||||
if [[ -z "$CURRENT_BRANCH" ]]; then
|
||||
log_error "Unable to determine current feature"
|
||||
if [[ "$HAS_GIT" == "true" ]]; then
|
||||
log_info "Make sure you're on a feature branch"
|
||||
else
|
||||
log_info "Set SPECIFY_FEATURE environment variable or create a feature first"
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if plan.md exists
|
||||
if [[ ! -f "$NEW_PLAN" ]]; then
|
||||
log_error "No plan.md found at $NEW_PLAN"
|
||||
log_info "Make sure you're working on a feature with a corresponding spec directory"
|
||||
if [[ "$HAS_GIT" != "true" ]]; then
|
||||
log_info "Use: export SPECIFY_FEATURE=your-feature-name or create a new feature first"
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if template exists (needed for new files)
|
||||
if [[ ! -f "$TEMPLATE_FILE" ]]; then
|
||||
log_warning "Template file not found at $TEMPLATE_FILE"
|
||||
log_warning "Creating new agent files will fail"
|
||||
fi
|
||||
}
|
||||
|
||||
#==============================================================================
|
||||
# Plan Parsing Functions
|
||||
#==============================================================================
|
||||
|
||||
extract_plan_field() {
|
||||
local field_pattern="$1"
|
||||
local plan_file="$2"
|
||||
|
||||
grep "^\*\*${field_pattern}\*\*: " "$plan_file" 2>/dev/null | \
|
||||
head -1 | \
|
||||
sed "s|^\*\*${field_pattern}\*\*: ||" | \
|
||||
sed 's/^[ \t]*//;s/[ \t]*$//' | \
|
||||
grep -v "NEEDS CLARIFICATION" | \
|
||||
grep -v "^N/A$" || echo ""
|
||||
}
|
||||
|
||||
parse_plan_data() {
|
||||
local plan_file="$1"
|
||||
|
||||
if [[ ! -f "$plan_file" ]]; then
|
||||
log_error "Plan file not found: $plan_file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ ! -r "$plan_file" ]]; then
|
||||
log_error "Plan file is not readable: $plan_file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_info "Parsing plan data from $plan_file"
|
||||
|
||||
NEW_LANG=$(extract_plan_field "Language/Version" "$plan_file")
|
||||
NEW_FRAMEWORK=$(extract_plan_field "Primary Dependencies" "$plan_file")
|
||||
NEW_DB=$(extract_plan_field "Storage" "$plan_file")
|
||||
NEW_PROJECT_TYPE=$(extract_plan_field "Project Type" "$plan_file")
|
||||
|
||||
# Log what we found
|
||||
if [[ -n "$NEW_LANG" ]]; then
|
||||
log_info "Found language: $NEW_LANG"
|
||||
else
|
||||
log_warning "No language information found in plan"
|
||||
fi
|
||||
|
||||
if [[ -n "$NEW_FRAMEWORK" ]]; then
|
||||
log_info "Found framework: $NEW_FRAMEWORK"
|
||||
fi
|
||||
|
||||
if [[ -n "$NEW_DB" ]] && [[ "$NEW_DB" != "N/A" ]]; then
|
||||
log_info "Found database: $NEW_DB"
|
||||
fi
|
||||
|
||||
if [[ -n "$NEW_PROJECT_TYPE" ]]; then
|
||||
log_info "Found project type: $NEW_PROJECT_TYPE"
|
||||
fi
|
||||
}
|
||||
|
||||
format_technology_stack() {
|
||||
local lang="$1"
|
||||
local framework="$2"
|
||||
local parts=()
|
||||
|
||||
# Add non-empty parts
|
||||
[[ -n "$lang" && "$lang" != "NEEDS CLARIFICATION" ]] && parts+=("$lang")
|
||||
[[ -n "$framework" && "$framework" != "NEEDS CLARIFICATION" && "$framework" != "N/A" ]] && parts+=("$framework")
|
||||
|
||||
# Join with proper formatting
|
||||
if [[ ${#parts[@]} -eq 0 ]]; then
|
||||
echo ""
|
||||
elif [[ ${#parts[@]} -eq 1 ]]; then
|
||||
echo "${parts[0]}"
|
||||
else
|
||||
# Join multiple parts with " + "
|
||||
local result="${parts[0]}"
|
||||
for ((i=1; i<${#parts[@]}; i++)); do
|
||||
result="$result + ${parts[i]}"
|
||||
done
|
||||
echo "$result"
|
||||
fi
|
||||
}
|
||||
|
||||
#==============================================================================
|
||||
# Template and Content Generation Functions
|
||||
#==============================================================================
|
||||
|
||||
get_project_structure() {
|
||||
local project_type="$1"
|
||||
|
||||
if [[ "$project_type" == *"web"* ]]; then
|
||||
echo "backend/\\nfrontend/\\ntests/"
|
||||
else
|
||||
echo "src/\\ntests/"
|
||||
fi
|
||||
}
|
||||
|
||||
get_commands_for_language() {
|
||||
local lang="$1"
|
||||
|
||||
case "$lang" in
|
||||
*"Python"*)
|
||||
echo "cd src && pytest && ruff check ."
|
||||
;;
|
||||
*"Rust"*)
|
||||
echo "cargo test && cargo clippy"
|
||||
;;
|
||||
*"JavaScript"*|*"TypeScript"*)
|
||||
echo "npm test && npm run lint"
|
||||
;;
|
||||
*)
|
||||
echo "# Add commands for $lang"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
get_language_conventions() {
|
||||
local lang="$1"
|
||||
echo "$lang: Follow standard conventions"
|
||||
}
|
||||
|
||||
create_new_agent_file() {
|
||||
local target_file="$1"
|
||||
local temp_file="$2"
|
||||
local project_name="$3"
|
||||
local current_date="$4"
|
||||
|
||||
if [[ ! -f "$TEMPLATE_FILE" ]]; then
|
||||
log_error "Template not found at $TEMPLATE_FILE"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ ! -r "$TEMPLATE_FILE" ]]; then
|
||||
log_error "Template file is not readable: $TEMPLATE_FILE"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_info "Creating new agent context file from template..."
|
||||
|
||||
if ! cp "$TEMPLATE_FILE" "$temp_file"; then
|
||||
log_error "Failed to copy template file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Replace template placeholders
|
||||
local project_structure
|
||||
project_structure=$(get_project_structure "$NEW_PROJECT_TYPE")
|
||||
|
||||
local commands
|
||||
commands=$(get_commands_for_language "$NEW_LANG")
|
||||
|
||||
local language_conventions
|
||||
language_conventions=$(get_language_conventions "$NEW_LANG")
|
||||
|
||||
# Perform substitutions with error checking using safer approach
|
||||
# Escape special characters for sed by using a different delimiter or escaping
|
||||
local escaped_lang=$(printf '%s\n' "$NEW_LANG" | sed 's/[\[\.*^$()+{}|]/\\&/g')
|
||||
local escaped_framework=$(printf '%s\n' "$NEW_FRAMEWORK" | sed 's/[\[\.*^$()+{}|]/\\&/g')
|
||||
local escaped_branch=$(printf '%s\n' "$CURRENT_BRANCH" | sed 's/[\[\.*^$()+{}|]/\\&/g')
|
||||
|
||||
# Build technology stack and recent change strings conditionally
|
||||
local tech_stack
|
||||
if [[ -n "$escaped_lang" && -n "$escaped_framework" ]]; then
|
||||
tech_stack="- $escaped_lang + $escaped_framework ($escaped_branch)"
|
||||
elif [[ -n "$escaped_lang" ]]; then
|
||||
tech_stack="- $escaped_lang ($escaped_branch)"
|
||||
elif [[ -n "$escaped_framework" ]]; then
|
||||
tech_stack="- $escaped_framework ($escaped_branch)"
|
||||
else
|
||||
tech_stack="- ($escaped_branch)"
|
||||
fi
|
||||
|
||||
local recent_change
|
||||
if [[ -n "$escaped_lang" && -n "$escaped_framework" ]]; then
|
||||
recent_change="- $escaped_branch: Added $escaped_lang + $escaped_framework"
|
||||
elif [[ -n "$escaped_lang" ]]; then
|
||||
recent_change="- $escaped_branch: Added $escaped_lang"
|
||||
elif [[ -n "$escaped_framework" ]]; then
|
||||
recent_change="- $escaped_branch: Added $escaped_framework"
|
||||
else
|
||||
recent_change="- $escaped_branch: Added"
|
||||
fi
|
||||
|
||||
local substitutions=(
|
||||
"s|\[PROJECT NAME\]|$project_name|"
|
||||
"s|\[DATE\]|$current_date|"
|
||||
"s|\[EXTRACTED FROM ALL PLAN.MD FILES\]|$tech_stack|"
|
||||
"s|\[ACTUAL STRUCTURE FROM PLANS\]|$project_structure|g"
|
||||
"s|\[ONLY COMMANDS FOR ACTIVE TECHNOLOGIES\]|$commands|"
|
||||
"s|\[LANGUAGE-SPECIFIC, ONLY FOR LANGUAGES IN USE\]|$language_conventions|"
|
||||
"s|\[LAST 3 FEATURES AND WHAT THEY ADDED\]|$recent_change|"
|
||||
)
|
||||
|
||||
for substitution in "${substitutions[@]}"; do
|
||||
if ! sed -i.bak -e "$substitution" "$temp_file"; then
|
||||
log_error "Failed to perform substitution: $substitution"
|
||||
rm -f "$temp_file" "$temp_file.bak"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Convert \n sequences to actual newlines
|
||||
newline=$(printf '\n')
|
||||
sed -i.bak2 "s/\\\\n/${newline}/g" "$temp_file"
|
||||
|
||||
# Clean up backup files
|
||||
rm -f "$temp_file.bak" "$temp_file.bak2"
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
update_existing_agent_file() {
|
||||
local target_file="$1"
|
||||
local current_date="$2"
|
||||
|
||||
log_info "Updating existing agent context file..."
|
||||
|
||||
# Use a single temporary file for atomic update
|
||||
local temp_file
|
||||
temp_file=$(mktemp) || {
|
||||
log_error "Failed to create temporary file"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Process the file in one pass
|
||||
local tech_stack=$(format_technology_stack "$NEW_LANG" "$NEW_FRAMEWORK")
|
||||
local new_tech_entries=()
|
||||
local new_change_entry=""
|
||||
|
||||
# Prepare new technology entries
|
||||
if [[ -n "$tech_stack" ]] && ! grep -q "$tech_stack" "$target_file"; then
|
||||
new_tech_entries+=("- $tech_stack ($CURRENT_BRANCH)")
|
||||
fi
|
||||
|
||||
if [[ -n "$NEW_DB" ]] && [[ "$NEW_DB" != "N/A" ]] && [[ "$NEW_DB" != "NEEDS CLARIFICATION" ]] && ! grep -q "$NEW_DB" "$target_file"; then
|
||||
new_tech_entries+=("- $NEW_DB ($CURRENT_BRANCH)")
|
||||
fi
|
||||
|
||||
# Prepare new change entry
|
||||
if [[ -n "$tech_stack" ]]; then
|
||||
new_change_entry="- $CURRENT_BRANCH: Added $tech_stack"
|
||||
elif [[ -n "$NEW_DB" ]] && [[ "$NEW_DB" != "N/A" ]] && [[ "$NEW_DB" != "NEEDS CLARIFICATION" ]]; then
|
||||
new_change_entry="- $CURRENT_BRANCH: Added $NEW_DB"
|
||||
fi
|
||||
|
||||
# Process file line by line
|
||||
local in_tech_section=false
|
||||
local in_changes_section=false
|
||||
local tech_entries_added=false
|
||||
local changes_entries_added=false
|
||||
local existing_changes_count=0
|
||||
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
# Handle Active Technologies section
|
||||
if [[ "$line" == "## Active Technologies" ]]; then
|
||||
echo "$line" >> "$temp_file"
|
||||
in_tech_section=true
|
||||
continue
|
||||
elif [[ $in_tech_section == true ]] && [[ "$line" =~ ^##[[:space:]] ]]; then
|
||||
# Add new tech entries before closing the section
|
||||
if [[ $tech_entries_added == false ]] && [[ ${#new_tech_entries[@]} -gt 0 ]]; then
|
||||
printf '%s\n' "${new_tech_entries[@]}" >> "$temp_file"
|
||||
tech_entries_added=true
|
||||
fi
|
||||
echo "$line" >> "$temp_file"
|
||||
in_tech_section=false
|
||||
continue
|
||||
elif [[ $in_tech_section == true ]] && [[ -z "$line" ]]; then
|
||||
# Add new tech entries before empty line in tech section
|
||||
if [[ $tech_entries_added == false ]] && [[ ${#new_tech_entries[@]} -gt 0 ]]; then
|
||||
printf '%s\n' "${new_tech_entries[@]}" >> "$temp_file"
|
||||
tech_entries_added=true
|
||||
fi
|
||||
echo "$line" >> "$temp_file"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Handle Recent Changes section
|
||||
if [[ "$line" == "## Recent Changes" ]]; then
|
||||
echo "$line" >> "$temp_file"
|
||||
# Add new change entry right after the heading
|
||||
if [[ -n "$new_change_entry" ]]; then
|
||||
echo "$new_change_entry" >> "$temp_file"
|
||||
fi
|
||||
in_changes_section=true
|
||||
changes_entries_added=true
|
||||
continue
|
||||
elif [[ $in_changes_section == true ]] && [[ "$line" =~ ^##[[:space:]] ]]; then
|
||||
echo "$line" >> "$temp_file"
|
||||
in_changes_section=false
|
||||
continue
|
||||
elif [[ $in_changes_section == true ]] && [[ "$line" == "- "* ]]; then
|
||||
# Keep only first 2 existing changes
|
||||
if [[ $existing_changes_count -lt 2 ]]; then
|
||||
echo "$line" >> "$temp_file"
|
||||
((existing_changes_count++))
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
# Update timestamp
|
||||
if [[ "$line" =~ \*\*Last\ updated\*\*:.*[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] ]]; then
|
||||
echo "$line" | sed "s/[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]/$current_date/" >> "$temp_file"
|
||||
else
|
||||
echo "$line" >> "$temp_file"
|
||||
fi
|
||||
done < "$target_file"
|
||||
|
||||
# Post-loop check: if we're still in the Active Technologies section and haven't added new entries
|
||||
if [[ $in_tech_section == true ]] && [[ $tech_entries_added == false ]] && [[ ${#new_tech_entries[@]} -gt 0 ]]; then
|
||||
printf '%s\n' "${new_tech_entries[@]}" >> "$temp_file"
|
||||
fi
|
||||
|
||||
# Move temp file to target atomically
|
||||
if ! mv "$temp_file" "$target_file"; then
|
||||
log_error "Failed to update target file"
|
||||
rm -f "$temp_file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
#==============================================================================
|
||||
# Main Agent File Update Function
|
||||
#==============================================================================
|
||||
|
||||
update_agent_file() {
|
||||
local target_file="$1"
|
||||
local agent_name="$2"
|
||||
|
||||
if [[ -z "$target_file" ]] || [[ -z "$agent_name" ]]; then
|
||||
log_error "update_agent_file requires target_file and agent_name parameters"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_info "Updating $agent_name context file: $target_file"
|
||||
|
||||
local project_name
|
||||
project_name=$(basename "$REPO_ROOT")
|
||||
local current_date
|
||||
current_date=$(date +%Y-%m-%d)
|
||||
|
||||
# Create directory if it doesn't exist
|
||||
local target_dir
|
||||
target_dir=$(dirname "$target_file")
|
||||
if [[ ! -d "$target_dir" ]]; then
|
||||
if ! mkdir -p "$target_dir"; then
|
||||
log_error "Failed to create directory: $target_dir"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ! -f "$target_file" ]]; then
|
||||
# Create new file from template
|
||||
local temp_file
|
||||
temp_file=$(mktemp) || {
|
||||
log_error "Failed to create temporary file"
|
||||
return 1
|
||||
}
|
||||
|
||||
if create_new_agent_file "$target_file" "$temp_file" "$project_name" "$current_date"; then
|
||||
if mv "$temp_file" "$target_file"; then
|
||||
log_success "Created new $agent_name context file"
|
||||
else
|
||||
log_error "Failed to move temporary file to $target_file"
|
||||
rm -f "$temp_file"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
log_error "Failed to create new agent file"
|
||||
rm -f "$temp_file"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
# Update existing file
|
||||
if [[ ! -r "$target_file" ]]; then
|
||||
log_error "Cannot read existing file: $target_file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ ! -w "$target_file" ]]; then
|
||||
log_error "Cannot write to existing file: $target_file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if update_existing_agent_file "$target_file" "$current_date"; then
|
||||
log_success "Updated existing $agent_name context file"
|
||||
else
|
||||
log_error "Failed to update existing agent file"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
#==============================================================================
|
||||
# Agent Selection and Processing
|
||||
#==============================================================================
|
||||
|
||||
update_specific_agent() {
|
||||
local agent_type="$1"
|
||||
|
||||
case "$agent_type" in
|
||||
claude)
|
||||
update_agent_file "$CLAUDE_FILE" "Claude Code"
|
||||
;;
|
||||
gemini)
|
||||
update_agent_file "$GEMINI_FILE" "Gemini CLI"
|
||||
;;
|
||||
copilot)
|
||||
update_agent_file "$COPILOT_FILE" "GitHub Copilot"
|
||||
;;
|
||||
cursor)
|
||||
update_agent_file "$CURSOR_FILE" "Cursor IDE"
|
||||
;;
|
||||
qwen)
|
||||
update_agent_file "$QWEN_FILE" "Qwen Code"
|
||||
;;
|
||||
opencode)
|
||||
update_agent_file "$AGENTS_FILE" "opencode"
|
||||
;;
|
||||
codex)
|
||||
update_agent_file "$AGENTS_FILE" "Codex CLI"
|
||||
;;
|
||||
windsurf)
|
||||
update_agent_file "$WINDSURF_FILE" "Windsurf"
|
||||
;;
|
||||
kilocode)
|
||||
update_agent_file "$KILOCODE_FILE" "Kilo Code"
|
||||
;;
|
||||
auggie)
|
||||
update_agent_file "$AUGGIE_FILE" "Auggie CLI"
|
||||
;;
|
||||
roo)
|
||||
update_agent_file "$ROO_FILE" "Roo Code"
|
||||
;;
|
||||
*)
|
||||
log_error "Unknown agent type '$agent_type'"
|
||||
log_error "Expected: claude|gemini|copilot|cursor|qwen|opencode|codex|windsurf|kilocode|auggie|roo"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
update_all_existing_agents() {
|
||||
local found_agent=false
|
||||
|
||||
# Check each possible agent file and update if it exists
|
||||
if [[ -f "$CLAUDE_FILE" ]]; then
|
||||
update_agent_file "$CLAUDE_FILE" "Claude Code"
|
||||
found_agent=true
|
||||
fi
|
||||
|
||||
if [[ -f "$GEMINI_FILE" ]]; then
|
||||
update_agent_file "$GEMINI_FILE" "Gemini CLI"
|
||||
found_agent=true
|
||||
fi
|
||||
|
||||
if [[ -f "$COPILOT_FILE" ]]; then
|
||||
update_agent_file "$COPILOT_FILE" "GitHub Copilot"
|
||||
found_agent=true
|
||||
fi
|
||||
|
||||
if [[ -f "$CURSOR_FILE" ]]; then
|
||||
update_agent_file "$CURSOR_FILE" "Cursor IDE"
|
||||
found_agent=true
|
||||
fi
|
||||
|
||||
if [[ -f "$QWEN_FILE" ]]; then
|
||||
update_agent_file "$QWEN_FILE" "Qwen Code"
|
||||
found_agent=true
|
||||
fi
|
||||
|
||||
if [[ -f "$AGENTS_FILE" ]]; then
|
||||
update_agent_file "$AGENTS_FILE" "Codex/opencode"
|
||||
found_agent=true
|
||||
fi
|
||||
|
||||
if [[ -f "$WINDSURF_FILE" ]]; then
|
||||
update_agent_file "$WINDSURF_FILE" "Windsurf"
|
||||
found_agent=true
|
||||
fi
|
||||
|
||||
if [[ -f "$KILOCODE_FILE" ]]; then
|
||||
update_agent_file "$KILOCODE_FILE" "Kilo Code"
|
||||
found_agent=true
|
||||
fi
|
||||
|
||||
if [[ -f "$AUGGIE_FILE" ]]; then
|
||||
update_agent_file "$AUGGIE_FILE" "Auggie CLI"
|
||||
found_agent=true
|
||||
fi
|
||||
|
||||
if [[ -f "$ROO_FILE" ]]; then
|
||||
update_agent_file "$ROO_FILE" "Roo Code"
|
||||
found_agent=true
|
||||
fi
|
||||
|
||||
# If no agent files exist, create a default Claude file
|
||||
if [[ "$found_agent" == false ]]; then
|
||||
log_info "No existing agent files found, creating default Claude file..."
|
||||
update_agent_file "$CLAUDE_FILE" "Claude Code"
|
||||
fi
|
||||
}
|
||||
print_summary() {
|
||||
echo
|
||||
log_info "Summary of changes:"
|
||||
|
||||
if [[ -n "$NEW_LANG" ]]; then
|
||||
echo " - Added language: $NEW_LANG"
|
||||
fi
|
||||
|
||||
if [[ -n "$NEW_FRAMEWORK" ]]; then
|
||||
echo " - Added framework: $NEW_FRAMEWORK"
|
||||
fi
|
||||
|
||||
if [[ -n "$NEW_DB" ]] && [[ "$NEW_DB" != "N/A" ]]; then
|
||||
echo " - Added database: $NEW_DB"
|
||||
fi
|
||||
|
||||
echo
|
||||
log_info "Usage: $0 [claude|gemini|copilot|cursor|qwen|opencode|codex|windsurf|kilocode|auggie|roo]"
|
||||
}
|
||||
|
||||
#==============================================================================
|
||||
# Main Execution
|
||||
#==============================================================================
|
||||
|
||||
main() {
|
||||
# Validate environment before proceeding
|
||||
validate_environment
|
||||
|
||||
log_info "=== Updating agent context files for feature $CURRENT_BRANCH ==="
|
||||
|
||||
# Parse the plan file to extract project information
|
||||
if ! parse_plan_data "$NEW_PLAN"; then
|
||||
log_error "Failed to parse plan data"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Process based on agent type argument
|
||||
local success=true
|
||||
|
||||
if [[ -z "$AGENT_TYPE" ]]; then
|
||||
# No specific agent provided - update all existing agent files
|
||||
log_info "No agent specified, updating all existing agent files..."
|
||||
if ! update_all_existing_agents; then
|
||||
success=false
|
||||
fi
|
||||
else
|
||||
# Specific agent provided - update only that agent
|
||||
log_info "Updating specific agent: $AGENT_TYPE"
|
||||
if ! update_specific_agent "$AGENT_TYPE"; then
|
||||
success=false
|
||||
fi
|
||||
fi
|
||||
|
||||
# Print summary
|
||||
print_summary
|
||||
|
||||
if [[ "$success" == true ]]; then
|
||||
log_success "Agent context update completed successfully"
|
||||
exit 0
|
||||
else
|
||||
log_error "Agent context update completed with errors"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Execute main function if script is run directly
|
||||
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
||||
main "$@"
|
||||
fi
|
||||
@@ -0,0 +1,23 @@
|
||||
# [PROJECT NAME] Development Guidelines
|
||||
|
||||
Auto-generated from all feature plans. Last updated: [DATE]
|
||||
|
||||
## Active Technologies
|
||||
[EXTRACTED FROM ALL PLAN.MD FILES]
|
||||
|
||||
## Project Structure
|
||||
```
|
||||
[ACTUAL STRUCTURE FROM PLANS]
|
||||
```
|
||||
|
||||
## Commands
|
||||
[ONLY COMMANDS FOR ACTIVE TECHNOLOGIES]
|
||||
|
||||
## Code Style
|
||||
[LANGUAGE-SPECIFIC, ONLY FOR LANGUAGES IN USE]
|
||||
|
||||
## Recent Changes
|
||||
[LAST 3 FEATURES AND WHAT THEY ADDED]
|
||||
|
||||
<!-- MANUAL ADDITIONS START -->
|
||||
<!-- MANUAL ADDITIONS END -->
|
||||
@@ -0,0 +1,219 @@
|
||||
|
||||
# Implementation Plan: [FEATURE]
|
||||
|
||||
**Branch**: `[###-feature-name]` | **Date**: [DATE] | **Spec**: [link]
|
||||
**Input**: Feature specification from `/specs/[###-feature-name]/spec.md`
|
||||
|
||||
## Execution Flow (/plan command scope)
|
||||
```
|
||||
1. Load feature spec from Input path
|
||||
→ If not found: ERROR "No feature spec at {path}"
|
||||
2. Fill Technical Context (scan for NEEDS CLARIFICATION)
|
||||
→ Detect Project Type from file system structure or context (web=frontend+backend, mobile=app+api)
|
||||
→ Set Structure Decision based on project type
|
||||
3. Fill the Constitution Check section based on the content of the constitution document.
|
||||
4. Evaluate Constitution Check section below
|
||||
→ If violations exist: Document in Complexity Tracking
|
||||
→ If no justification possible: ERROR "Simplify approach first"
|
||||
→ Update Progress Tracking: Initial Constitution Check
|
||||
5. Execute Phase 0 → research.md
|
||||
→ If NEEDS CLARIFICATION remain: ERROR "Resolve unknowns"
|
||||
6. Execute Phase 1 → contracts, data-model.md, quickstart.md, agent-specific template file (e.g., `CLAUDE.md` for Claude Code, `.github/copilot-instructions.md` for GitHub Copilot, `GEMINI.md` for Gemini CLI, `QWEN.md` for Qwen Code or `AGENTS.md` for opencode).
|
||||
7. Re-evaluate Constitution Check section
|
||||
→ If new violations: Refactor design, return to Phase 1
|
||||
→ Update Progress Tracking: Post-Design Constitution Check
|
||||
8. Plan Phase 2 → Describe task generation approach (DO NOT create tasks.md)
|
||||
9. STOP - Ready for /tasks command
|
||||
```
|
||||
|
||||
**IMPORTANT**: The /plan command STOPS at step 7. Phases 2-4 are executed by other commands:
|
||||
- Phase 2: /tasks command creates tasks.md
|
||||
- Phase 3-4: Implementation execution (manual or via tools)
|
||||
|
||||
## Summary
|
||||
[Extract from feature spec: primary requirement + technical approach from research]
|
||||
|
||||
## Technical Context
|
||||
**Language/Version**: [e.g., Python 3.11, Swift 5.9, Rust 1.75 or NEEDS CLARIFICATION]
|
||||
**Primary Dependencies**: [e.g., FastAPI, UIKit, LLVM or NEEDS CLARIFICATION]
|
||||
**Storage**: [if applicable, e.g., PostgreSQL, CoreData, files or N/A]
|
||||
**Testing**: [e.g., pytest, XCTest, cargo test or NEEDS CLARIFICATION]
|
||||
**Target Platform**: [e.g., Linux server, iOS 15+, WASM or NEEDS CLARIFICATION]
|
||||
**Project Type**: [single/web/mobile - determines source structure]
|
||||
**Performance Goals**: [domain-specific, e.g., 1000 req/s, 10k lines/sec, 60 fps or NEEDS CLARIFICATION]
|
||||
**Constraints**: [domain-specific, e.g., <200ms p95, <100MB memory, offline-capable or NEEDS CLARIFICATION]
|
||||
**Scale/Scope**: [domain-specific, e.g., 10k users, 1M LOC, 50 screens or NEEDS CLARIFICATION]
|
||||
|
||||
## Constitution Check
|
||||
*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.*
|
||||
|
||||
[Gates determined based on constitution file]
|
||||
|
||||
## Project Structure
|
||||
|
||||
### Documentation (this feature)
|
||||
```
|
||||
specs/[###-feature]/
|
||||
├── plan.md # This file (/plan command output)
|
||||
├── research.md # Phase 0 output (/plan command)
|
||||
├── data-model.md # Phase 1 output (/plan command)
|
||||
├── quickstart.md # Phase 1 output (/plan command)
|
||||
├── contracts/ # Phase 1 output (/plan command)
|
||||
└── tasks.md # Phase 2 output (/tasks command - NOT created by /plan)
|
||||
```
|
||||
|
||||
### Source Code (repository root)
|
||||
<!--
|
||||
ACTION REQUIRED: Replace the placeholder tree below with the concrete layout
|
||||
for this feature. Delete unused options and expand the chosen structure with
|
||||
real paths (e.g., apps/admin, packages/something). The delivered plan must
|
||||
not include Option labels.
|
||||
-->
|
||||
```
|
||||
# [REMOVE IF UNUSED] Option 1: Single project (DEFAULT)
|
||||
src/
|
||||
├── models/
|
||||
├── services/
|
||||
├── cli/
|
||||
└── lib/
|
||||
|
||||
tests/
|
||||
├── contract/
|
||||
├── integration/
|
||||
└── unit/
|
||||
|
||||
# [REMOVE IF UNUSED] Option 2: Web application (when "frontend" + "backend" detected)
|
||||
backend/
|
||||
├── src/
|
||||
│ ├── models/
|
||||
│ ├── services/
|
||||
│ └── api/
|
||||
└── tests/
|
||||
|
||||
frontend/
|
||||
├── src/
|
||||
│ ├── components/
|
||||
│ ├── pages/
|
||||
│ └── services/
|
||||
└── tests/
|
||||
|
||||
# [REMOVE IF UNUSED] Option 3: Mobile + API (when "iOS/Android" detected)
|
||||
api/
|
||||
└── [same as backend above]
|
||||
|
||||
ios/ or android/
|
||||
└── [platform-specific structure: feature modules, UI flows, platform tests]
|
||||
```
|
||||
|
||||
**Structure Decision**: [Document the selected structure and reference the real
|
||||
directories captured above]
|
||||
|
||||
## Phase 0: Outline & Research
|
||||
1. **Extract unknowns from Technical Context** above:
|
||||
- For each NEEDS CLARIFICATION → research task
|
||||
- For each dependency → best practices task
|
||||
- For each integration → patterns task
|
||||
|
||||
2. **Generate and dispatch research agents**:
|
||||
```
|
||||
For each unknown in Technical Context:
|
||||
Task: "Research {unknown} for {feature context}"
|
||||
For each technology choice:
|
||||
Task: "Find best practices for {tech} in {domain}"
|
||||
```
|
||||
|
||||
3. **Consolidate findings** in `research.md` using format:
|
||||
- Decision: [what was chosen]
|
||||
- Rationale: [why chosen]
|
||||
- Alternatives considered: [what else evaluated]
|
||||
|
||||
**Output**: research.md with all NEEDS CLARIFICATION resolved
|
||||
|
||||
## Phase 1: Design & Contracts
|
||||
*Prerequisites: research.md complete*
|
||||
|
||||
1. **Extract entities from feature spec** → `data-model.md`:
|
||||
- Entity name, fields, relationships
|
||||
- Validation rules from requirements
|
||||
- State transitions if applicable
|
||||
|
||||
2. **Generate API contracts** from functional requirements:
|
||||
- For each user action → endpoint
|
||||
- Use standard REST/GraphQL patterns
|
||||
- Output OpenAPI/GraphQL schema to `/contracts/`
|
||||
|
||||
3. **Generate contract tests** from contracts:
|
||||
- One test file per endpoint
|
||||
- Assert request/response schemas
|
||||
- Tests must fail (no implementation yet)
|
||||
|
||||
4. **Extract test scenarios** from user stories:
|
||||
- Each story → integration test scenario
|
||||
- Quickstart test = story validation steps
|
||||
|
||||
5. **Update agent file incrementally** (O(1) operation):
|
||||
- Run `.specify/scripts/bash/update-agent-context.sh cursor`
|
||||
**IMPORTANT**: Execute it exactly as specified above. Do not add or remove any arguments.
|
||||
- If exists: Add only NEW tech from current plan
|
||||
- Preserve manual additions between markers
|
||||
- Update recent changes (keep last 3)
|
||||
- Keep under 150 lines for token efficiency
|
||||
- Output to repository root
|
||||
|
||||
**Output**: data-model.md, /contracts/*, failing tests, quickstart.md, agent-specific file
|
||||
|
||||
## Phase 2: Task Planning Approach
|
||||
*This section describes what the /tasks command will do - DO NOT execute during /plan*
|
||||
|
||||
**Task Generation Strategy**:
|
||||
- Load `.specify/templates/tasks-template.md` as base
|
||||
- Generate tasks from Phase 1 design docs (contracts, data model, quickstart)
|
||||
- Each contract → contract test task [P]
|
||||
- Each entity → model creation task [P]
|
||||
- Each user story → integration test task
|
||||
- Implementation tasks to make tests pass
|
||||
|
||||
**Ordering Strategy**:
|
||||
- TDD order: Tests before implementation
|
||||
- Dependency order: Models before services before UI
|
||||
- Mark [P] for parallel execution (independent files)
|
||||
|
||||
**Estimated Output**: 25-30 numbered, ordered tasks in tasks.md
|
||||
|
||||
**IMPORTANT**: This phase is executed by the /tasks command, NOT by /plan
|
||||
|
||||
## Phase 3+: Future Implementation
|
||||
*These phases are beyond the scope of the /plan command*
|
||||
|
||||
**Phase 3**: Task execution (/tasks command creates tasks.md)
|
||||
**Phase 4**: Implementation (execute tasks.md following constitutional principles)
|
||||
**Phase 5**: Validation (run tests, execute quickstart.md, performance validation)
|
||||
|
||||
## Complexity Tracking
|
||||
*Fill ONLY if Constitution Check has violations that must be justified*
|
||||
|
||||
| Violation | Why Needed | Simpler Alternative Rejected Because |
|
||||
|-----------|------------|-------------------------------------|
|
||||
| [e.g., 4th project] | [current need] | [why 3 projects insufficient] |
|
||||
| [e.g., Repository pattern] | [specific problem] | [why direct DB access insufficient] |
|
||||
|
||||
|
||||
## Progress Tracking
|
||||
*This checklist is updated during execution flow*
|
||||
|
||||
**Phase Status**:
|
||||
- [ ] Phase 0: Research complete (/plan command)
|
||||
- [ ] Phase 1: Design complete (/plan command)
|
||||
- [ ] Phase 2: Task planning complete (/plan command - describe approach only)
|
||||
- [ ] Phase 3: Tasks generated (/tasks command)
|
||||
- [ ] Phase 4: Implementation complete
|
||||
- [ ] Phase 5: Validation passed
|
||||
|
||||
**Gate Status**:
|
||||
- [ ] Initial Constitution Check: PASS
|
||||
- [ ] Post-Design Constitution Check: PASS
|
||||
- [ ] All NEEDS CLARIFICATION resolved
|
||||
- [ ] Complexity deviations documented
|
||||
|
||||
---
|
||||
*Based on Constitution v2.1.1 - See `/memory/constitution.md`*
|
||||
@@ -0,0 +1,116 @@
|
||||
# Feature Specification: [FEATURE NAME]
|
||||
|
||||
**Feature Branch**: `[###-feature-name]`
|
||||
**Created**: [DATE]
|
||||
**Status**: Draft
|
||||
**Input**: User description: "$ARGUMENTS"
|
||||
|
||||
## Execution Flow (main)
|
||||
```
|
||||
1. Parse user description from Input
|
||||
→ If empty: ERROR "No feature description provided"
|
||||
2. Extract key concepts from description
|
||||
→ Identify: actors, actions, data, constraints
|
||||
3. For each unclear aspect:
|
||||
→ Mark with [NEEDS CLARIFICATION: specific question]
|
||||
4. Fill User Scenarios & Testing section
|
||||
→ If no clear user flow: ERROR "Cannot determine user scenarios"
|
||||
5. Generate Functional Requirements
|
||||
→ Each requirement must be testable
|
||||
→ Mark ambiguous requirements
|
||||
6. Identify Key Entities (if data involved)
|
||||
7. Run Review Checklist
|
||||
→ If any [NEEDS CLARIFICATION]: WARN "Spec has uncertainties"
|
||||
→ If implementation details found: ERROR "Remove tech details"
|
||||
8. Return: SUCCESS (spec ready for planning)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ⚡ Quick Guidelines
|
||||
- ✅ Focus on WHAT users need and WHY
|
||||
- ❌ Avoid HOW to implement (no tech stack, APIs, code structure)
|
||||
- 👥 Written for business stakeholders, not developers
|
||||
|
||||
### Section Requirements
|
||||
- **Mandatory sections**: Must be completed for every feature
|
||||
- **Optional sections**: Include only when relevant to the feature
|
||||
- When a section doesn't apply, remove it entirely (don't leave as "N/A")
|
||||
|
||||
### For AI Generation
|
||||
When creating this spec from a user prompt:
|
||||
1. **Mark all ambiguities**: Use [NEEDS CLARIFICATION: specific question] for any assumption you'd need to make
|
||||
2. **Don't guess**: If the prompt doesn't specify something (e.g., "login system" without auth method), mark it
|
||||
3. **Think like a tester**: Every vague requirement should fail the "testable and unambiguous" checklist item
|
||||
4. **Common underspecified areas**:
|
||||
- User types and permissions
|
||||
- Data retention/deletion policies
|
||||
- Performance targets and scale
|
||||
- Error handling behaviors
|
||||
- Integration requirements
|
||||
- Security/compliance needs
|
||||
|
||||
---
|
||||
|
||||
## User Scenarios & Testing *(mandatory)*
|
||||
|
||||
### Primary User Story
|
||||
[Describe the main user journey in plain language]
|
||||
|
||||
### Acceptance Scenarios
|
||||
1. **Given** [initial state], **When** [action], **Then** [expected outcome]
|
||||
2. **Given** [initial state], **When** [action], **Then** [expected outcome]
|
||||
|
||||
### Edge Cases
|
||||
- What happens when [boundary condition]?
|
||||
- How does system handle [error scenario]?
|
||||
|
||||
## Requirements *(mandatory)*
|
||||
|
||||
### Functional Requirements
|
||||
- **FR-001**: System MUST [specific capability, e.g., "allow users to create accounts"]
|
||||
- **FR-002**: System MUST [specific capability, e.g., "validate email addresses"]
|
||||
- **FR-003**: Users MUST be able to [key interaction, e.g., "reset their password"]
|
||||
- **FR-004**: System MUST [data requirement, e.g., "persist user preferences"]
|
||||
- **FR-005**: System MUST [behavior, e.g., "log all security events"]
|
||||
|
||||
*Example of marking unclear requirements:*
|
||||
- **FR-006**: System MUST authenticate users via [NEEDS CLARIFICATION: auth method not specified - email/password, SSO, OAuth?]
|
||||
- **FR-007**: System MUST retain user data for [NEEDS CLARIFICATION: retention period not specified]
|
||||
|
||||
### Key Entities *(include if feature involves data)*
|
||||
- **[Entity 1]**: [What it represents, key attributes without implementation]
|
||||
- **[Entity 2]**: [What it represents, relationships to other entities]
|
||||
|
||||
---
|
||||
|
||||
## Review & Acceptance Checklist
|
||||
*GATE: Automated checks run during main() execution*
|
||||
|
||||
### Content Quality
|
||||
- [ ] No implementation details (languages, frameworks, APIs)
|
||||
- [ ] Focused on user value and business needs
|
||||
- [ ] Written for non-technical stakeholders
|
||||
- [ ] All mandatory sections completed
|
||||
|
||||
### Requirement Completeness
|
||||
- [ ] No [NEEDS CLARIFICATION] markers remain
|
||||
- [ ] Requirements are testable and unambiguous
|
||||
- [ ] Success criteria are measurable
|
||||
- [ ] Scope is clearly bounded
|
||||
- [ ] Dependencies and assumptions identified
|
||||
|
||||
---
|
||||
|
||||
## Execution Status
|
||||
*Updated by main() during processing*
|
||||
|
||||
- [ ] User description parsed
|
||||
- [ ] Key concepts extracted
|
||||
- [ ] Ambiguities marked
|
||||
- [ ] User scenarios defined
|
||||
- [ ] Requirements generated
|
||||
- [ ] Entities identified
|
||||
- [ ] Review checklist passed
|
||||
|
||||
---
|
||||
@@ -0,0 +1,127 @@
|
||||
# Tasks: [FEATURE NAME]
|
||||
|
||||
**Input**: Design documents from `/specs/[###-feature-name]/`
|
||||
**Prerequisites**: plan.md (required), research.md, data-model.md, contracts/
|
||||
|
||||
## Execution Flow (main)
|
||||
```
|
||||
1. Load plan.md from feature directory
|
||||
→ If not found: ERROR "No implementation plan found"
|
||||
→ Extract: tech stack, libraries, structure
|
||||
2. Load optional design documents:
|
||||
→ data-model.md: Extract entities → model tasks
|
||||
→ contracts/: Each file → contract test task
|
||||
→ research.md: Extract decisions → setup tasks
|
||||
3. Generate tasks by category:
|
||||
→ Setup: project init, dependencies, linting
|
||||
→ Tests: contract tests, integration tests
|
||||
→ Core: models, services, CLI commands
|
||||
→ Integration: DB, middleware, logging
|
||||
→ Polish: unit tests, performance, docs
|
||||
4. Apply task rules:
|
||||
→ Different files = mark [P] for parallel
|
||||
→ Same file = sequential (no [P])
|
||||
→ Tests before implementation (TDD)
|
||||
5. Number tasks sequentially (T001, T002...)
|
||||
6. Generate dependency graph
|
||||
7. Create parallel execution examples
|
||||
8. Validate task completeness:
|
||||
→ All contracts have tests?
|
||||
→ All entities have models?
|
||||
→ All endpoints implemented?
|
||||
9. Return: SUCCESS (tasks ready for execution)
|
||||
```
|
||||
|
||||
## Format: `[ID] [P?] Description`
|
||||
- **[P]**: Can run in parallel (different files, no dependencies)
|
||||
- Include exact file paths in descriptions
|
||||
|
||||
## Path Conventions
|
||||
- **Single project**: `src/`, `tests/` at repository root
|
||||
- **Web app**: `backend/src/`, `frontend/src/`
|
||||
- **Mobile**: `api/src/`, `ios/src/` or `android/src/`
|
||||
- Paths shown below assume single project - adjust based on plan.md structure
|
||||
|
||||
## Phase 3.1: Setup
|
||||
- [ ] T001 Create project structure per implementation plan
|
||||
- [ ] T002 Initialize [language] project with [framework] dependencies
|
||||
- [ ] T003 [P] Configure linting and formatting tools
|
||||
|
||||
## Phase 3.2: Tests First (TDD) ⚠️ MUST COMPLETE BEFORE 3.3
|
||||
**CRITICAL: These tests MUST be written and MUST FAIL before ANY implementation**
|
||||
- [ ] T004 [P] Contract test POST /api/users in tests/contract/test_users_post.py
|
||||
- [ ] T005 [P] Contract test GET /api/users/{id} in tests/contract/test_users_get.py
|
||||
- [ ] T006 [P] Integration test user registration in tests/integration/test_registration.py
|
||||
- [ ] T007 [P] Integration test auth flow in tests/integration/test_auth.py
|
||||
|
||||
## Phase 3.3: Core Implementation (ONLY after tests are failing)
|
||||
- [ ] T008 [P] User model in src/models/user.py
|
||||
- [ ] T009 [P] UserService CRUD in src/services/user_service.py
|
||||
- [ ] T010 [P] CLI --create-user in src/cli/user_commands.py
|
||||
- [ ] T011 POST /api/users endpoint
|
||||
- [ ] T012 GET /api/users/{id} endpoint
|
||||
- [ ] T013 Input validation
|
||||
- [ ] T014 Error handling and logging
|
||||
|
||||
## Phase 3.4: Integration
|
||||
- [ ] T015 Connect UserService to DB
|
||||
- [ ] T016 Auth middleware
|
||||
- [ ] T017 Request/response logging
|
||||
- [ ] T018 CORS and security headers
|
||||
|
||||
## Phase 3.5: Polish
|
||||
- [ ] T019 [P] Unit tests for validation in tests/unit/test_validation.py
|
||||
- [ ] T020 Performance tests (<200ms)
|
||||
- [ ] T021 [P] Update docs/api.md
|
||||
- [ ] T022 Remove duplication
|
||||
- [ ] T023 Run manual-testing.md
|
||||
|
||||
## Dependencies
|
||||
- Tests (T004-T007) before implementation (T008-T014)
|
||||
- T008 blocks T009, T015
|
||||
- T016 blocks T018
|
||||
- Implementation before polish (T019-T023)
|
||||
|
||||
## Parallel Example
|
||||
```
|
||||
# Launch T004-T007 together:
|
||||
Task: "Contract test POST /api/users in tests/contract/test_users_post.py"
|
||||
Task: "Contract test GET /api/users/{id} in tests/contract/test_users_get.py"
|
||||
Task: "Integration test registration in tests/integration/test_registration.py"
|
||||
Task: "Integration test auth in tests/integration/test_auth.py"
|
||||
```
|
||||
|
||||
## Notes
|
||||
- [P] tasks = different files, no dependencies
|
||||
- Verify tests fail before implementing
|
||||
- Commit after each task
|
||||
- Avoid: vague tasks, same file conflicts
|
||||
|
||||
## Task Generation Rules
|
||||
*Applied during main() execution*
|
||||
|
||||
1. **From Contracts**:
|
||||
- Each contract file → contract test task [P]
|
||||
- Each endpoint → implementation task
|
||||
|
||||
2. **From Data Model**:
|
||||
- Each entity → model creation task [P]
|
||||
- Relationships → service layer tasks
|
||||
|
||||
3. **From User Stories**:
|
||||
- Each story → integration test [P]
|
||||
- Quickstart scenarios → validation tasks
|
||||
|
||||
4. **Ordering**:
|
||||
- Setup → Tests → Models → Services → Endpoints → Polish
|
||||
- Dependencies block parallel execution
|
||||
|
||||
## Validation Checklist
|
||||
*GATE: Checked by main() before returning*
|
||||
|
||||
- [ ] All contracts have corresponding tests
|
||||
- [ ] All entities have model tasks
|
||||
- [ ] All tests come before implementation
|
||||
- [ ] Parallel tasks truly independent
|
||||
- [ ] Each task specifies exact file path
|
||||
- [ ] No task modifies same file as another [P] task
|
||||
-271
@@ -1,271 +0,0 @@
|
||||
# Karaoke Generator - Technical Architecture Overview
|
||||
|
||||
## Core Functionality
|
||||
|
||||
The Karaoke Generator is a comprehensive pipeline that transforms audio files (local files or YouTube URLs) into professional karaoke videos with synchronized lyrics. It handles the complete workflow from raw audio input to final deliverable files.
|
||||
|
||||
## Main Processing Pipeline
|
||||
|
||||
### 1. **Input Processing**
|
||||
- **Input Sources**: Local audio files (FLAC, WAV, MP3) or YouTube URLs
|
||||
- **Metadata**: Artist name and song title (can search YouTube if URL not provided)
|
||||
- **File Handling**: Converts input media to WAV format for processing
|
||||
|
||||
### 2. **Parallel Processing Phase**
|
||||
The tool runs two major operations concurrently:
|
||||
|
||||
#### **Audio Separation** (Compute-Intensive)
|
||||
- **Models Used**: Multiple AI models for different separation tasks:
|
||||
- `model_bs_roformer_ep_317_sdr_12.9755.ckpt` (clean instrumental)
|
||||
- `htdemucs_6s.yaml` (6-stem separation)
|
||||
- `mel_band_roformer_karaoke_aufr33_viperx_sdr_10.1956.ckpt` (backing vocals)
|
||||
- **Hardware Requirements**:
|
||||
- GPU acceleration preferred (uses Apple Silicon MPS, CUDA, or CPU fallback)
|
||||
- Significant processing time (12+ minutes for a ~5-minute song)
|
||||
- Large model files (requires dedicated model storage directory)
|
||||
- **Output**: Multiple stems (vocals, instrumental, backing vocals, drums, bass, etc.)
|
||||
|
||||
#### **Lyrics Processing**
|
||||
- **Lyrics Fetching**: Genius API integration for lyric retrieval
|
||||
- **Transcription**: AudioShake API for timing synchronization
|
||||
- **Correction**: AI-powered lyrics correction using reference sources
|
||||
- **Processing Time**: ~5-6 minutes for transcription + correction
|
||||
|
||||
### 3. **Human Review Phase** (Interactive)
|
||||
- **Web Interface**: React-based web UI launches in browser
|
||||
- **Functionality**: Users review and correct automatically transcribed lyrics
|
||||
- **Timing**: Allows fine-tuning of lyric timing and text accuracy
|
||||
- **Critical Dependency**: Requires human interaction - cannot be fully automated
|
||||
|
||||
### 4. **Output Generation**
|
||||
Creates multiple synchronized lyric formats:
|
||||
- **LRC files** (timed lyrics for media players)
|
||||
- **CDG files** (CD+Graphics karaoke format)
|
||||
- **ASS/SSA files** (Advanced SubStation Alpha for video)
|
||||
- **Plain text files** (reference versions)
|
||||
- **Video files** with lyric overlays
|
||||
|
||||
### 5. **Video Production**
|
||||
- **Title/End Screens**: Generated with custom branding and styling
|
||||
- **Video Composition**: Combines title + karaoke content + end screen
|
||||
- **Multiple Formats**:
|
||||
- 4K lossless (PCM audio)
|
||||
- 4K lossy (AAC audio)
|
||||
- 720p compressed
|
||||
- MKV with FLAC audio (YouTube optimized)
|
||||
|
||||
## External API Dependencies
|
||||
|
||||
### **Required APIs**
|
||||
1. **AudioShake API**: Core dependency for lyrics transcription/timing
|
||||
- Upload audio files (can be large - 280+ seconds in example)
|
||||
- Wait for processing completion
|
||||
- Retrieve timed transcription data
|
||||
|
||||
2. **Genius API**: Lyrics text retrieval
|
||||
- Search by artist/title
|
||||
- Fetch reference lyrics text
|
||||
|
||||
3. **YouTube Data API v3**: Video management
|
||||
- Upload videos
|
||||
- Set thumbnails
|
||||
- Check for existing videos
|
||||
|
||||
### **Optional Integrations**
|
||||
- **Discord Webhooks**: Completion notifications
|
||||
- **Email APIs**: Draft creation for delivery notifications
|
||||
- **Rclone**: Cloud storage synchronization (Google Drive, Dropbox)
|
||||
|
||||
## User Interaction Points
|
||||
|
||||
### **Critical Interactive Elements**
|
||||
1. **Instrumental Selection**:
|
||||
- Terminal-based choice between separation models
|
||||
- Presents numbered options (e.g., "Choose instrumental audio file: [1]/2")
|
||||
- Requires user input to proceed
|
||||
|
||||
2. **Human Review Interface**:
|
||||
- Launches React web application
|
||||
- Browser-based editing of transcribed lyrics
|
||||
- Users can modify text and timing
|
||||
- Must submit reviewed data to continue processing
|
||||
|
||||
3. **Confirmation Prompts**:
|
||||
- Feature confirmation before finalization
|
||||
- Video quality check before proceeding
|
||||
- Manual approval gates throughout process
|
||||
|
||||
## Computing Requirements
|
||||
|
||||
### **Hardware Needs**
|
||||
- **GPU/Accelerated Processing**:
|
||||
- NVIDIA CUDA (preferred)
|
||||
- Apple Silicon MPS (Metal Performance Shaders)
|
||||
- CPU fallback available but significantly slower
|
||||
- **Storage**:
|
||||
- Large model files (multiple GB)
|
||||
- Temporary processing files during separation
|
||||
- Final output files (videos can be several GB each)
|
||||
- **Memory**: Significant RAM for audio processing (8GB+ recommended)
|
||||
- **CPU**: Multi-core processing for parallel operations
|
||||
|
||||
### **Processing Time Estimates**
|
||||
Based on a ~5-minute song:
|
||||
- Audio separation: ~15 minutes (GPU) / 45+ minutes (CPU)
|
||||
- Lyrics transcription: ~5 minutes (API dependent)
|
||||
- Human review: ~5-15 minutes (user-dependent)
|
||||
- Video encoding: ~7 minutes
|
||||
- Upload/organization: ~5 minutes
|
||||
- **Total**: 32-47 minutes per track
|
||||
|
||||
### **External Dependencies**
|
||||
- **FFmpeg**: Video/audio processing and encoding
|
||||
- **Python 3.10+**: Runtime environment
|
||||
- **PyTorch**: AI model execution framework
|
||||
- **Audio Processing Libraries**: librosa, soundfile, pydub
|
||||
- **Web Framework**: For human review interface
|
||||
- **Rclone**: Cloud storage synchronization (optional)
|
||||
|
||||
## File Input/Output Structure
|
||||
|
||||
### **Input Files**
|
||||
- **Audio Sources**: Any FFmpeg-supported format (FLAC, WAV, MP3, etc.)
|
||||
- **Configuration**:
|
||||
- Style parameters JSON (branding, colors, fonts)
|
||||
- YouTube credentials and templates
|
||||
- Email templates
|
||||
- **Assets**:
|
||||
- Background images for title/end screens
|
||||
- Custom fonts
|
||||
- Branding elements
|
||||
|
||||
### **Working Directory Structure**
|
||||
```
|
||||
Artist - Song Title/
|
||||
├── Artist - Song Title (Original).wav # Converted input
|
||||
├── Artist - Song Title (Title).mov # Generated title screen
|
||||
├── Artist - Song Title (End).mov # Generated end screen
|
||||
├── Artist - Song Title (With Vocals).mkv # Karaoke video with vocals
|
||||
├── Artist - Song Title (Karaoke).lrc # Timed lyrics
|
||||
├── stems/ # Audio separation outputs
|
||||
│ ├── Artist - Song Title (Vocals ...).flac
|
||||
│ ├── Artist - Song Title (Drums ...).flac
|
||||
│ └── ...
|
||||
└── lyrics/ # Lyrics processing files
|
||||
├── Artist - Song Title (Lyrics Genius).txt
|
||||
├── Artist - Song Title (Lyrics Corrected).txt
|
||||
├── Artist - Song Title (Karaoke).cdg
|
||||
└── ...
|
||||
```
|
||||
|
||||
### **Final Output Files**
|
||||
- **Video Formats**:
|
||||
- 4K lossless MP4 (PCM audio)
|
||||
- 4K lossy MP4 (AAC audio)
|
||||
- 720p MP4 (compressed for sharing)
|
||||
- MKV with FLAC audio (YouTube optimized)
|
||||
- **Karaoke Packages**:
|
||||
- CDG+MP3 ZIP (CD+Graphics format)
|
||||
- TXT+MP3 ZIP (text-based karaoke)
|
||||
- **Individual Formats**: LRC, CDG, ASS subtitle files
|
||||
|
||||
### **Organization System**
|
||||
- **Brand-coded directories**: `BRAND-####` sequence numbering
|
||||
- **Automated file organization** to configured paths
|
||||
- **Public sharing directory** with cloud synchronization
|
||||
- **Multi-location backup** integration
|
||||
|
||||
## Current Architecture Limitations for Cloud Deployment
|
||||
|
||||
The tool currently assumes:
|
||||
|
||||
### **Local System Dependencies**
|
||||
- Local file system access for model storage and temporary files
|
||||
- Terminal-based user interaction for choices and confirmations
|
||||
- Browser access on same machine for web UI components
|
||||
- Local application launching (e.g., Audacity integration)
|
||||
- Direct file system paths for organization and sharing
|
||||
|
||||
### **Interactive Elements**
|
||||
- Terminal prompts requiring keyboard input
|
||||
- Web browser launching on local machine
|
||||
- File system browsing for path selection
|
||||
- Manual approval gates that pause execution
|
||||
|
||||
### **Resource Management**
|
||||
- Static model file paths
|
||||
- Local GPU/hardware detection and usage
|
||||
- Temporary file cleanup assumes local storage
|
||||
- Process management tied to single-machine execution
|
||||
|
||||
## Cloud Deployment Challenges
|
||||
|
||||
Key architectural changes needed for cloud deployment:
|
||||
|
||||
1. **User Interface Migration**
|
||||
- Replace terminal interactions with web-based interfaces
|
||||
- Implement session management for multi-step workflows
|
||||
- Handle asynchronous processing with status updates
|
||||
|
||||
2. **Resource Management**
|
||||
- Dynamic GPU allocation and queuing
|
||||
- Distributed model file storage and caching
|
||||
- Scalable temporary storage management
|
||||
- Container orchestration for processing jobs
|
||||
|
||||
3. **Interactive Workflow Adaptation**
|
||||
- Web-based instrumental selection interface
|
||||
- Embedded lyrics review interface
|
||||
- Progress tracking and resumable workflows
|
||||
- Real-time collaboration features
|
||||
|
||||
4. **File Management**
|
||||
- Cloud storage integration for inputs/outputs
|
||||
- Streaming upload/download for large files
|
||||
- Multi-tenant file isolation
|
||||
- Automated cleanup and retention policies
|
||||
|
||||
5. **API Integration Scaling**
|
||||
- Rate limiting and quota management
|
||||
- API key rotation and security
|
||||
- Fallback providers for critical services
|
||||
- Cost optimization for external API usage
|
||||
|
||||
This architecture provides a foundation for understanding the current system's capabilities and the considerations needed for cloud deployment transformation.
|
||||
|
||||
|
||||
## Approach 1: The Managed Platform (Easier Start)
|
||||
|
||||
This approach uses "Platform as a Service" (PaaS) providers that specialize in running containerized applications and backend tasks, including those requiring GPUs. This abstracts away much of the server management.
|
||||
|
||||
A great fit for this would be Modal. It's designed almost exactly for your use case: turning Python scripts into serverless functions that can be called via an API and can have GPUs attached.
|
||||
|
||||
How it would work with Modal:
|
||||
|
||||
Refactor karaoke-gen: You'd wrap your core processing logic in functions within a Modal script. You would use decorators to specify requirements like the GPU type (@modal.function(gpu="any")), secrets (API keys), and shared storage for your models.
|
||||
|
||||
Web Frontend: You could build a simple web app using Flask or FastAPI that also runs on Modal. This app would serve your HTML/React interface.
|
||||
|
||||
Job Submission: When you hit "Generate" in the web app, it calls your Modal GPU function asynchronously (my_gpu_function.spawn(...)). This immediately returns a job ID.
|
||||
|
||||
State Management: The worker function updates its status in a shared dictionary or a simple database. The web app can have an endpoint that you poll from the frontend to check on the job's progress.
|
||||
|
||||
Lyric Review: When the job reaches the AWAITING_REVIEW state, it saves the transcribed lyrics to storage. The web app loads this data into your React editor. When you submit your corrections, it saves the updated data and triggers the next part of the Modal function to resume the job.
|
||||
|
||||
Technology Stack:
|
||||
|
||||
Backend/Worker: Python on Modal
|
||||
|
||||
Web Framework: Flask or FastAPI (also on Modal)
|
||||
|
||||
Frontend: Your existing React app, served by the web framework.
|
||||
|
||||
Storage: Modal's Network File System for models and temporary files.
|
||||
|
||||
Pros & Cons:
|
||||
|
||||
✅ Fastest to implement: Drastically reduces infrastructure setup.
|
||||
|
||||
✅ Cost-effective for sporadic use: You only pay for the GPU while it's actively processing a job.
|
||||
|
||||
❌ Less control: You're dependent on the Modal platform's features and pricing.
|
||||
@@ -1,326 +0,0 @@
|
||||
This is a detailed, step-by-step plan for an IDE-based AI agent to refactor your `karaoke-gen` CLI tool into a multi-step, GPU-powered web application using Modal.
|
||||
|
||||
This plan prioritizes minimal refactoring of your core generation logic and focuses on wrapping it in the necessary Modal structure to achieve your goal.
|
||||
|
||||
-----
|
||||
|
||||
### **Objective:**
|
||||
|
||||
Transform the `karaoke-gen` Python CLI tool into a web-based solution hosted on Modal. The solution will feature a simple web interface for job submission and lyric review, with all heavy processing (audio separation, video rendering) handled by GPU-powered, serverless functions.
|
||||
|
||||
### **Core Principles:**
|
||||
|
||||
1. **Isolate Core Logic:** Your existing processing functions are valuable. We will treat them as a library, changing as little as possible internally.
|
||||
2. **Embrace Asynchronicity:** Web requests must be fast. All long-running tasks will be "spawned" as background jobs on Modal, allowing the web UI to remain responsive.
|
||||
3. **Use Persistent Storage:** AI Models and job artifacts (audio, lyrics, videos) will be stored in a `modal.Volume` to persist them between function runs and share them across different steps.
|
||||
4. **Manage State:** The multi-step process (e.g., waiting for human lyric review) will be managed using a `modal.Dict` as a simple, serverless database to track the state of each job.
|
||||
|
||||
-----
|
||||
|
||||
### \#\# Phase 1: Setup and Initial Refactoring
|
||||
|
||||
**Goal:** Prepare the codebase and set up the basic Modal application structure.
|
||||
|
||||
**Instructions for the Agent:**
|
||||
|
||||
1. **Install Modal:** Ensure the `modal` client is installed in the development environment. If not, run:
|
||||
|
||||
```bash
|
||||
pip install modal
|
||||
```
|
||||
|
||||
2. **Set Up Modal Token:** Authenticate with the Modal service by running the following command in the terminal. This will require a one-time browser login.
|
||||
|
||||
```bash
|
||||
modal setup
|
||||
```
|
||||
|
||||
3. **Create New Project Files:**
|
||||
|
||||
* Create a new file named `app.py`. This will be the main entry point for our Modal application.
|
||||
* Create a file named `core.py`.
|
||||
|
||||
4. **Isolate Core Logic:**
|
||||
|
||||
* Move the primary processing functions from your `karaoke-gen` script into `core.py`. This includes functions for audio separation, lyrics fetching, transcription, and video generation.
|
||||
* Modify these functions to **remove all `print()` statements and `input()` prompts**. They should now accept all necessary information as function arguments and return results (like file paths or data structures) instead of printing to the console.
|
||||
|
||||
5. **Create a Persistent Volume for Models:** In the project's root directory, run the following one-time command in your terminal. This creates a remote, persistent filesystem on Modal to store your large AI models so they aren't downloaded every single time.
|
||||
|
||||
```bash
|
||||
modal volume create karaoke-models
|
||||
```
|
||||
|
||||
* Next, upload your models to this volume. Create a temporary script called `upload_models.py`:
|
||||
```python
|
||||
import modal
|
||||
|
||||
volume = modal.Volume.from_name("karaoke-models")
|
||||
local_model_dir = "./path/to/your/models" # Change this path
|
||||
|
||||
def upload():
|
||||
import os
|
||||
model_dest_dir = "/models"
|
||||
os.makedirs(model_dest_dir, exist_ok=True)
|
||||
for model_file in os.listdir(local_model_dir):
|
||||
source_path = os.path.join(local_model_dir, model_file)
|
||||
dest_path = os.path.join(model_dest_dir, model_file)
|
||||
with open(source_path, "rb") as f:
|
||||
volume.write_file(dest_path, f.read())
|
||||
print(f"Uploaded {model_file}")
|
||||
volume.commit()
|
||||
|
||||
if __name__ == "__main__":
|
||||
upload()
|
||||
```
|
||||
* Run this script with `python upload_models.py`. After it completes, you can delete the script.
|
||||
|
||||
-----
|
||||
|
||||
### \#\# Phase 2: Create the Modal App and Worker Functions
|
||||
|
||||
**Goal:** Define the Modal application, its environment, and the GPU-powered functions that will execute the karaoke generation steps.
|
||||
|
||||
**Instructions for the Agent:**
|
||||
|
||||
1. **Define the Modal App in `app.py`:**
|
||||
|
||||
* At the top of `app.py`, define the basic app structure, including the custom Docker image, secrets, and volumes.
|
||||
|
||||
<!-- end list -->
|
||||
|
||||
```python
|
||||
import modal
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
# Define the environment for our functions
|
||||
karaoke_image = (
|
||||
modal.Image.debian_slim(python_version="3.10")
|
||||
.pip_install(
|
||||
"demucs", "youtube-dl", "pydub", "ffmpeg-python", "torch", "torchaudio",
|
||||
"requests", "genius", "faster-whisper", "numpy" # Add all your project's dependencies
|
||||
)
|
||||
.apt_install("ffmpeg")
|
||||
)
|
||||
|
||||
# Define the Modal app
|
||||
app = modal.App("karaoke-generator-webapp")
|
||||
|
||||
# Define persistent storage volumes
|
||||
model_volume = modal.Volume.from_name("karaoke-models", create_if_missing=True)
|
||||
output_volume = modal.Volume.from_name("karaoke-output", create_if_missing=True)
|
||||
|
||||
# Define a serverless dictionary to hold job states
|
||||
job_status_dict = modal.Dict.from_name("karaoke-job-statuses", create_if_missing=True)
|
||||
|
||||
# Mount volumes to a specific path inside the container
|
||||
VOLUME_CONFIG = {
|
||||
"/models": model_volume,
|
||||
"/output": output_volume
|
||||
}
|
||||
```
|
||||
|
||||
2. **Create the GPU Worker Function:**
|
||||
|
||||
* In `app.py`, create a main processing function decorated with `@app.function`. This function will perform the first heavy step: audio separation and transcription.
|
||||
* This function must be *asynchronous* and should accept a unique `job_id` and the song details.
|
||||
|
||||
<!-- end list -->
|
||||
|
||||
```python
|
||||
@app.function(
|
||||
image=karaoke_image,
|
||||
gpu="any", # Request any available GPU
|
||||
volumes=VOLUME_CONFIG,
|
||||
secrets=[modal.Secret.from_name("my-api-keys")], # Create this in the Modal UI for Genius/AudioShake/etc.
|
||||
timeout=1800 # 30 minutes
|
||||
)
|
||||
def process_part_one(job_id: str, youtube_url: str):
|
||||
from core import download_and_prep_audio, run_audio_separation, transcribe_lyrics
|
||||
|
||||
job_status_dict[job_id] = {"status": "processing_audio", "progress": 10}
|
||||
|
||||
# Paths inside the container
|
||||
output_dir = Path(f"/output/{job_id}")
|
||||
output_dir.mkdir(exist_ok=True)
|
||||
|
||||
# 1. Download and Separate Audio
|
||||
original_audio_path = download_and_prep_audio(youtube_url, output_dir)
|
||||
instrumental_path, vocals_path = run_audio_separation(original_audio_path, "/models")
|
||||
|
||||
job_status_dict[job_id] = {"status": "transcribing", "progress": 50}
|
||||
|
||||
# 2. Transcribe
|
||||
transcription_data = transcribe_lyrics(vocals_path) # Assumes a function in core.py
|
||||
|
||||
# 3. Save artifacts and update state
|
||||
output_volume.write_file(f"{job_id}/transcription_raw.json", transcription_data)
|
||||
output_volume.commit()
|
||||
|
||||
job_status_dict[job_id] = {"status": "awaiting_review", "progress": 75}
|
||||
return "Awaiting lyric review."
|
||||
```
|
||||
|
||||
3. **Create the Finalization Function:**
|
||||
|
||||
* Create a second function for the steps that happen *after* human review.
|
||||
|
||||
<!-- end list -->
|
||||
|
||||
```python
|
||||
@app.function(
|
||||
image=karaoke_image,
|
||||
gpu="any", # Video rendering can also benefit from a GPU
|
||||
volumes=VOLUME_CONFIG,
|
||||
timeout=1800
|
||||
)
|
||||
def process_part_two(job_id: str):
|
||||
from core import generate_video_assets
|
||||
|
||||
job_status_dict[job_id] = {"status": "rendering", "progress": 80}
|
||||
|
||||
# Load corrected lyrics and instrumental from the volume
|
||||
corrected_lyrics = output_volume.read_file(f"{job_id}/lyrics_corrected.json")
|
||||
instrumental_path = f"/output/{job_id}/instrumental.wav"
|
||||
|
||||
# Generate final video
|
||||
video_path = generate_video_assets(corrected_lyrics, instrumental_path)
|
||||
|
||||
job_status_dict[job_id] = {"status": "complete", "progress": 100, "video_url": f"/output/{job_id}/final.mp4"}
|
||||
output_volume.commit()
|
||||
return "Job complete."
|
||||
```
|
||||
|
||||
-----
|
||||
|
||||
### \#\# Phase 3: Build the Web Interface
|
||||
|
||||
**Goal:** Create API endpoints using Modal to submit jobs, check status, and handle the lyric review process.
|
||||
|
||||
**Instructions for the Agent:**
|
||||
|
||||
1. **Create a FastAPI Web Endpoint:**
|
||||
|
||||
* In `app.py`, we will create a web frontend using FastAPI, which is seamlessly integrated into Modal.
|
||||
|
||||
<!-- end list -->
|
||||
|
||||
```python
|
||||
from fastapi import FastAPI, Request, Form
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
|
||||
web_app = FastAPI()
|
||||
|
||||
# Basic HTML for the UI
|
||||
HTML_TEMPLATE = """
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head><title>Karaoke Gen</title></head>
|
||||
<body>
|
||||
<h1>Submit a new Karaoke Job</h1>
|
||||
<form action="/submit" method="post">
|
||||
<input type="text" name="url" placeholder="YouTube URL" size="50">
|
||||
<button type="submit">Generate</button>
|
||||
</form>
|
||||
<h2>Job Statuses</h2>
|
||||
<div id="jobs">Loading...</div>
|
||||
<script>
|
||||
async function updateJobs() {
|
||||
const response = await fetch('/jobs');
|
||||
const jobs = await response.json();
|
||||
const jobsDiv = document.getElementById('jobs');
|
||||
jobsDiv.innerHTML = '';
|
||||
for (const [job_id, data] of Object.entries(jobs)) {
|
||||
jobsDiv.innerHTML += `<p><b>${job_id}</b>: ${data.status} (${data.progress}%)</p>`;
|
||||
if (data.status === 'awaiting_review') {
|
||||
jobsDiv.innerHTML += `<a href="/review/${job_id}" target="_blank">Review Lyrics</a>`;
|
||||
}
|
||||
}
|
||||
}
|
||||
setInterval(updateJobs, 5000);
|
||||
updateJobs();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
def home():
|
||||
return HTML_TEMPLATE
|
||||
|
||||
@app.post("/submit")
|
||||
async def submit_job(url: str = Form(...)):
|
||||
job_id = str(uuid.uuid4())[:8]
|
||||
job_status_dict[job_id] = {"status": "queued", "progress": 0}
|
||||
|
||||
# Spawn the long-running job in the background
|
||||
process_part_one.spawn(job_id, url)
|
||||
|
||||
return JSONResponse({"status": "success", "job_id": job_id}, status_code=202)
|
||||
|
||||
@app.get("/jobs")
|
||||
async def get_all_job_statuses():
|
||||
return dict(job_status_dict.items())
|
||||
|
||||
# This is a placeholder for your React app.
|
||||
# For now, it's a simple form.
|
||||
@app.get("/review/{job_id}", response_class=HTMLResponse)
|
||||
async def review_lyrics_page(job_id: str):
|
||||
# In a real app, this would load data into your React component
|
||||
raw_lyrics = output_volume.read_file(f"{job_id}/transcription_raw.json").decode()
|
||||
return f"""
|
||||
<h1>Review Lyrics for {job_id}</h1>
|
||||
<form action="/review/{job_id}" method="post">
|
||||
<textarea name="lyrics" rows="20" cols="80">{raw_lyrics}</textarea><br>
|
||||
<button type="submit">Approve and Render Video</button>
|
||||
</form>
|
||||
"""
|
||||
|
||||
@app.post("/review/{job_id}")
|
||||
async def submit_review(job_id: str, lyrics: str = Form(...)):
|
||||
# Save the corrected lyrics
|
||||
output_volume.write_file(f"{job_id}/lyrics_corrected.json", lyrics.encode())
|
||||
output_volume.commit()
|
||||
|
||||
# Spawn the second part of the job
|
||||
process_part_two.spawn(job_id)
|
||||
|
||||
return JSONResponse({"status": "Final rendering started"}, status_code=202)
|
||||
```
|
||||
|
||||
2. **Mount the Web App:**
|
||||
|
||||
* Finally, at the bottom of `app.py`, mount the FastAPI application to the Modal app.
|
||||
|
||||
<!-- end list -->
|
||||
|
||||
```python
|
||||
@app.function()
|
||||
def web_endpoint():
|
||||
return web_app
|
||||
```
|
||||
|
||||
-----
|
||||
|
||||
### \#\# Phase 4: Deployment and Iteration
|
||||
|
||||
**Goal:** Run the application locally for testing and deploy it for production access.
|
||||
|
||||
**Instructions for the Agent:**
|
||||
|
||||
1. **Run for Development:** To test the application, run the following command in your terminal. This will create a temporary, live-reloading URL.
|
||||
|
||||
```bash
|
||||
modal serve app.py
|
||||
```
|
||||
|
||||
* You can now access the UI from the provided URL, submit jobs, and see the workflow in action. Check the logs in your terminal for debugging information from the worker functions.
|
||||
|
||||
2. **Deploy for Production:** Once you are satisfied with the functionality, deploy it to a permanent URL.
|
||||
|
||||
```bash
|
||||
modal deploy app.py
|
||||
```
|
||||
|
||||
* Modal will provide a persistent URL for your web application. You can then point your `gen.nomadkaraoke.com` subdomain to this URL using a CNAME record in your DNS settings. Modal's documentation provides instructions for setting up custom domains.
|
||||
@@ -5,115 +5,516 @@
|
||||

|
||||

|
||||
|
||||
Generate karaoke videos with instrumental audio and synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens, uploading the resulting video to YouTube.
|
||||
Generate professional karaoke videos with instrumental audio and synchronized lyrics. Available as a **local CLI** (`karaoke-gen`) or **cloud-based CLI** (`karaoke-gen-remote`) that offloads processing to Google Cloud.
|
||||
|
||||
## Overview
|
||||
## ✨ Two Ways to Generate Karaoke
|
||||
|
||||
Karaoke Generator is a comprehensive tool for creating high-quality karaoke videos. It automates the entire workflow:
|
||||
### 1. Local CLI (`karaoke-gen`)
|
||||
Run all processing locally on your machine. Requires GPU for optimal audio separation performance.
|
||||
|
||||
1. **Download** audio and lyrics for a specified song
|
||||
2. **Separate** audio stems (vocals, instrumental)
|
||||
3. **Synchronize** lyrics with the audio
|
||||
4. **Generate** title and end screens
|
||||
5. **Combine** everything into a polished final video
|
||||
6. **Organize** and **share** the output files
|
||||
```bash
|
||||
karaoke-gen "ABBA" "Waterloo"
|
||||
```
|
||||
|
||||
## Installation
|
||||
### 2. Remote CLI (`karaoke-gen-remote`)
|
||||
Offload all processing to a cloud backend. No GPU required - just authenticate and submit jobs.
|
||||
|
||||
```bash
|
||||
karaoke-gen-remote ./song.flac "ABBA" "Waterloo"
|
||||
```
|
||||
|
||||
Both CLIs produce identical outputs: 4K karaoke videos, CDG+MP3 packages, audio stems, and more.
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Features
|
||||
|
||||
### Core Pipeline
|
||||
- **Audio Separation**: AI-powered vocal/instrumental separation using MDX and Demucs models
|
||||
- **Lyrics Transcription**: Word-level timestamps via AudioShake API
|
||||
- **Lyrics Correction**: Match transcription against online lyrics (Genius, Spotify, Musixmatch)
|
||||
- **Human Review**: Interactive UI for correcting lyrics before final render
|
||||
- **Video Rendering**: High-quality 4K karaoke videos with customizable styles
|
||||
- **Multiple Outputs**: MP4 (4K lossless/lossy, 720p), MKV, CDG+MP3, TXT+MP3
|
||||
|
||||
### Distribution Features
|
||||
- **YouTube Upload**: Automatic upload to your YouTube channel
|
||||
- **Dropbox Integration**: Organize output in brand-coded folders
|
||||
- **Google Drive**: Upload to public share folders
|
||||
- **Discord Notifications**: Webhook notifications on completion
|
||||
|
||||
---
|
||||
|
||||
## 📦 Installation
|
||||
|
||||
```bash
|
||||
pip install karaoke-gen
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
This installs both `karaoke-gen` (local) and `karaoke-gen-remote` (cloud) CLIs.
|
||||
|
||||
### Requirements
|
||||
- Python 3.10+
|
||||
- FFmpeg
|
||||
- For local processing: CUDA-capable GPU or Apple Silicon CPU recommended
|
||||
|
||||
---
|
||||
|
||||
## 🖥️ Local CLI (`karaoke-gen`)
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```bash
|
||||
# Generate a karaoke video from a YouTube URL
|
||||
karaoke-gen "https://www.youtube.com/watch?v=dQw4w9WgXcQ" "Rick Astley" "Never Gonna Give You Up"
|
||||
# Generate from local audio file
|
||||
karaoke-gen ./song.mp3 "Artist Name" "Song Title"
|
||||
|
||||
# Or let it search YouTube for you
|
||||
# Search and download audio automatically
|
||||
karaoke-gen "Rick Astley" "Never Gonna Give You Up"
|
||||
|
||||
# Process from YouTube URL
|
||||
karaoke-gen "https://www.youtube.com/watch?v=dQw4w9WgXcQ" "Rick Astley" "Never Gonna Give You Up"
|
||||
```
|
||||
|
||||
## Workflow Options
|
||||
### Remote Audio Separation (Optional)
|
||||
|
||||
Karaoke Gen supports different workflow options to fit your needs:
|
||||
Offload just the GPU-intensive audio separation to Modal.com while keeping other processing local:
|
||||
|
||||
```bash
|
||||
# Run only the preparation phase (download, separate stems, create title screens)
|
||||
karaoke-gen --prep-only "Rick Astley" "Never Gonna Give You Up"
|
||||
|
||||
# Run only the finalisation phase (must be run in a directory prepared by the prep phase)
|
||||
karaoke-gen --finalise-only
|
||||
|
||||
# Skip automatic lyrics transcription/synchronization (for manual syncing)
|
||||
karaoke-gen --skip-transcription "Rick Astley" "Never Gonna Give You Up"
|
||||
|
||||
# Skip audio separation (if you already have instrumental)
|
||||
karaoke-gen --skip-separation --existing-instrumental="path/to/instrumental.mp3" "Rick Astley" "Never Gonna Give You Up"
|
||||
export AUDIO_SEPARATOR_API_URL="https://USERNAME--audio-separator-api.modal.run"
|
||||
karaoke-gen "Artist" "Title"
|
||||
```
|
||||
|
||||
## Advanced Features
|
||||
|
||||
### Audio Processing
|
||||
### Key Options
|
||||
|
||||
```bash
|
||||
# Specify custom audio separation models
|
||||
karaoke-gen --clean_instrumental_model="model_name.ckpt" "Rick Astley" "Never Gonna Give You Up"
|
||||
# Custom styling
|
||||
karaoke-gen --style_params_json="./styles.json" "Artist" "Title"
|
||||
|
||||
# Generate CDG and TXT packages
|
||||
karaoke-gen --enable_cdg --enable_txt "Artist" "Title"
|
||||
|
||||
# YouTube upload
|
||||
karaoke-gen --enable_youtube_upload --youtube_description_file="./desc.txt" "Artist" "Title"
|
||||
|
||||
# Full production run
|
||||
karaoke-gen \
|
||||
--style_params_json="./branding.json" \
|
||||
--enable_cdg \
|
||||
--enable_txt \
|
||||
--brand_prefix="BRAND" \
|
||||
--enable_youtube_upload \
|
||||
--youtube_description_file="./description.txt" \
|
||||
"Artist" "Title"
|
||||
```
|
||||
|
||||
### Lyrics Handling
|
||||
|
||||
```bash
|
||||
# Use a local lyrics file instead of fetching from online
|
||||
karaoke-gen --lyrics_file="path/to/lyrics.txt" "Rick Astley" "Never Gonna Give You Up"
|
||||
|
||||
# Adjust subtitle timing
|
||||
karaoke-gen --subtitle_offset_ms=500 "Rick Astley" "Never Gonna Give You Up"
|
||||
```
|
||||
|
||||
### Finalisation Options
|
||||
|
||||
```bash
|
||||
# Enable CDG ZIP generation
|
||||
karaoke-gen --enable_cdg --style_params_json="path/to/style.json" "Rick Astley" "Never Gonna Give You Up"
|
||||
|
||||
# Enable TXT ZIP generation
|
||||
karaoke-gen --enable_txt "Rick Astley" "Never Gonna Give You Up"
|
||||
|
||||
# Upload to YouTube
|
||||
karaoke-gen --youtube_client_secrets_file="path/to/client_secret.json" --youtube_description_file="path/to/description.txt" "Rick Astley" "Never Gonna Give You Up"
|
||||
|
||||
# Organize files with brand code
|
||||
karaoke-gen --brand_prefix="BRAND" --organised_dir="path/to/Tracks-Organized" "Rick Astley" "Never Gonna Give You Up"
|
||||
```
|
||||
|
||||
## Full Command Reference
|
||||
|
||||
For a complete list of options:
|
||||
### Full Options Reference
|
||||
|
||||
```bash
|
||||
karaoke-gen --help
|
||||
```
|
||||
|
||||
## Development
|
||||
---
|
||||
|
||||
## ☁️ Remote CLI (`karaoke-gen-remote`)
|
||||
|
||||
The remote CLI submits jobs to a Google Cloud backend that handles all processing. You don't need a GPU or any audio processing libraries installed locally.
|
||||
|
||||
### Setup
|
||||
|
||||
1. **Set the backend URL:**
|
||||
```bash
|
||||
export KARAOKE_GEN_URL="https://api.nomadkaraoke.com" # Or your own backend
|
||||
```
|
||||
|
||||
2. **Authenticate with Google Cloud:**
|
||||
```bash
|
||||
gcloud auth login
|
||||
```
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```bash
|
||||
# Submit a job
|
||||
karaoke-gen-remote ./song.flac "ABBA" "Waterloo"
|
||||
|
||||
# The CLI will:
|
||||
# 1. Upload your audio file
|
||||
# 2. Monitor processing progress
|
||||
# 3. Open lyrics review UI when ready
|
||||
# 4. Prompt for instrumental selection
|
||||
# 5. Download all outputs when complete
|
||||
```
|
||||
|
||||
### Job Management
|
||||
|
||||
```bash
|
||||
# List all jobs
|
||||
karaoke-gen-remote --list
|
||||
|
||||
# Resume monitoring an existing job
|
||||
karaoke-gen-remote --resume abc12345
|
||||
|
||||
# Cancel a running job
|
||||
karaoke-gen-remote --cancel abc12345
|
||||
|
||||
# Delete a job and its files
|
||||
karaoke-gen-remote --delete abc12345
|
||||
```
|
||||
|
||||
### Full Production Run
|
||||
|
||||
```bash
|
||||
karaoke-gen-remote \
|
||||
--style_params_json="./karaoke-styles.json" \
|
||||
--enable_cdg \
|
||||
--enable_txt \
|
||||
--brand_prefix=NOMAD \
|
||||
--enable_youtube_upload \
|
||||
--youtube_description_file="./youtube-description.txt" \
|
||||
./song.flac "Artist" "Title"
|
||||
```
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `KARAOKE_GEN_URL` | Backend service URL | Required |
|
||||
| `KARAOKE_GEN_AUTH_TOKEN` | Admin auth token (for protected endpoints) | Optional |
|
||||
| `REVIEW_UI_URL` | Lyrics review UI URL | `https://lyrics.nomadkaraoke.com` |
|
||||
| `POLL_INTERVAL` | Seconds between status polls | `5` |
|
||||
|
||||
### Authentication
|
||||
|
||||
The backend uses token-based authentication for admin operations (bulk delete, internal worker triggers). For basic job submission and monitoring, authentication is optional.
|
||||
|
||||
**For admin access:**
|
||||
```bash
|
||||
export KARAOKE_GEN_AUTH_TOKEN="your-admin-token"
|
||||
```
|
||||
|
||||
The token must match one of the tokens configured in the backend's `ADMIN_TOKENS` environment variable.
|
||||
|
||||
### Non-Interactive Mode
|
||||
|
||||
For automated/CI usage:
|
||||
|
||||
```bash
|
||||
karaoke-gen-remote -y ./song.flac "Artist" "Title"
|
||||
```
|
||||
|
||||
The `-y` flag auto-accepts default corrections and selects clean instrumental.
|
||||
|
||||
---
|
||||
|
||||
## 🎨 Style Configuration
|
||||
|
||||
Create a `styles.json` file to customize the karaoke video appearance:
|
||||
|
||||
```json
|
||||
{
|
||||
"intro": {
|
||||
"video_duration": 5,
|
||||
"background_image": "/path/to/title-background.png",
|
||||
"font": "/path/to/Font.ttf",
|
||||
"artist_color": "#ffdf6b",
|
||||
"title_color": "#ffffff"
|
||||
},
|
||||
"karaoke": {
|
||||
"background_image": "/path/to/karaoke-background.png",
|
||||
"font_path": "/path/to/Font.ttf"
|
||||
},
|
||||
"end": {
|
||||
"background_image": "/path/to/end-background.png"
|
||||
},
|
||||
"cdg": {
|
||||
"font_path": "/path/to/Font.ttf",
|
||||
"instrumental_background": "/path/to/cdg-background.png"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
When using `karaoke-gen-remote`, all referenced files are automatically uploaded with your job.
|
||||
|
||||
---
|
||||
|
||||
## 📤 Output Files
|
||||
|
||||
A completed job produces:
|
||||
|
||||
```
|
||||
BRAND-1234 - Artist - Title/
|
||||
├── Artist - Title (Final Karaoke Lossless 4k).mp4 # ProRes 4K
|
||||
├── Artist - Title (Final Karaoke Lossless 4k).mkv # FLAC audio 4K
|
||||
├── Artist - Title (Final Karaoke Lossy 4k).mp4 # H.264 4K
|
||||
├── Artist - Title (Final Karaoke Lossy 720p).mp4 # H.264 720p
|
||||
├── Artist - Title (Final Karaoke CDG).zip # CDG+MP3 package
|
||||
├── Artist - Title (Final Karaoke TXT).zip # TXT+MP3 package
|
||||
├── Artist - Title (Karaoke).cdg # Individual CDG
|
||||
├── Artist - Title (Karaoke).mp3 # Karaoke audio
|
||||
├── Artist - Title (Karaoke).lrc # LRC lyrics
|
||||
├── Artist - Title (Karaoke).ass # ASS subtitles
|
||||
├── Artist - Title (Title).mov # Title screen video
|
||||
├── Artist - Title (End).mov # End screen video
|
||||
├── Artist - Title (Instrumental...).flac # Clean instrumental
|
||||
├── Artist - Title (Instrumental +BV...).flac # With backing vocals
|
||||
└── stems/ # All audio stems
|
||||
├── ...Vocals....flac
|
||||
├── ...Bass....flac
|
||||
├── ...Drums....flac
|
||||
└── ...
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🏗️ Deploy Your Own Backend
|
||||
|
||||
The cloud backend runs on Google Cloud Platform using:
|
||||
- **Cloud Run**: Serverless API hosting
|
||||
- **Firestore**: Job state management
|
||||
- **Cloud Storage**: File uploads and outputs
|
||||
- **Modal.com**: GPU-accelerated audio separation
|
||||
- **AudioShake**: Lyrics transcription API
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Google Cloud account with billing enabled
|
||||
- [Pulumi CLI](https://www.pulumi.com/docs/install/)
|
||||
- Modal.com account (for audio separation)
|
||||
- AudioShake API key
|
||||
|
||||
### Infrastructure Setup
|
||||
|
||||
```bash
|
||||
cd infrastructure
|
||||
|
||||
# Install dependencies
|
||||
pip install -r requirements.txt
|
||||
|
||||
# Login to Pulumi
|
||||
pulumi login
|
||||
|
||||
# Create a stack
|
||||
pulumi stack init prod
|
||||
|
||||
# Configure GCP project
|
||||
pulumi config set gcp:project your-project-id
|
||||
pulumi config set gcp:region us-central1
|
||||
|
||||
# Deploy infrastructure
|
||||
pulumi up
|
||||
```
|
||||
|
||||
This creates:
|
||||
- Firestore database
|
||||
- Cloud Storage bucket
|
||||
- Artifact Registry
|
||||
- Service account with IAM roles
|
||||
- Secret Manager secrets (you add values)
|
||||
|
||||
### Add Secret Values
|
||||
|
||||
```bash
|
||||
# AudioShake API key
|
||||
echo -n "your-audioshake-key" | gcloud secrets versions add audioshake-api-key --data-file=-
|
||||
|
||||
# Genius API key
|
||||
echo -n "your-genius-key" | gcloud secrets versions add genius-api-key --data-file=-
|
||||
|
||||
# Modal API URL
|
||||
echo -n "https://your-modal-url" | gcloud secrets versions add audio-separator-api-url --data-file=-
|
||||
|
||||
# YouTube OAuth credentials (JSON)
|
||||
gcloud secrets versions add youtube-oauth-credentials --data-file=./youtube-creds.json
|
||||
|
||||
# Dropbox OAuth credentials (JSON)
|
||||
gcloud secrets versions add dropbox-oauth-credentials --data-file=./dropbox-creds.json
|
||||
|
||||
# Google Drive service account (JSON)
|
||||
gcloud secrets versions add gdrive-service-account --data-file=./gdrive-sa.json
|
||||
```
|
||||
|
||||
### Deploy Cloud Run
|
||||
|
||||
```bash
|
||||
# Build and deploy
|
||||
gcloud builds submit --config=cloudbuild.yaml
|
||||
|
||||
# Get outputs from Pulumi
|
||||
SA_EMAIL=$(pulumi stack output service_account_email)
|
||||
BUCKET_NAME=$(pulumi stack output bucket_name)
|
||||
|
||||
# Deploy Cloud Run service
|
||||
gcloud run deploy karaoke-backend \
|
||||
--image us-central1-docker.pkg.dev/YOUR-PROJECT/karaoke-repo/karaoke-backend:latest \
|
||||
--platform managed \
|
||||
--region us-central1 \
|
||||
--allow-unauthenticated \
|
||||
--service-account $SA_EMAIL \
|
||||
--memory 2Gi \
|
||||
--cpu 2 \
|
||||
--timeout 600 \
|
||||
--set-env-vars="GOOGLE_CLOUD_PROJECT=YOUR-PROJECT,GCS_BUCKET_NAME=$BUCKET_NAME"
|
||||
```
|
||||
|
||||
### Point CLI to Your Backend
|
||||
|
||||
```bash
|
||||
export KARAOKE_GEN_URL="https://your-backend.run.app"
|
||||
karaoke-gen-remote ./song.flac "Artist" "Title"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔌 Backend API Reference
|
||||
|
||||
The backend exposes a REST API for job management.
|
||||
|
||||
### Job Submission
|
||||
|
||||
**POST** `/api/jobs/upload`
|
||||
|
||||
Submit a new karaoke generation job with audio file and options.
|
||||
|
||||
```bash
|
||||
curl -X POST "https://api.example.com/api/jobs/upload" \
|
||||
-F "file=@song.flac" \
|
||||
-F "artist=ABBA" \
|
||||
-F "title=Waterloo" \
|
||||
-F "enable_cdg=true" \
|
||||
-F "enable_txt=true" \
|
||||
-F "brand_prefix=NOMAD" \
|
||||
-F "style_params=@styles.json" \
|
||||
-F "style_karaoke_background=@background.png"
|
||||
```
|
||||
|
||||
### Job Status
|
||||
|
||||
**GET** `/api/jobs/{job_id}`
|
||||
|
||||
Get job status and details.
|
||||
|
||||
```bash
|
||||
curl "https://api.example.com/api/jobs/abc12345"
|
||||
```
|
||||
|
||||
### List Jobs
|
||||
|
||||
**GET** `/api/jobs`
|
||||
|
||||
List all jobs with optional status filter.
|
||||
|
||||
```bash
|
||||
curl "https://api.example.com/api/jobs?status=complete&limit=10"
|
||||
```
|
||||
|
||||
### Cancel Job
|
||||
|
||||
**POST** `/api/jobs/{job_id}/cancel`
|
||||
|
||||
Cancel a running job.
|
||||
|
||||
```bash
|
||||
curl -X POST "https://api.example.com/api/jobs/abc12345/cancel" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"reason": "User cancelled"}'
|
||||
```
|
||||
|
||||
### Delete Job
|
||||
|
||||
**DELETE** `/api/jobs/{job_id}`
|
||||
|
||||
Delete a job and its files.
|
||||
|
||||
```bash
|
||||
curl -X DELETE "https://api.example.com/api/jobs/abc12345?delete_files=true"
|
||||
```
|
||||
|
||||
### Lyrics Review
|
||||
|
||||
**GET** `/api/review/{job_id}/correction-data`
|
||||
|
||||
Get correction data for lyrics review.
|
||||
|
||||
**POST** `/api/review/{job_id}/complete`
|
||||
|
||||
Submit corrected lyrics and trigger video rendering.
|
||||
|
||||
### Instrumental Selection
|
||||
|
||||
**GET** `/api/jobs/{job_id}/instrumental-options`
|
||||
|
||||
Get available instrumental options.
|
||||
|
||||
**POST** `/api/jobs/{job_id}/select-instrumental`
|
||||
|
||||
Submit instrumental selection (clean or with_backing).
|
||||
|
||||
```bash
|
||||
curl -X POST "https://api.example.com/api/jobs/abc12345/select-instrumental" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"selection": "clean"}'
|
||||
```
|
||||
|
||||
### Download Files
|
||||
|
||||
**GET** `/api/jobs/{job_id}/download-urls`
|
||||
|
||||
Get download URLs for all output files.
|
||||
|
||||
**GET** `/api/jobs/{job_id}/download/{category}/{file_key}`
|
||||
|
||||
Stream download a specific file.
|
||||
|
||||
### Health Check
|
||||
|
||||
**GET** `/api/health`
|
||||
|
||||
Check backend health status.
|
||||
|
||||
---
|
||||
|
||||
## 🧪 Development
|
||||
|
||||
### Running Tests
|
||||
|
||||
The project uses pytest for testing with unit and integration tests:
|
||||
|
||||
```bash
|
||||
# Run all tests (unit tests first, then integration tests)
|
||||
pytest
|
||||
# Run all tests
|
||||
pytest tests/ backend/tests/ -v
|
||||
|
||||
# Run only unit tests (fast feedback during development)
|
||||
pytest -m "not integration"
|
||||
# Run only unit tests
|
||||
pytest tests/unit/ -v
|
||||
|
||||
# Run only integration tests (comprehensive end-to-end testing)
|
||||
pytest -m integration
|
||||
# Run with coverage
|
||||
pytest tests/unit/ -v --cov=karaoke_gen --cov-report=term-missing
|
||||
```
|
||||
|
||||
Unit tests run quickly and provide fast feedback, while integration tests are slower but test the full workflow end-to-end.
|
||||
### Project Structure
|
||||
|
||||
## License
|
||||
```
|
||||
karaoke-gen/
|
||||
├── karaoke_gen/ # Core CLI package
|
||||
│ ├── utils/
|
||||
│ │ ├── gen_cli.py # Local CLI (karaoke-gen)
|
||||
│ │ └── remote_cli.py # Remote CLI (karaoke-gen-remote)
|
||||
│ ├── karaoke_finalise/ # Video encoding, packaging, distribution
|
||||
│ └── style_loader.py # Unified style configuration
|
||||
├── backend/ # Cloud backend (FastAPI)
|
||||
│ ├── api/routes/ # API endpoints
|
||||
│ ├── workers/ # Background processing workers
|
||||
│ └── services/ # Business logic services
|
||||
├── infrastructure/ # Pulumi IaC for GCP
|
||||
├── docs/ # Documentation
|
||||
└── tests/ # Test suite
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📄 License
|
||||
|
||||
MIT
|
||||
|
||||
---
|
||||
|
||||
## 🤝 Contributing
|
||||
|
||||
Contributions are welcome! Please see our contributing guidelines.
|
||||
@@ -1,634 +0,0 @@
|
||||
"""
|
||||
Modal Karaoke Generator API Backend
|
||||
|
||||
This module contains the Modal application structure for running karaoke generation
|
||||
as serverless functions with GPU acceleration and API endpoints for the frontend.
|
||||
"""
|
||||
|
||||
import modal
|
||||
import uuid
|
||||
import json
|
||||
import traceback
|
||||
import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
from fastapi import FastAPI, Request, Form, HTTPException
|
||||
from fastapi.responses import JSONResponse, FileResponse
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from pydantic import BaseModel
|
||||
|
||||
# Define the environment for our functions - using Python 3.13 for latest features
|
||||
karaoke_image = (
|
||||
modal.Image.debian_slim(python_version="3.13")
|
||||
.pip_install([
|
||||
# Core dependencies
|
||||
"torch>=2.7",
|
||||
"requests>=2",
|
||||
"beautifulsoup4>=4",
|
||||
"yt-dlp",
|
||||
"lyricsgenius>=3",
|
||||
"fetch-lyrics-from-genius>=0.1",
|
||||
"pillow>=10.1",
|
||||
"google-api-python-client",
|
||||
"google-auth",
|
||||
"google-auth-oauthlib",
|
||||
"google-auth-httplib2",
|
||||
"thefuzz>=0.22",
|
||||
"numpy>=2",
|
||||
"audio-separator[cpu]>=0.34.0",
|
||||
"lyrics-converter>=0.2.1",
|
||||
"lyrics-transcriber>=0.54",
|
||||
"kbputils>=0.0.12",
|
||||
"fuzzywuzzy>=0.18",
|
||||
"ffmpeg-python>=0.2.0",
|
||||
"pydub>=0.25",
|
||||
"opencv-python>=4.8",
|
||||
"openai-whisper>=20240930",
|
||||
"soundfile>=0.12",
|
||||
"librosa>=0.10",
|
||||
"demucs>=4.0.1",
|
||||
# FastAPI dependencies
|
||||
"fastapi>=0.104.0",
|
||||
"uvicorn>=0.24.0",
|
||||
"python-multipart>=0.0.6",
|
||||
])
|
||||
.apt_install([
|
||||
"ffmpeg",
|
||||
"libsndfile1",
|
||||
"libsox-dev",
|
||||
"sox",
|
||||
])
|
||||
.add_local_dir("karaoke_gen", "/root/karaoke_gen")
|
||||
.add_local_file("core.py", "/root/core.py")
|
||||
)
|
||||
|
||||
# Define the Modal app
|
||||
app = modal.App("karaoke-generator-webapp")
|
||||
|
||||
# Define persistent storage volumes
|
||||
model_volume = modal.Volume.from_name("karaoke-models", create_if_missing=True)
|
||||
output_volume = modal.Volume.from_name("karaoke-output", create_if_missing=True)
|
||||
|
||||
# Define serverless dictionaries to hold job states and logs
|
||||
job_status_dict = modal.Dict.from_name("karaoke-job-statuses", create_if_missing=True)
|
||||
job_logs_dict = modal.Dict.from_name("karaoke-job-logs", create_if_missing=True)
|
||||
|
||||
# Mount volumes to specific paths inside the container
|
||||
VOLUME_CONFIG = {
|
||||
"/models": model_volume,
|
||||
"/output": output_volume
|
||||
}
|
||||
|
||||
# Pydantic models for API requests
|
||||
class JobSubmissionRequest(BaseModel):
|
||||
url: str
|
||||
|
||||
class LyricsReviewRequest(BaseModel):
|
||||
lyrics: str
|
||||
|
||||
# GPU Worker Functions
|
||||
@app.function(
|
||||
image=karaoke_image,
|
||||
gpu="any",
|
||||
volumes=VOLUME_CONFIG,
|
||||
secrets=[modal.Secret.from_name("karaoke-api-keys")],
|
||||
timeout=1800,
|
||||
)
|
||||
def process_part_one(job_id: str, youtube_url: str):
|
||||
"""First phase: Download audio, separate, and transcribe lyrics."""
|
||||
import sys
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
|
||||
def log_message(level: str, message: str):
|
||||
"""Log a message with timestamp and level."""
|
||||
timestamp = datetime.now().isoformat()
|
||||
log_entry = {
|
||||
"timestamp": timestamp,
|
||||
"level": level,
|
||||
"message": message
|
||||
}
|
||||
|
||||
# Get existing logs or create new list
|
||||
existing_logs = job_logs_dict.get(job_id, [])
|
||||
existing_logs.append(log_entry)
|
||||
job_logs_dict[job_id] = existing_logs
|
||||
|
||||
print(f"[{level}] {message}")
|
||||
|
||||
try:
|
||||
from core import CoreKaraokeProcessor
|
||||
|
||||
log_message("INFO", f"Starting job {job_id} for URL: {youtube_url}")
|
||||
|
||||
# Update status
|
||||
job_status_dict[job_id] = {
|
||||
"status": "processing_audio",
|
||||
"progress": 10,
|
||||
"url": youtube_url,
|
||||
"created_at": datetime.now().isoformat()
|
||||
}
|
||||
|
||||
# Initialize processor
|
||||
processor = CoreKaraokeProcessor()
|
||||
|
||||
# Phase 1: Download and prep audio
|
||||
log_message("INFO", "Downloading and preparing audio...")
|
||||
output_dir = Path(f"/output/{job_id}")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
original_audio_path = processor.download_and_prep_audio(youtube_url, str(output_dir))
|
||||
log_message("INFO", f"Audio downloaded to: {original_audio_path}")
|
||||
|
||||
# Phase 2: Audio separation
|
||||
job_status_dict[job_id] = {"status": "processing_audio", "progress": 30, "url": youtube_url}
|
||||
log_message("INFO", "Starting audio separation...")
|
||||
|
||||
instrumental_path, vocals_path = processor.run_audio_separation(original_audio_path, "/models")
|
||||
log_message("INFO", f"Audio separated - Instrumental: {instrumental_path}, Vocals: {vocals_path}")
|
||||
|
||||
# Phase 3: Transcribe lyrics
|
||||
job_status_dict[job_id] = {"status": "transcribing", "progress": 60, "url": youtube_url}
|
||||
log_message("INFO", "Starting lyrics transcription...")
|
||||
|
||||
transcription_data = processor.transcribe_lyrics(vocals_path)
|
||||
log_message("INFO", "Lyrics transcribed successfully")
|
||||
|
||||
# Save transcription data
|
||||
transcription_path = output_dir / "transcription_raw.json"
|
||||
with open(transcription_path, 'w') as f:
|
||||
json.dump(transcription_data, f, indent=2)
|
||||
|
||||
# Update status to awaiting review
|
||||
job_status_dict[job_id] = {
|
||||
"status": "awaiting_review",
|
||||
"progress": 75,
|
||||
"url": youtube_url,
|
||||
"transcription_path": str(transcription_path)
|
||||
}
|
||||
|
||||
log_message("SUCCESS", f"Phase 1 completed for job {job_id}. Awaiting lyrics review.")
|
||||
return {"status": "success", "message": "Phase 1 completed, awaiting lyrics review"}
|
||||
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
error_traceback = traceback.format_exc()
|
||||
|
||||
log_message("ERROR", f"Phase 1 failed: {error_msg}")
|
||||
log_message("ERROR", f"Traceback: {error_traceback}")
|
||||
|
||||
job_status_dict[job_id] = {
|
||||
"status": "error",
|
||||
"progress": 0,
|
||||
"url": youtube_url,
|
||||
"error": error_msg,
|
||||
"traceback": error_traceback
|
||||
}
|
||||
|
||||
raise Exception(f"Phase 1 failed: {error_msg}")
|
||||
|
||||
@app.function(
|
||||
image=karaoke_image,
|
||||
gpu="any",
|
||||
volumes=VOLUME_CONFIG,
|
||||
timeout=1800,
|
||||
)
|
||||
def process_part_two(job_id: str):
|
||||
"""Second phase: Generate final video with corrected lyrics."""
|
||||
import sys
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
|
||||
def log_message(level: str, message: str):
|
||||
"""Log a message with timestamp and level."""
|
||||
timestamp = datetime.now().isoformat()
|
||||
log_entry = {
|
||||
"timestamp": timestamp,
|
||||
"level": level,
|
||||
"message": message
|
||||
}
|
||||
|
||||
# Get existing logs or create new list
|
||||
existing_logs = job_logs_dict.get(job_id, [])
|
||||
existing_logs.append(log_entry)
|
||||
job_logs_dict[job_id] = existing_logs
|
||||
|
||||
print(f"[{level}] {message}")
|
||||
|
||||
try:
|
||||
from core import CoreKaraokeProcessor
|
||||
|
||||
log_message("INFO", f"Starting phase 2 for job {job_id}")
|
||||
|
||||
# Update status
|
||||
job_status_dict[job_id] = {"status": "rendering", "progress": 80}
|
||||
|
||||
# Initialize processor
|
||||
processor = CoreKaraokeProcessor()
|
||||
|
||||
# Load corrected lyrics
|
||||
output_dir = Path(f"/output/{job_id}")
|
||||
corrected_lyrics_path = output_dir / "lyrics_corrected.json"
|
||||
|
||||
if not corrected_lyrics_path.exists():
|
||||
raise Exception("Corrected lyrics file not found")
|
||||
|
||||
with open(corrected_lyrics_path, 'r') as f:
|
||||
corrected_lyrics = json.load(f)
|
||||
|
||||
log_message("INFO", "Corrected lyrics loaded")
|
||||
|
||||
# Generate video assets
|
||||
log_message("INFO", "Starting video generation...")
|
||||
|
||||
instrumental_path = output_dir / "instrumental.wav"
|
||||
video_path = processor.generate_video_assets(corrected_lyrics, str(instrumental_path))
|
||||
|
||||
log_message("INFO", f"Video generated: {video_path}")
|
||||
|
||||
# Update status to complete
|
||||
job_status_dict[job_id] = {
|
||||
"status": "complete",
|
||||
"progress": 100,
|
||||
"video_path": video_path,
|
||||
"video_url": f"/api/jobs/{job_id}/download"
|
||||
}
|
||||
|
||||
log_message("SUCCESS", f"Job {job_id} completed successfully!")
|
||||
return {"status": "success", "message": "Video generation completed", "video_path": video_path}
|
||||
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
error_traceback = traceback.format_exc()
|
||||
|
||||
log_message("ERROR", f"Phase 2 failed: {error_msg}")
|
||||
log_message("ERROR", f"Traceback: {error_traceback}")
|
||||
|
||||
job_status_dict[job_id] = {
|
||||
"status": "error",
|
||||
"progress": 0,
|
||||
"error": error_msg,
|
||||
"traceback": error_traceback
|
||||
}
|
||||
|
||||
raise Exception(f"Phase 2 failed: {error_msg}")
|
||||
|
||||
# FastAPI Application for API endpoints
|
||||
api_app = FastAPI(title="Karaoke Generator API", version="1.0.0")
|
||||
|
||||
# Add CORS middleware
|
||||
api_app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"], # In production, replace with specific origins
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# API Routes
|
||||
@api_app.post("/api/submit")
|
||||
async def submit_job(request: JobSubmissionRequest):
|
||||
"""Submit a new karaoke generation job."""
|
||||
try:
|
||||
job_id = str(uuid.uuid4())[:8]
|
||||
|
||||
# Initialize job status
|
||||
job_status_dict[job_id] = {
|
||||
"status": "queued",
|
||||
"progress": 0,
|
||||
"url": request.url,
|
||||
"created_at": datetime.datetime.now().isoformat()
|
||||
}
|
||||
|
||||
# Initialize job logs
|
||||
job_logs_dict[job_id] = []
|
||||
|
||||
# Spawn the background job
|
||||
process_part_one.spawn(job_id, request.url)
|
||||
|
||||
return JSONResponse({
|
||||
"status": "success",
|
||||
"job_id": job_id,
|
||||
"message": "Job submitted successfully"
|
||||
}, status_code=202)
|
||||
|
||||
except Exception as e:
|
||||
return JSONResponse({
|
||||
"status": "error",
|
||||
"message": str(e)
|
||||
}, status_code=500)
|
||||
|
||||
@api_app.get("/api/jobs")
|
||||
async def get_all_jobs():
|
||||
"""Get status of all jobs."""
|
||||
try:
|
||||
jobs = dict(job_status_dict.items())
|
||||
return JSONResponse(jobs)
|
||||
except Exception as e:
|
||||
return JSONResponse({"error": str(e)}, status_code=500)
|
||||
|
||||
@api_app.get("/api/jobs/{job_id}")
|
||||
async def get_job_status(job_id: str):
|
||||
"""Get status of a specific job."""
|
||||
try:
|
||||
job_data = job_status_dict.get(job_id)
|
||||
if not job_data:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
return JSONResponse(job_data)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
return JSONResponse({"error": str(e)}, status_code=500)
|
||||
|
||||
@api_app.delete("/api/jobs/{job_id}")
|
||||
async def delete_job(job_id: str):
|
||||
"""Delete a specific job."""
|
||||
try:
|
||||
if job_id not in job_status_dict:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
# Remove from status and logs
|
||||
del job_status_dict[job_id]
|
||||
if job_id in job_logs_dict:
|
||||
del job_logs_dict[job_id]
|
||||
|
||||
return JSONResponse({"status": "success", "message": f"Job {job_id} deleted"})
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
return JSONResponse({"error": str(e)}, status_code=500)
|
||||
|
||||
@api_app.post("/api/jobs/{job_id}/retry")
|
||||
async def retry_job(job_id: str):
|
||||
"""Retry a failed job."""
|
||||
try:
|
||||
job_data = job_status_dict.get(job_id)
|
||||
if not job_data:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
if job_data.get("status") != "error":
|
||||
raise HTTPException(status_code=400, detail="Job is not in error state")
|
||||
|
||||
# Reset job status
|
||||
job_status_dict[job_id] = {
|
||||
"status": "queued",
|
||||
"progress": 0,
|
||||
"url": job_data.get("url", ""),
|
||||
"created_at": datetime.datetime.now().isoformat()
|
||||
}
|
||||
|
||||
# Clear error logs and add retry log
|
||||
job_logs_dict[job_id] = [{
|
||||
"timestamp": datetime.datetime.now().isoformat(),
|
||||
"level": "INFO",
|
||||
"message": "Job retry initiated"
|
||||
}]
|
||||
|
||||
# Respawn the job
|
||||
process_part_one.spawn(job_id, job_data.get("url", ""))
|
||||
|
||||
return JSONResponse({"status": "success", "message": f"Job {job_id} retry initiated"})
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
return JSONResponse({"error": str(e)}, status_code=500)
|
||||
|
||||
@api_app.get("/api/logs")
|
||||
async def get_all_logs():
|
||||
"""Get logs for all jobs."""
|
||||
try:
|
||||
logs = dict(job_logs_dict.items())
|
||||
return JSONResponse(logs)
|
||||
except Exception as e:
|
||||
return JSONResponse({"error": str(e)}, status_code=500)
|
||||
|
||||
@api_app.get("/api/logs/{job_id}")
|
||||
async def get_job_logs(job_id: str):
|
||||
"""Get logs for a specific job."""
|
||||
try:
|
||||
logs = job_logs_dict.get(job_id, [])
|
||||
return JSONResponse(logs)
|
||||
except Exception as e:
|
||||
return JSONResponse({"error": str(e)}, status_code=500)
|
||||
|
||||
@api_app.get("/api/stats")
|
||||
async def get_stats():
|
||||
"""Get statistics about all jobs."""
|
||||
try:
|
||||
jobs = dict(job_status_dict.items())
|
||||
|
||||
stats = {
|
||||
"total": len(jobs),
|
||||
"processing": len([j for j in jobs.values() if j.get("status") in ["queued", "processing_audio", "transcribing", "rendering"]]),
|
||||
"awaiting_review": len([j for j in jobs.values() if j.get("status") == "awaiting_review"]),
|
||||
"complete": len([j for j in jobs.values() if j.get("status") == "complete"]),
|
||||
"error": len([j for j in jobs.values() if j.get("status") == "error"])
|
||||
}
|
||||
|
||||
return JSONResponse(stats)
|
||||
except Exception as e:
|
||||
return JSONResponse({"error": str(e)}, status_code=500)
|
||||
|
||||
@api_app.get("/api/review/{job_id}")
|
||||
async def get_lyrics_for_review(job_id: str):
|
||||
"""Get lyrics for review (returns HTML page)."""
|
||||
try:
|
||||
job_data = job_status_dict.get(job_id)
|
||||
if not job_data:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
if job_data.get("status") != "awaiting_review":
|
||||
raise HTTPException(status_code=400, detail="Job is not awaiting review")
|
||||
|
||||
# Load raw transcription
|
||||
output_dir = Path(f"/output/{job_id}")
|
||||
transcription_path = output_dir / "transcription_raw.json"
|
||||
|
||||
if not transcription_path.exists():
|
||||
raise HTTPException(status_code=404, detail="Transcription not found")
|
||||
|
||||
with open(transcription_path, 'r') as f:
|
||||
raw_lyrics = json.load(f)
|
||||
|
||||
# Return HTML page for lyrics review
|
||||
html_content = f"""
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Review Lyrics - Job {job_id}</title>
|
||||
<style>
|
||||
body {{ font-family: Arial, sans-serif; max-width: 800px; margin: 0 auto; padding: 20px; }}
|
||||
textarea {{ width: 100%; height: 400px; font-family: monospace; }}
|
||||
button {{ padding: 10px 20px; background: #007bff; color: white; border: none; border-radius: 5px; cursor: pointer; }}
|
||||
button:hover {{ background: #0056b3; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Review Lyrics for Job {job_id}</h1>
|
||||
<form action="/api/review/{job_id}" method="post">
|
||||
<textarea name="lyrics" placeholder="Edit the lyrics here...">{json.dumps(raw_lyrics, indent=2)}</textarea>
|
||||
<br><br>
|
||||
<button type="submit">✅ Approve and Continue to Video Generation</button>
|
||||
</form>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
from fastapi.responses import HTMLResponse
|
||||
return HTMLResponse(content=html_content)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
return JSONResponse({"error": str(e)}, status_code=500)
|
||||
|
||||
@api_app.post("/api/review/{job_id}")
|
||||
async def submit_lyrics_review(job_id: str, lyrics: str = Form(...)):
|
||||
"""Submit reviewed lyrics and continue to video generation."""
|
||||
try:
|
||||
job_data = job_status_dict.get(job_id)
|
||||
if not job_data:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
# Save corrected lyrics
|
||||
output_dir = Path(f"/output/{job_id}")
|
||||
corrected_lyrics_path = output_dir / "lyrics_corrected.json"
|
||||
|
||||
# Parse the lyrics JSON
|
||||
try:
|
||||
lyrics_data = json.loads(lyrics)
|
||||
except json.JSONDecodeError:
|
||||
# If not valid JSON, treat as plain text
|
||||
lyrics_data = {"text": lyrics}
|
||||
|
||||
with open(corrected_lyrics_path, 'w') as f:
|
||||
json.dump(lyrics_data, f, indent=2)
|
||||
|
||||
# Update job status
|
||||
job_status_dict[job_id] = {
|
||||
**job_data,
|
||||
"status": "rendering",
|
||||
"progress": 80
|
||||
}
|
||||
|
||||
# Add log entry
|
||||
existing_logs = job_logs_dict.get(job_id, [])
|
||||
existing_logs.append({
|
||||
"timestamp": datetime.datetime.now().isoformat(),
|
||||
"level": "INFO",
|
||||
"message": "Lyrics approved, starting video generation"
|
||||
})
|
||||
job_logs_dict[job_id] = existing_logs
|
||||
|
||||
# Spawn phase 2
|
||||
process_part_two.spawn(job_id)
|
||||
|
||||
return JSONResponse({
|
||||
"status": "success",
|
||||
"message": "Lyrics approved, video generation started"
|
||||
})
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
return JSONResponse({"error": str(e)}, status_code=500)
|
||||
|
||||
@api_app.get("/api/jobs/{job_id}/download")
|
||||
async def download_video(job_id: str):
|
||||
"""Download the completed video."""
|
||||
try:
|
||||
job_data = job_status_dict.get(job_id)
|
||||
if not job_data:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
if job_data.get("status") != "complete":
|
||||
raise HTTPException(status_code=400, detail="Job is not complete")
|
||||
|
||||
video_path = job_data.get("video_path")
|
||||
if not video_path or not Path(video_path).exists():
|
||||
raise HTTPException(status_code=404, detail="Video file not found")
|
||||
|
||||
return FileResponse(
|
||||
path=video_path,
|
||||
filename=f"karaoke-{job_id}.mp4",
|
||||
media_type="video/mp4"
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
return JSONResponse({"error": str(e)}, status_code=500)
|
||||
|
||||
# Admin Routes
|
||||
@api_app.post("/api/admin/clear-errors")
|
||||
async def clear_error_jobs():
|
||||
"""Clear all jobs with error status."""
|
||||
try:
|
||||
jobs_to_delete = []
|
||||
for job_id, job_data in job_status_dict.items():
|
||||
if job_data.get("status") == "error":
|
||||
jobs_to_delete.append(job_id)
|
||||
|
||||
for job_id in jobs_to_delete:
|
||||
del job_status_dict[job_id]
|
||||
if job_id in job_logs_dict:
|
||||
del job_logs_dict[job_id]
|
||||
|
||||
return JSONResponse({
|
||||
"status": "success",
|
||||
"message": f"Cleared {len(jobs_to_delete)} error jobs"
|
||||
})
|
||||
except Exception as e:
|
||||
return JSONResponse({"error": str(e)}, status_code=500)
|
||||
|
||||
@api_app.get("/api/admin/export-logs")
|
||||
async def export_logs():
|
||||
"""Export all logs as JSON file."""
|
||||
try:
|
||||
logs_data = {
|
||||
"exported_at": datetime.datetime.now().isoformat(),
|
||||
"jobs": dict(job_status_dict.items()),
|
||||
"logs": dict(job_logs_dict.items())
|
||||
}
|
||||
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
# Create temporary file
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
|
||||
json.dump(logs_data, f, indent=2)
|
||||
temp_path = f.name
|
||||
|
||||
return FileResponse(
|
||||
path=temp_path,
|
||||
filename=f"karaoke-logs-{datetime.datetime.now().strftime('%Y%m%d')}.json",
|
||||
media_type="application/json"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
return JSONResponse({"error": str(e)}, status_code=500)
|
||||
|
||||
# Health check
|
||||
@api_app.get("/api/health")
|
||||
async def health_check():
|
||||
"""Health check endpoint."""
|
||||
return JSONResponse({
|
||||
"status": "healthy",
|
||||
"timestamp": datetime.datetime.now().isoformat(),
|
||||
"version": "1.0.0"
|
||||
})
|
||||
|
||||
# Mount the FastAPI application to the Modal app
|
||||
@app.function(
|
||||
image=karaoke_image,
|
||||
volumes=VOLUME_CONFIG,
|
||||
min_containers=1, # Keep at least 1 container warm
|
||||
max_containers=10, # Allow scaling up to 10 containers
|
||||
scaledown_window=5 * 60, # Wait 5 minutes before scaling down
|
||||
)
|
||||
@modal.concurrent(max_inputs=100)
|
||||
@modal.asgi_app()
|
||||
def web_endpoint():
|
||||
"""Mount the FastAPI application."""
|
||||
return api_app
|
||||
@@ -0,0 +1,20 @@
|
||||
[run]
|
||||
source = backend
|
||||
omit =
|
||||
backend/tests/*
|
||||
backend/validate.py
|
||||
backend/tests/emulator/*
|
||||
# Workers that require external APIs are tested via integration tests
|
||||
backend/workers/screens_worker.py
|
||||
backend/workers/video_worker.py
|
||||
|
||||
[report]
|
||||
exclude_lines =
|
||||
pragma: no cover
|
||||
def __repr__
|
||||
raise NotImplementedError
|
||||
if __name__ == .__main__.:
|
||||
pass
|
||||
|
||||
show_missing = True
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# Ignore Python cache and virtual environments
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Environment files
|
||||
.env
|
||||
.env.local
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# Temporary files
|
||||
*.log
|
||||
temp/
|
||||
tmp/
|
||||
|
||||
# Google Cloud
|
||||
.gcloud/
|
||||
*.json
|
||||
|
||||
# Testing
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
# Fast application image - uses pre-built base with all dependencies
|
||||
#
|
||||
# The base image (karaoke-backend-base) contains:
|
||||
# - Python 3.11, ffmpeg, sox, system deps
|
||||
# - All pip dependencies from requirements-frozen.txt (NO code)
|
||||
#
|
||||
# This Dockerfile adds the actual application code on top.
|
||||
# Build time: ~1-2 minutes (deps already cached in base)
|
||||
#
|
||||
# If the base image doesn't exist yet, build it first:
|
||||
# gcloud builds submit --config=cloudbuild-base.yaml --project=nomadkaraoke
|
||||
|
||||
# Use pre-built base image with all dependencies
|
||||
FROM us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend-base:latest
|
||||
|
||||
# Set working directory (should already exist from base)
|
||||
WORKDIR /app
|
||||
|
||||
# Copy backend code (this is the only thing that changes frequently)
|
||||
COPY backend /app/backend
|
||||
|
||||
# Copy and reinstall karaoke_gen from local source
|
||||
# This ensures new modules (like style_loader.py) are included without rebuilding base image
|
||||
# Note: lyrics_transcriber is vendored in lyrics_transcriber_temp/ and bundled via pyproject.toml
|
||||
COPY karaoke_gen /app/karaoke_gen
|
||||
COPY lyrics_transcriber_temp /app/lyrics_transcriber_temp
|
||||
COPY pyproject.toml README.md LICENSE /app/
|
||||
RUN pip install --no-cache-dir -e .
|
||||
|
||||
# Set environment variables
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV PORT=8080
|
||||
|
||||
# Expose port
|
||||
EXPOSE 8080
|
||||
|
||||
# Run the application
|
||||
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8080"]
|
||||
@@ -0,0 +1,42 @@
|
||||
# Base image with all Python dependencies pre-installed
|
||||
# This image is rebuilt only when poetry.lock changes
|
||||
#
|
||||
# Build with: gcloud builds submit --config=cloudbuild-base.yaml
|
||||
# Or manually: docker build -f backend/Dockerfile.base -t karaoke-backend-base .
|
||||
|
||||
FROM python:3.11-slim
|
||||
|
||||
# Install system dependencies required by karaoke-gen
|
||||
RUN apt-get update && apt-get install -y \
|
||||
ffmpeg \
|
||||
libsndfile1 \
|
||||
libsox-dev \
|
||||
sox \
|
||||
build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
# Copy dependency files and source code needed for pip install
|
||||
COPY pyproject.toml README.md LICENSE /app/
|
||||
COPY karaoke_gen /app/karaoke_gen
|
||||
COPY lyrics_transcriber_temp /app/lyrics_transcriber_temp
|
||||
|
||||
# Install all Python dependencies (the slow part - cached in base image)
|
||||
RUN pip install --no-cache-dir --upgrade pip && \
|
||||
pip install --no-cache-dir -e .
|
||||
|
||||
# Download spaCy language model for lyrics transcription
|
||||
# This must be done at build time, not runtime (Cloud Run network restrictions)
|
||||
RUN python -m spacy download en_core_web_sm
|
||||
|
||||
# Label with build info
|
||||
ARG BUILD_DATE
|
||||
ARG BUILD_ID
|
||||
ARG POETRY_LOCK_HASH
|
||||
LABEL org.opencontainers.image.created="${BUILD_DATE}"
|
||||
LABEL org.opencontainers.image.revision="${BUILD_ID}"
|
||||
LABEL org.opencontainers.image.title="karaoke-backend-base"
|
||||
LABEL org.opencontainers.image.description="Base image with Python dependencies for karaoke-backend"
|
||||
LABEL poetry.lock.hash="${POETRY_LOCK_HASH}"
|
||||
@@ -0,0 +1,242 @@
|
||||
# Backend Development Setup
|
||||
|
||||
## One-Time Setup
|
||||
|
||||
### 1. Create Virtual Environment
|
||||
|
||||
```bash
|
||||
cd /Users/andrew/Projects/karaoke-gen
|
||||
python3 -m venv backend/venv
|
||||
```
|
||||
|
||||
### 2. Activate Virtual Environment
|
||||
|
||||
```bash
|
||||
source backend/venv/bin/activate
|
||||
```
|
||||
|
||||
### 3. Install Dependencies
|
||||
|
||||
**IMPORTANT:** Use Python 3.12 (not 3.13/3.14 - pydantic has compatibility issues)
|
||||
|
||||
```bash
|
||||
# Backend dependencies
|
||||
pip install -r backend/requirements.txt
|
||||
|
||||
# Install karaoke_gen in development mode
|
||||
pip install -e .
|
||||
|
||||
# Install lyrics_transcriber submodule
|
||||
pip install -e lyrics_transcriber_local/
|
||||
|
||||
# Install Google API client (for YouTube upload)
|
||||
pip install google-api-python-client google-auth-httplib2 google-auth-oauthlib
|
||||
```
|
||||
|
||||
### 4. Add Shell Alias (Optional)
|
||||
|
||||
Add to `~/.zshrc`:
|
||||
|
||||
```bash
|
||||
alias backend='cd /Users/andrew/Projects/karaoke-gen && source backend/venv/bin/activate'
|
||||
```
|
||||
|
||||
Then you can just run:
|
||||
```bash
|
||||
backend
|
||||
# Now you're in the project directory with venv activated!
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Daily Workflow
|
||||
|
||||
### Before Making Changes
|
||||
|
||||
```bash
|
||||
# Activate venv
|
||||
source backend/venv/bin/activate
|
||||
|
||||
# Or if you set up the alias:
|
||||
backend
|
||||
```
|
||||
|
||||
### After Making Changes
|
||||
|
||||
```bash
|
||||
# 1. Run full validation (catches import errors!)
|
||||
python3 backend/validate.py
|
||||
|
||||
# 2. If validation passes, deploy
|
||||
./scripts/deploy.sh
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Validation Options
|
||||
|
||||
### Full Validation (Recommended - Requires venv)
|
||||
|
||||
```bash
|
||||
source backend/venv/bin/activate
|
||||
python3 backend/validate.py
|
||||
```
|
||||
|
||||
**Catches:**
|
||||
- ✅ Syntax errors
|
||||
- ✅ **Import errors** (like the processing_service issue!)
|
||||
- ✅ Configuration issues
|
||||
- ✅ FastAPI app creation problems
|
||||
- ✅ Missing dependencies
|
||||
|
||||
**This would have caught the ModuleNotFoundError!**
|
||||
|
||||
### Quick Check (No venv needed)
|
||||
|
||||
```bash
|
||||
./backend/quick-check.sh
|
||||
```
|
||||
|
||||
**Catches:**
|
||||
- ✅ Syntax errors
|
||||
- ✅ Missing files
|
||||
- ✅ Some obvious import issues
|
||||
|
||||
**Good for quick checks, but won't catch all import errors**
|
||||
|
||||
---
|
||||
|
||||
## Example: Complete Development Session
|
||||
|
||||
```bash
|
||||
# 1. Activate environment
|
||||
backend # or: source backend/venv/bin/activate
|
||||
|
||||
# 2. Make your changes
|
||||
vim backend/api/routes/jobs.py
|
||||
|
||||
# 3. Validate (catches import errors!)
|
||||
python3 backend/validate.py
|
||||
# ✅ All validations passed!
|
||||
|
||||
# 4. Deploy
|
||||
./scripts/deploy.sh
|
||||
|
||||
# 5. Test
|
||||
export BACKEND_URL="https://karaoke-backend-ipzqd2k4yq-uc.a.run.app"
|
||||
export AUTH_TOKEN=$(gcloud auth print-identity-token)
|
||||
curl -H "Authorization: Bearer $AUTH_TOKEN" $BACKEND_URL/api/health
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## What About karaoke_gen Imports?
|
||||
|
||||
The backend imports from `karaoke_gen` (the CLI package):
|
||||
|
||||
```python
|
||||
from karaoke_gen.karaoke_gen import KaraokePrep
|
||||
from karaoke_gen.audio_processor import AudioProcessor
|
||||
```
|
||||
|
||||
**Options:**
|
||||
|
||||
### Option 1: Install in development mode (Recommended)
|
||||
|
||||
```bash
|
||||
source backend/venv/bin/activate
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
This makes `karaoke_gen` available in the venv without copying files.
|
||||
|
||||
### Option 2: Use system Python
|
||||
|
||||
Your system Python already has `karaoke_gen` installed, so validation will work even if the venv doesn't have it (Python will fall back to system packages).
|
||||
|
||||
---
|
||||
|
||||
## Running the Backend Locally
|
||||
|
||||
```bash
|
||||
# Activate venv
|
||||
source backend/venv/bin/activate
|
||||
|
||||
# Set required environment variables
|
||||
export GOOGLE_CLOUD_PROJECT="nomadkaraoke"
|
||||
export GCS_BUCKET_NAME="karaoke-gen-storage-nomadkaraoke"
|
||||
|
||||
# Optional: Point to local credentials
|
||||
export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account-key.json"
|
||||
|
||||
# Run the server
|
||||
cd backend
|
||||
uvicorn main:app --reload --port 8080
|
||||
```
|
||||
|
||||
Visit: http://localhost:8080/api/health
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### "No module named 'fastapi'"
|
||||
|
||||
**Fix:**
|
||||
```bash
|
||||
source backend/venv/bin/activate
|
||||
pip install -r backend/requirements.txt
|
||||
```
|
||||
|
||||
### "No module named 'karaoke_gen'"
|
||||
|
||||
**Fix:**
|
||||
```bash
|
||||
source backend/venv/bin/activate
|
||||
cd /Users/andrew/Projects/karaoke-gen
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
### Validation passes locally but fails in Cloud Run
|
||||
|
||||
**Check:**
|
||||
1. Is `requirements.txt` up to date?
|
||||
2. Did you add a new dependency without adding it to `requirements.txt`?
|
||||
3. Run: `pip freeze | grep <package-name>`
|
||||
|
||||
---
|
||||
|
||||
## Updating Dependencies
|
||||
|
||||
When you add new packages:
|
||||
|
||||
```bash
|
||||
# 1. Install in venv
|
||||
source backend/venv/bin/activate
|
||||
pip install new-package
|
||||
|
||||
# 2. Update requirements.txt
|
||||
pip freeze > backend/requirements.txt
|
||||
|
||||
# 3. Or manually add just the new package:
|
||||
echo "new-package==1.2.3" >> backend/requirements.txt
|
||||
|
||||
# 4. Validate
|
||||
python3 backend/validate.py
|
||||
|
||||
# 5. Deploy
|
||||
./scripts/deploy.sh
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
**Best practice workflow:**
|
||||
|
||||
1. **One-time setup:** Create venv and install dependencies
|
||||
2. **Daily:** Activate venv when working on backend
|
||||
3. **Before deploy:** Run `python3 backend/validate.py` (catches import errors!)
|
||||
4. **Deploy:** Use `./scripts/deploy.sh`
|
||||
|
||||
**The import error would have been caught if you ran validation in a venv!** ✅
|
||||
Whitespace-only changes.
Whitespace-only changes.
@@ -0,0 +1,138 @@
|
||||
"""
|
||||
FastAPI dependencies for authentication and authorization.
|
||||
"""
|
||||
import logging
|
||||
from typing import Optional, Tuple
|
||||
from fastapi import Depends, HTTPException, Header, Query, Request
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
|
||||
from backend.services.auth_service import get_auth_service, UserType, AuthService
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# HTTP Bearer security scheme (Authorization: Bearer <token>)
|
||||
security = HTTPBearer(auto_error=False)
|
||||
|
||||
|
||||
async def get_token_from_request(
|
||||
request: Request,
|
||||
credentials: Optional[HTTPAuthorizationCredentials] = Depends(security),
|
||||
token: Optional[str] = Query(None, description="Access token (alternative to Bearer header)")
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
Extract access token from request.
|
||||
|
||||
Supports two methods:
|
||||
1. Authorization header: Authorization: Bearer <token>
|
||||
2. Query parameter: ?token=<token> (for download links)
|
||||
|
||||
Returns:
|
||||
Token string or None
|
||||
"""
|
||||
# Try Authorization header first
|
||||
if credentials:
|
||||
return credentials.credentials
|
||||
|
||||
# Try query parameter
|
||||
if token:
|
||||
return token
|
||||
|
||||
return None
|
||||
|
||||
|
||||
async def require_auth(
|
||||
request: Request,
|
||||
auth_service: AuthService = Depends(get_auth_service),
|
||||
credentials: Optional[HTTPAuthorizationCredentials] = Depends(security),
|
||||
token: Optional[str] = Query(None)
|
||||
) -> Tuple[str, UserType, int]:
|
||||
"""
|
||||
Require authentication for an endpoint.
|
||||
|
||||
Returns:
|
||||
(token, user_type, remaining_uses)
|
||||
|
||||
Raises:
|
||||
HTTPException: 401 if authentication fails
|
||||
"""
|
||||
# Get token from request
|
||||
token_str = await get_token_from_request(request, credentials, token)
|
||||
|
||||
if not token_str:
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail="Authentication required. Provide token via Authorization header or ?token= parameter"
|
||||
)
|
||||
|
||||
# Validate token
|
||||
is_valid, user_type, remaining_uses, message = auth_service.validate_token(token_str)
|
||||
|
||||
if not is_valid:
|
||||
logger.warning(f"Authentication failed: {message}")
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail=f"Authentication failed: {message}"
|
||||
)
|
||||
|
||||
logger.info(f"Authenticated as {user_type} (remaining: {remaining_uses})")
|
||||
|
||||
return token_str, user_type, remaining_uses
|
||||
|
||||
|
||||
async def require_admin(
|
||||
auth_data: Tuple[str, UserType, int] = Depends(require_auth)
|
||||
) -> Tuple[str, UserType, int]:
|
||||
"""
|
||||
Require admin access for an endpoint.
|
||||
|
||||
Returns:
|
||||
(token, user_type, remaining_uses)
|
||||
|
||||
Raises:
|
||||
HTTPException: 403 if user is not admin
|
||||
"""
|
||||
token, user_type, remaining_uses = auth_data
|
||||
|
||||
if user_type != UserType.ADMIN:
|
||||
logger.warning(f"Admin access denied for {user_type} user")
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="Admin access required"
|
||||
)
|
||||
|
||||
return token, user_type, remaining_uses
|
||||
|
||||
|
||||
def optional_auth(
|
||||
auth_service: AuthService = Depends(get_auth_service),
|
||||
credentials: Optional[HTTPAuthorizationCredentials] = Depends(security),
|
||||
token: Optional[str] = Query(None)
|
||||
) -> Optional[Tuple[str, UserType, int]]:
|
||||
"""
|
||||
Optional authentication - doesn't fail if no token provided.
|
||||
|
||||
Useful for endpoints that have different behavior for authenticated users.
|
||||
|
||||
Returns:
|
||||
(token, user_type, remaining_uses) if authenticated, None otherwise
|
||||
"""
|
||||
# Get token
|
||||
token_str = None
|
||||
if credentials:
|
||||
token_str = credentials.credentials
|
||||
elif token:
|
||||
token_str = token
|
||||
|
||||
if not token_str:
|
||||
return None
|
||||
|
||||
# Validate token
|
||||
is_valid, user_type, remaining_uses, message = auth_service.validate_token(token_str)
|
||||
|
||||
if not is_valid:
|
||||
logger.debug(f"Optional auth failed: {message}")
|
||||
return None
|
||||
|
||||
return token_str, user_type, remaining_uses
|
||||
Whitespace-only changes.
@@ -0,0 +1,348 @@
|
||||
"""
|
||||
OAuth credential management API routes.
|
||||
|
||||
Provides endpoints for:
|
||||
1. Checking credential status
|
||||
2. Device authorization flow for re-authentication
|
||||
3. Credential validation before job submission
|
||||
"""
|
||||
import logging
|
||||
from typing import Optional
|
||||
from datetime import datetime
|
||||
from fastapi import APIRouter, HTTPException, BackgroundTasks
|
||||
from pydantic import BaseModel
|
||||
|
||||
from backend.services.credential_manager import (
|
||||
get_credential_manager,
|
||||
CredentialStatus,
|
||||
CredentialCheckResult,
|
||||
)
|
||||
from backend.config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/auth", tags=["auth"])
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Request/Response Models
|
||||
# ===========================================================================
|
||||
|
||||
class CredentialStatusResponse(BaseModel):
|
||||
"""Response model for credential status."""
|
||||
service: str
|
||||
status: str
|
||||
message: str
|
||||
last_checked: datetime
|
||||
expires_at: Optional[datetime] = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class AllCredentialsStatusResponse(BaseModel):
|
||||
"""Response model for all credentials status."""
|
||||
youtube: CredentialStatusResponse
|
||||
gdrive: CredentialStatusResponse
|
||||
dropbox: CredentialStatusResponse
|
||||
all_valid: bool
|
||||
services_needing_auth: list[str]
|
||||
|
||||
|
||||
class DeviceAuthStartRequest(BaseModel):
|
||||
"""Request to start device authorization flow."""
|
||||
client_id: Optional[str] = None # Optional - reads from Secret Manager if not provided
|
||||
client_secret: Optional[str] = None # Optional - reads from Secret Manager if not provided
|
||||
|
||||
|
||||
class DeviceAuthStartResponse(BaseModel):
|
||||
"""Response with device authorization info."""
|
||||
device_code: str
|
||||
user_code: str
|
||||
verification_url: str
|
||||
expires_in: int
|
||||
interval: int
|
||||
instructions: str
|
||||
|
||||
|
||||
class DeviceAuthPollResponse(BaseModel):
|
||||
"""Response from polling device authorization."""
|
||||
status: str # "pending", "complete", "expired", "error"
|
||||
message: Optional[str] = None
|
||||
credentials_saved: bool = False
|
||||
|
||||
|
||||
class CredentialValidationRequest(BaseModel):
|
||||
"""Request to validate credentials for specific services."""
|
||||
youtube: bool = False
|
||||
gdrive: bool = False
|
||||
dropbox: bool = False
|
||||
|
||||
|
||||
class CredentialValidationResponse(BaseModel):
|
||||
"""Response from credential validation."""
|
||||
valid: bool
|
||||
invalid_services: list[str]
|
||||
message: str
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Endpoints
|
||||
# ===========================================================================
|
||||
|
||||
@router.get("/status", response_model=AllCredentialsStatusResponse)
|
||||
async def get_credentials_status():
|
||||
"""
|
||||
Get the status of all OAuth credentials.
|
||||
|
||||
Returns the validation status for YouTube, Google Drive, and Dropbox
|
||||
credentials, including whether they are valid, expired, or need
|
||||
re-authorization.
|
||||
"""
|
||||
manager = get_credential_manager()
|
||||
results = manager.check_all_credentials()
|
||||
|
||||
def to_response(result: CredentialCheckResult) -> CredentialStatusResponse:
|
||||
return CredentialStatusResponse(
|
||||
service=result.service,
|
||||
status=result.status.value,
|
||||
message=result.message,
|
||||
last_checked=result.last_checked,
|
||||
expires_at=result.expires_at
|
||||
)
|
||||
|
||||
services_needing_auth = [
|
||||
name for name, result in results.items()
|
||||
if result.status in (CredentialStatus.INVALID, CredentialStatus.EXPIRED, CredentialStatus.NOT_CONFIGURED)
|
||||
]
|
||||
|
||||
return AllCredentialsStatusResponse(
|
||||
youtube=to_response(results["youtube"]),
|
||||
gdrive=to_response(results["gdrive"]),
|
||||
dropbox=to_response(results["dropbox"]),
|
||||
all_valid=len(services_needing_auth) == 0,
|
||||
services_needing_auth=services_needing_auth
|
||||
)
|
||||
|
||||
|
||||
@router.get("/status/{service}", response_model=CredentialStatusResponse)
|
||||
async def get_service_credential_status(service: str):
|
||||
"""
|
||||
Get the status of a specific service's OAuth credentials.
|
||||
|
||||
Args:
|
||||
service: One of "youtube", "gdrive", "dropbox"
|
||||
"""
|
||||
manager = get_credential_manager()
|
||||
|
||||
if service == "youtube":
|
||||
result = manager.check_youtube_credentials()
|
||||
elif service == "gdrive":
|
||||
result = manager.check_gdrive_credentials()
|
||||
elif service == "dropbox":
|
||||
result = manager.check_dropbox_credentials()
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown service: {service}")
|
||||
|
||||
return CredentialStatusResponse(
|
||||
service=result.service,
|
||||
status=result.status.value,
|
||||
message=result.message,
|
||||
last_checked=result.last_checked,
|
||||
expires_at=result.expires_at
|
||||
)
|
||||
|
||||
|
||||
@router.post("/validate", response_model=CredentialValidationResponse)
|
||||
async def validate_credentials(request: CredentialValidationRequest):
|
||||
"""
|
||||
Validate that credentials are available for requested services.
|
||||
|
||||
Use this before submitting a job to ensure all required
|
||||
credentials are valid.
|
||||
"""
|
||||
manager = get_credential_manager()
|
||||
invalid_services = []
|
||||
|
||||
if request.youtube:
|
||||
result = manager.check_youtube_credentials()
|
||||
if result.status != CredentialStatus.VALID:
|
||||
invalid_services.append("youtube")
|
||||
|
||||
if request.gdrive:
|
||||
result = manager.check_gdrive_credentials()
|
||||
if result.status != CredentialStatus.VALID:
|
||||
invalid_services.append("gdrive")
|
||||
|
||||
if request.dropbox:
|
||||
result = manager.check_dropbox_credentials()
|
||||
if result.status != CredentialStatus.VALID:
|
||||
invalid_services.append("dropbox")
|
||||
|
||||
if invalid_services:
|
||||
return CredentialValidationResponse(
|
||||
valid=False,
|
||||
invalid_services=invalid_services,
|
||||
message=f"The following services need re-authorization: {', '.join(invalid_services)}"
|
||||
)
|
||||
|
||||
return CredentialValidationResponse(
|
||||
valid=True,
|
||||
invalid_services=[],
|
||||
message="All requested credentials are valid"
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Device Authorization Flow Endpoints
|
||||
# ===========================================================================
|
||||
|
||||
@router.post("/youtube/device", response_model=DeviceAuthStartResponse)
|
||||
async def start_youtube_device_auth(request: Optional[DeviceAuthStartRequest] = None):
|
||||
"""
|
||||
Start YouTube device authorization flow.
|
||||
|
||||
This initiates a device auth flow that allows authorization
|
||||
from any device. The user must visit the verification URL
|
||||
and enter the user code.
|
||||
|
||||
Client credentials are loaded from Secret Manager ('youtube-client-credentials')
|
||||
unless explicitly provided in the request body.
|
||||
|
||||
After starting, poll /auth/youtube/device/{device_code} to
|
||||
check for completion.
|
||||
"""
|
||||
manager = get_credential_manager()
|
||||
|
||||
try:
|
||||
device_info = manager.start_youtube_device_auth(
|
||||
client_id=request.client_id if request else None,
|
||||
client_secret=request.client_secret if request else None
|
||||
)
|
||||
|
||||
return DeviceAuthStartResponse(
|
||||
device_code=device_info.device_code,
|
||||
user_code=device_info.user_code,
|
||||
verification_url=device_info.verification_url,
|
||||
expires_in=device_info.expires_in,
|
||||
interval=device_info.interval,
|
||||
instructions=f"Visit {device_info.verification_url} and enter code: {device_info.user_code}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start YouTube device auth: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/youtube/device/{device_code}", response_model=DeviceAuthPollResponse)
|
||||
async def poll_youtube_device_auth(device_code: str):
|
||||
"""
|
||||
Poll for YouTube device authorization completion.
|
||||
|
||||
Call this endpoint at the interval specified in the device auth
|
||||
start response. Keep polling while status is "pending".
|
||||
|
||||
Status values:
|
||||
- pending: User has not yet authorized
|
||||
- complete: Authorization successful, credentials saved
|
||||
- expired: Device code expired, start new flow
|
||||
- error: An error occurred
|
||||
"""
|
||||
manager = get_credential_manager()
|
||||
|
||||
status, data = manager.poll_device_auth("youtube", device_code)
|
||||
|
||||
return DeviceAuthPollResponse(
|
||||
status=status,
|
||||
message=data.get("message") if data else None,
|
||||
credentials_saved=(status == "complete")
|
||||
)
|
||||
|
||||
|
||||
@router.post("/gdrive/device", response_model=DeviceAuthStartResponse)
|
||||
async def start_gdrive_device_auth(request: Optional[DeviceAuthStartRequest] = None):
|
||||
"""
|
||||
Start Google Drive device authorization flow.
|
||||
|
||||
Client credentials are loaded from Secret Manager ('gdrive-client-credentials')
|
||||
unless explicitly provided in the request body.
|
||||
|
||||
After starting, poll /auth/gdrive/device/{device_code} to check for completion.
|
||||
"""
|
||||
manager = get_credential_manager()
|
||||
|
||||
try:
|
||||
device_info = manager.start_gdrive_device_auth(
|
||||
client_id=request.client_id if request else None,
|
||||
client_secret=request.client_secret if request else None
|
||||
)
|
||||
|
||||
return DeviceAuthStartResponse(
|
||||
device_code=device_info.device_code,
|
||||
user_code=device_info.user_code,
|
||||
verification_url=device_info.verification_url,
|
||||
expires_in=device_info.expires_in,
|
||||
interval=device_info.interval,
|
||||
instructions=f"Visit {device_info.verification_url} and enter code: {device_info.user_code}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start Google Drive device auth: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/gdrive/device/{device_code}", response_model=DeviceAuthPollResponse)
|
||||
async def poll_gdrive_device_auth(device_code: str):
|
||||
"""
|
||||
Poll for Google Drive device authorization completion.
|
||||
"""
|
||||
manager = get_credential_manager()
|
||||
|
||||
status, data = manager.poll_device_auth("gdrive", device_code)
|
||||
|
||||
return DeviceAuthPollResponse(
|
||||
status=status,
|
||||
message=data.get("message") if data else None,
|
||||
credentials_saved=(status == "complete")
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Alert Testing
|
||||
# ===========================================================================
|
||||
|
||||
@router.post("/test-alert")
|
||||
async def test_credential_alert(background_tasks: BackgroundTasks):
|
||||
"""
|
||||
Test the credential alert mechanism.
|
||||
|
||||
Sends a test alert to Discord if configured.
|
||||
"""
|
||||
settings = get_settings()
|
||||
discord_url = settings.get_secret("discord-alert-webhook")
|
||||
|
||||
if not discord_url:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Discord alert webhook not configured"
|
||||
)
|
||||
|
||||
manager = get_credential_manager()
|
||||
|
||||
# Create a fake invalid result for testing
|
||||
from backend.services.credential_manager import CredentialCheckResult
|
||||
test_results = [
|
||||
CredentialCheckResult(
|
||||
service="test",
|
||||
status=CredentialStatus.INVALID,
|
||||
message="This is a test alert",
|
||||
last_checked=datetime.utcnow()
|
||||
)
|
||||
]
|
||||
|
||||
background_tasks.add_task(
|
||||
manager.send_credential_alert,
|
||||
test_results,
|
||||
discord_url
|
||||
)
|
||||
|
||||
return {"message": "Test alert queued"}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,26 @@
|
||||
"""
|
||||
Health check routes.
|
||||
"""
|
||||
from fastapi import APIRouter
|
||||
from typing import Dict
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
async def health_check() -> Dict[str, str]:
|
||||
"""Health check endpoint."""
|
||||
return {
|
||||
"status": "healthy",
|
||||
"service": "karaoke-gen-backend"
|
||||
}
|
||||
|
||||
|
||||
@router.get("/readiness")
|
||||
async def readiness_check() -> Dict[str, str]:
|
||||
"""Readiness check endpoint for Cloud Run."""
|
||||
return {
|
||||
"status": "ready",
|
||||
"service": "karaoke-gen-backend"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
"""
|
||||
Internal API routes for worker coordination.
|
||||
|
||||
These endpoints are for internal use only (backend → workers).
|
||||
They are protected by admin authentication.
|
||||
|
||||
With Cloud Tasks integration, these endpoints may be called multiple times
|
||||
(retry on failure). Idempotency checks prevent duplicate processing.
|
||||
"""
|
||||
import logging
|
||||
import asyncio
|
||||
from typing import Tuple, Optional
|
||||
from fastapi import APIRouter, HTTPException, BackgroundTasks, Depends
|
||||
from pydantic import BaseModel
|
||||
|
||||
from backend.workers.audio_worker import process_audio_separation
|
||||
from backend.workers.lyrics_worker import process_lyrics_transcription
|
||||
from backend.workers.screens_worker import generate_screens
|
||||
from backend.workers.video_worker import generate_video
|
||||
from backend.workers.render_video_worker import process_render_video
|
||||
from backend.api.dependencies import require_admin
|
||||
from backend.services.auth_service import UserType
|
||||
from backend.services.job_manager import JobManager
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/internal", tags=["internal"])
|
||||
|
||||
|
||||
class WorkerRequest(BaseModel):
|
||||
"""Request to trigger a worker."""
|
||||
job_id: str
|
||||
|
||||
|
||||
class WorkerResponse(BaseModel):
|
||||
"""Response from worker trigger."""
|
||||
status: str
|
||||
job_id: str
|
||||
message: str
|
||||
|
||||
|
||||
def _check_worker_idempotency(job_id: str, worker_name: str) -> Optional[WorkerResponse]:
|
||||
"""
|
||||
Check if a worker is already running or completed for this job.
|
||||
|
||||
This provides idempotency for Cloud Tasks retries - if a task is retried
|
||||
but the worker is already running or has completed, we skip processing.
|
||||
|
||||
Args:
|
||||
job_id: Job ID to check
|
||||
worker_name: Worker name (audio, lyrics, screens, render, video)
|
||||
|
||||
Returns:
|
||||
WorkerResponse if should skip (already running/complete), None to proceed
|
||||
"""
|
||||
job_manager = JobManager()
|
||||
job = job_manager.get_job(job_id)
|
||||
|
||||
if not job:
|
||||
logger.warning(f"Job {job_id} not found for {worker_name} worker")
|
||||
return WorkerResponse(
|
||||
status="not_found",
|
||||
job_id=job_id,
|
||||
message=f"Job {job_id} not found"
|
||||
)
|
||||
|
||||
# Check worker-specific progress in state_data
|
||||
progress_key = f"{worker_name}_progress"
|
||||
worker_progress = job.state_data.get(progress_key, {})
|
||||
stage = worker_progress.get('stage')
|
||||
|
||||
if stage == 'running':
|
||||
logger.info(f"{worker_name.capitalize()} worker already running for job {job_id}, skipping")
|
||||
return WorkerResponse(
|
||||
status="already_running",
|
||||
job_id=job_id,
|
||||
message=f"{worker_name.capitalize()} worker already in progress"
|
||||
)
|
||||
|
||||
if stage == 'complete':
|
||||
logger.info(f"{worker_name.capitalize()} worker already complete for job {job_id}, skipping")
|
||||
return WorkerResponse(
|
||||
status="already_complete",
|
||||
job_id=job_id,
|
||||
message=f"{worker_name.capitalize()} worker already completed"
|
||||
)
|
||||
|
||||
# Mark as running before starting (for idempotency on next retry)
|
||||
job_manager.update_state_data(job_id, progress_key, {'stage': 'running'})
|
||||
return None
|
||||
|
||||
|
||||
@router.post("/workers/audio", response_model=WorkerResponse)
|
||||
async def trigger_audio_worker(
|
||||
request: WorkerRequest,
|
||||
background_tasks: BackgroundTasks,
|
||||
auth_data: Tuple[str, UserType, int] = Depends(require_admin)
|
||||
):
|
||||
"""
|
||||
Trigger audio separation worker for a job.
|
||||
|
||||
This endpoint is called internally after job creation to start
|
||||
the audio processing track (parallel with lyrics processing).
|
||||
|
||||
Idempotency: If worker is already running or complete, returns early.
|
||||
|
||||
The worker runs in the background and updates job state as it progresses.
|
||||
"""
|
||||
job_id = request.job_id
|
||||
logger.info(f"Triggering audio worker for job {job_id}")
|
||||
|
||||
# Idempotency check
|
||||
skip_response = _check_worker_idempotency(job_id, "audio")
|
||||
if skip_response:
|
||||
return skip_response
|
||||
|
||||
# Add task to background tasks
|
||||
# This allows the HTTP response to return immediately
|
||||
# while the worker continues processing
|
||||
background_tasks.add_task(process_audio_separation, job_id)
|
||||
|
||||
return WorkerResponse(
|
||||
status="started",
|
||||
job_id=job_id,
|
||||
message="Audio separation worker started"
|
||||
)
|
||||
|
||||
|
||||
@router.post("/workers/lyrics", response_model=WorkerResponse)
|
||||
async def trigger_lyrics_worker(
|
||||
request: WorkerRequest,
|
||||
background_tasks: BackgroundTasks,
|
||||
auth_data: Tuple[str, UserType, int] = Depends(require_admin)
|
||||
):
|
||||
"""
|
||||
Trigger lyrics transcription worker for a job.
|
||||
|
||||
This endpoint is called internally after job creation to start
|
||||
the lyrics processing track (parallel with audio processing).
|
||||
|
||||
Idempotency: If worker is already running or complete, returns early.
|
||||
|
||||
The worker runs in the background and updates job state as it progresses.
|
||||
"""
|
||||
job_id = request.job_id
|
||||
logger.info(f"Triggering lyrics worker for job {job_id}")
|
||||
|
||||
# Idempotency check
|
||||
skip_response = _check_worker_idempotency(job_id, "lyrics")
|
||||
if skip_response:
|
||||
return skip_response
|
||||
|
||||
# Add task to background tasks
|
||||
background_tasks.add_task(process_lyrics_transcription, job_id)
|
||||
|
||||
return WorkerResponse(
|
||||
status="started",
|
||||
job_id=job_id,
|
||||
message="Lyrics transcription worker started"
|
||||
)
|
||||
|
||||
|
||||
@router.post("/workers/screens", response_model=WorkerResponse)
|
||||
async def trigger_screens_worker(
|
||||
request: WorkerRequest,
|
||||
background_tasks: BackgroundTasks,
|
||||
auth_data: Tuple[str, UserType, int] = Depends(require_admin)
|
||||
):
|
||||
"""
|
||||
Trigger title/end screen generation worker.
|
||||
|
||||
This is called automatically when both audio and lyrics are complete.
|
||||
|
||||
Idempotency: If worker is already running or complete, returns early.
|
||||
"""
|
||||
job_id = request.job_id
|
||||
logger.info(f"Triggering screens worker for job {job_id}")
|
||||
|
||||
# Idempotency check
|
||||
skip_response = _check_worker_idempotency(job_id, "screens")
|
||||
if skip_response:
|
||||
return skip_response
|
||||
|
||||
# Add task to background tasks
|
||||
background_tasks.add_task(generate_screens, job_id)
|
||||
|
||||
return WorkerResponse(
|
||||
status="started",
|
||||
job_id=job_id,
|
||||
message="Screens generation worker started"
|
||||
)
|
||||
|
||||
|
||||
@router.post("/workers/video", response_model=WorkerResponse)
|
||||
async def trigger_video_worker(
|
||||
request: WorkerRequest,
|
||||
background_tasks: BackgroundTasks,
|
||||
auth_data: Tuple[str, UserType, int] = Depends(require_admin)
|
||||
):
|
||||
"""
|
||||
Trigger final video generation and encoding worker.
|
||||
|
||||
This is called after user selects their preferred instrumental.
|
||||
This is the longest-running stage (15-20 minutes).
|
||||
|
||||
Idempotency: If worker is already running or complete, returns early.
|
||||
"""
|
||||
job_id = request.job_id
|
||||
logger.info(f"Triggering video worker for job {job_id}")
|
||||
|
||||
# Idempotency check
|
||||
skip_response = _check_worker_idempotency(job_id, "video")
|
||||
if skip_response:
|
||||
return skip_response
|
||||
|
||||
# Add task to background tasks
|
||||
background_tasks.add_task(generate_video, job_id)
|
||||
|
||||
return WorkerResponse(
|
||||
status="started",
|
||||
job_id=job_id,
|
||||
message="Video generation worker started"
|
||||
)
|
||||
|
||||
|
||||
@router.post("/workers/render-video", response_model=WorkerResponse)
|
||||
async def trigger_render_video_worker(
|
||||
request: WorkerRequest,
|
||||
background_tasks: BackgroundTasks,
|
||||
auth_data: Tuple[str, UserType, int] = Depends(require_admin)
|
||||
):
|
||||
"""
|
||||
Trigger render video worker (post-review).
|
||||
|
||||
This is called after human review is complete.
|
||||
Uses OutputGenerator from LyricsTranscriber to generate the karaoke video
|
||||
with the corrected lyrics.
|
||||
|
||||
Idempotency: If worker is already running or complete, returns early.
|
||||
|
||||
Output: with_vocals.mkv in GCS
|
||||
Next state: AWAITING_INSTRUMENTAL_SELECTION
|
||||
"""
|
||||
job_id = request.job_id
|
||||
logger.info(f"Triggering render-video worker for job {job_id}")
|
||||
|
||||
# Idempotency check
|
||||
skip_response = _check_worker_idempotency(job_id, "render")
|
||||
if skip_response:
|
||||
return skip_response
|
||||
|
||||
# Add task to background tasks
|
||||
background_tasks.add_task(process_render_video, job_id)
|
||||
|
||||
return WorkerResponse(
|
||||
status="started",
|
||||
job_id=job_id,
|
||||
message="Render video worker started (post-review)"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
async def internal_health(
|
||||
auth_data: Tuple[str, UserType, int] = Depends(require_admin)
|
||||
):
|
||||
"""
|
||||
Internal health check endpoint.
|
||||
|
||||
Used to verify the internal API is responsive.
|
||||
Requires admin authentication.
|
||||
"""
|
||||
return {"status": "healthy", "service": "karaoke-backend-internal"}
|
||||
|
||||
@@ -0,0 +1,914 @@
|
||||
"""
|
||||
Job management routes.
|
||||
|
||||
Handles job lifecycle endpoints including:
|
||||
- Job creation and submission
|
||||
- Status polling
|
||||
- Human-in-the-loop interactions (lyrics review, instrumental selection)
|
||||
- Job deletion and cancellation
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import httpx
|
||||
from typing import List, Optional, Dict, Any, Tuple
|
||||
from fastapi import APIRouter, HTTPException, BackgroundTasks, Depends
|
||||
|
||||
from backend.models.job import Job, JobCreate, JobResponse, JobStatus
|
||||
from backend.models.requests import (
|
||||
URLSubmissionRequest,
|
||||
CorrectionsSubmission,
|
||||
InstrumentalSelection,
|
||||
StartReviewRequest,
|
||||
CancelJobRequest
|
||||
)
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.worker_service import get_worker_service
|
||||
from backend.services.storage_service import StorageService
|
||||
from backend.config import get_settings
|
||||
from backend.api.dependencies import require_admin
|
||||
from backend.services.auth_service import UserType
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/jobs", tags=["jobs"])
|
||||
|
||||
# Initialize services
|
||||
job_manager = JobManager()
|
||||
worker_service = get_worker_service()
|
||||
settings = get_settings()
|
||||
|
||||
|
||||
async def _trigger_workers_parallel(job_id: str) -> None:
|
||||
"""
|
||||
Trigger both audio and lyrics workers in parallel.
|
||||
|
||||
FastAPI's BackgroundTasks runs async tasks sequentially, so we use
|
||||
asyncio.gather to ensure both workers start at the same time.
|
||||
"""
|
||||
await asyncio.gather(
|
||||
worker_service.trigger_audio_worker(job_id),
|
||||
worker_service.trigger_lyrics_worker(job_id)
|
||||
)
|
||||
|
||||
|
||||
@router.post("", response_model=JobResponse)
|
||||
async def create_job(
|
||||
request: URLSubmissionRequest,
|
||||
background_tasks: BackgroundTasks
|
||||
) -> JobResponse:
|
||||
"""
|
||||
Create a new karaoke generation job from a URL.
|
||||
|
||||
This triggers the complete workflow:
|
||||
1. Job created in PENDING state
|
||||
2. Audio and lyrics workers triggered in parallel
|
||||
3. Both workers update job state as they progress
|
||||
4. When both complete, job transitions to AWAITING_REVIEW
|
||||
"""
|
||||
try:
|
||||
# Create job with all preferences
|
||||
job_create = JobCreate(
|
||||
url=str(request.url),
|
||||
artist=request.artist,
|
||||
title=request.title,
|
||||
enable_cdg=request.enable_cdg,
|
||||
enable_txt=request.enable_txt,
|
||||
enable_youtube_upload=request.enable_youtube_upload,
|
||||
youtube_description=request.youtube_description,
|
||||
webhook_url=request.webhook_url,
|
||||
user_email=request.user_email
|
||||
)
|
||||
job = job_manager.create_job(job_create)
|
||||
|
||||
# Trigger both workers in parallel using asyncio.gather
|
||||
# (FastAPI's BackgroundTasks runs async tasks sequentially)
|
||||
background_tasks.add_task(_trigger_workers_parallel, job.job_id)
|
||||
|
||||
logger.info(f"Job {job.job_id} created, workers triggered")
|
||||
|
||||
return JobResponse(
|
||||
status="success",
|
||||
job_id=job.job_id,
|
||||
message="Job created successfully. Processing started."
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating job: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
# Worker triggering is now handled by WorkerService
|
||||
# See backend/services/worker_service.py
|
||||
|
||||
|
||||
@router.get("/{job_id}", response_model=Job)
|
||||
async def get_job(job_id: str) -> Job:
|
||||
"""Get job status and details."""
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
# If job is complete, include download URLs
|
||||
if job.status == JobStatus.COMPLETE:
|
||||
job.download_urls = job_manager.get_output_urls(job_id)
|
||||
|
||||
return job
|
||||
|
||||
|
||||
@router.get("", response_model=List[Job])
|
||||
async def list_jobs(
|
||||
status: Optional[JobStatus] = None,
|
||||
environment: Optional[str] = None,
|
||||
client_id: Optional[str] = None,
|
||||
created_after: Optional[str] = None,
|
||||
created_before: Optional[str] = None,
|
||||
limit: int = 100
|
||||
) -> List[Job]:
|
||||
"""
|
||||
List jobs with optional filters.
|
||||
|
||||
Args:
|
||||
status: Filter by job status (pending, complete, failed, etc.)
|
||||
environment: Filter by request_metadata.environment (test/production/development)
|
||||
client_id: Filter by request_metadata.client_id (customer identifier)
|
||||
created_after: Filter jobs created after this ISO datetime (e.g., 2024-01-01T00:00:00Z)
|
||||
created_before: Filter jobs created before this ISO datetime
|
||||
limit: Maximum number of jobs to return (default 100)
|
||||
|
||||
Returns:
|
||||
List of jobs matching filters, ordered by created_at descending
|
||||
"""
|
||||
from datetime import datetime
|
||||
|
||||
try:
|
||||
# Parse datetime strings if provided
|
||||
created_after_dt = None
|
||||
created_before_dt = None
|
||||
|
||||
if created_after:
|
||||
try:
|
||||
created_after_dt = datetime.fromisoformat(created_after.replace('Z', '+00:00'))
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid created_after format: {created_after}")
|
||||
|
||||
if created_before:
|
||||
try:
|
||||
created_before_dt = datetime.fromisoformat(created_before.replace('Z', '+00:00'))
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid created_before format: {created_before}")
|
||||
|
||||
jobs = job_manager.list_jobs(
|
||||
status=status,
|
||||
environment=environment,
|
||||
client_id=client_id,
|
||||
created_after=created_after_dt,
|
||||
created_before=created_before_dt,
|
||||
limit=limit
|
||||
)
|
||||
return jobs
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error listing jobs: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.delete("/{job_id}")
|
||||
async def delete_job(job_id: str, delete_files: bool = True) -> dict:
|
||||
"""Delete a job and optionally its output files."""
|
||||
try:
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
job_manager.delete_job(job_id, delete_files=delete_files)
|
||||
|
||||
return {"status": "success", "message": f"Job {job_id} deleted"}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error deleting job {job_id}: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.delete("")
|
||||
async def bulk_delete_jobs(
|
||||
environment: Optional[str] = None,
|
||||
client_id: Optional[str] = None,
|
||||
status: Optional[JobStatus] = None,
|
||||
created_before: Optional[str] = None,
|
||||
delete_files: bool = True,
|
||||
confirm: bool = False,
|
||||
auth_data: Tuple[str, UserType, int] = Depends(require_admin)
|
||||
) -> dict:
|
||||
"""
|
||||
Delete multiple jobs matching filter criteria.
|
||||
|
||||
CAUTION: This is a destructive operation. Requires confirm=true.
|
||||
|
||||
Use cases:
|
||||
- Delete all test jobs: ?environment=test&confirm=true
|
||||
- Delete jobs from a specific client: ?client_id=test-runner&confirm=true
|
||||
- Delete old failed jobs: ?status=failed&created_before=2024-01-01T00:00:00Z&confirm=true
|
||||
|
||||
Args:
|
||||
environment: Delete jobs with this environment (test/production/development)
|
||||
client_id: Delete jobs from this client
|
||||
status: Delete jobs with this status
|
||||
created_before: Delete jobs created before this ISO datetime
|
||||
delete_files: Also delete GCS files (default True)
|
||||
confirm: Must be True to execute deletion (safety check)
|
||||
|
||||
Returns:
|
||||
Statistics about the deletion
|
||||
"""
|
||||
from datetime import datetime
|
||||
|
||||
# Require at least one filter to prevent accidental deletion of all jobs
|
||||
if not any([environment, client_id, status, created_before]):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="At least one filter (environment, client_id, status, created_before) is required"
|
||||
)
|
||||
|
||||
# Require explicit confirmation
|
||||
if not confirm:
|
||||
# Return preview of what would be deleted
|
||||
created_before_dt = None
|
||||
if created_before:
|
||||
try:
|
||||
created_before_dt = datetime.fromisoformat(created_before.replace('Z', '+00:00'))
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid created_before format: {created_before}")
|
||||
|
||||
jobs = job_manager.list_jobs(
|
||||
status=status,
|
||||
environment=environment,
|
||||
client_id=client_id,
|
||||
created_before=created_before_dt,
|
||||
limit=1000
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "preview",
|
||||
"message": "Add &confirm=true to execute deletion",
|
||||
"jobs_to_delete": len(jobs),
|
||||
"sample_jobs": [
|
||||
{
|
||||
"job_id": j.job_id,
|
||||
"artist": j.artist,
|
||||
"title": j.title,
|
||||
"status": j.status,
|
||||
"environment": j.request_metadata.get('environment'),
|
||||
"client_id": j.request_metadata.get('client_id'),
|
||||
"created_at": j.created_at.isoformat() if j.created_at else None
|
||||
}
|
||||
for j in jobs[:10] # Show first 10 as sample
|
||||
]
|
||||
}
|
||||
|
||||
try:
|
||||
# Parse datetime string
|
||||
created_before_dt = None
|
||||
if created_before:
|
||||
try:
|
||||
created_before_dt = datetime.fromisoformat(created_before.replace('Z', '+00:00'))
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid created_before format: {created_before}")
|
||||
|
||||
result = job_manager.delete_jobs_by_filter(
|
||||
environment=environment,
|
||||
client_id=client_id,
|
||||
status=status,
|
||||
created_before=created_before_dt,
|
||||
delete_files=delete_files
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Deleted {result['jobs_deleted']} jobs",
|
||||
"jobs_deleted": result['jobs_deleted'],
|
||||
"files_deleted": result.get('files_deleted', 0)
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error bulk deleting jobs: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Human-in-the-Loop Interaction Endpoints
|
||||
# ============================================================================
|
||||
|
||||
@router.get("/{job_id}/review-data")
|
||||
async def get_review_data(job_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Get data needed for lyrics review interface.
|
||||
|
||||
Returns corrections JSON URL and audio URL.
|
||||
Frontend loads these to render the review UI.
|
||||
"""
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
if job.status not in [JobStatus.AWAITING_REVIEW, JobStatus.IN_REVIEW]:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Job not ready for review (current status: {job.status})"
|
||||
)
|
||||
|
||||
# Get URLs from file_urls
|
||||
corrections_url = job.file_urls.get('lyrics', {}).get('corrections')
|
||||
audio_url = job.file_urls.get('lyrics', {}).get('audio')
|
||||
|
||||
if not corrections_url or not audio_url:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="Review data not available"
|
||||
)
|
||||
|
||||
# Generate signed URLs for direct access
|
||||
from backend.services.storage_service import StorageService
|
||||
storage = StorageService()
|
||||
|
||||
return {
|
||||
"corrections_url": storage.generate_signed_url(corrections_url, expiration_minutes=120),
|
||||
"audio_url": storage.generate_signed_url(audio_url, expiration_minutes=120),
|
||||
"status": job.status,
|
||||
"artist": job.artist,
|
||||
"title": job.title
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{job_id}/start-review")
|
||||
async def start_review(job_id: str, request: StartReviewRequest) -> dict:
|
||||
"""
|
||||
Mark job as IN_REVIEW (user opened review interface).
|
||||
|
||||
This helps track that the user is actively working on the review.
|
||||
"""
|
||||
success = job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.IN_REVIEW,
|
||||
message="User started reviewing lyrics"
|
||||
)
|
||||
|
||||
if not success:
|
||||
raise HTTPException(status_code=400, detail="Cannot start review")
|
||||
|
||||
return {"status": "success", "job_status": "in_review"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/corrections")
|
||||
async def submit_corrections(
|
||||
job_id: str,
|
||||
submission: CorrectionsSubmission,
|
||||
background_tasks: BackgroundTasks
|
||||
) -> dict:
|
||||
"""
|
||||
Save corrected lyrics during human review.
|
||||
|
||||
This endpoint saves review progress but does NOT complete the review.
|
||||
Call POST /{job_id}/complete-review to finish and trigger video rendering.
|
||||
|
||||
Can be called multiple times to save progress.
|
||||
"""
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
if job.status not in [JobStatus.AWAITING_REVIEW, JobStatus.IN_REVIEW]:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Job not in review state (current status: {job.status})"
|
||||
)
|
||||
|
||||
try:
|
||||
# Store corrected lyrics in state_data
|
||||
job_manager.update_state_data(job_id, 'corrected_lyrics', submission.corrections)
|
||||
if submission.user_notes:
|
||||
job_manager.update_state_data(job_id, 'review_notes', submission.user_notes)
|
||||
|
||||
# Transition to IN_REVIEW if not already
|
||||
if job.status == JobStatus.AWAITING_REVIEW:
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.IN_REVIEW,
|
||||
message="User is reviewing lyrics"
|
||||
)
|
||||
|
||||
# Save updated corrections to GCS for the render worker
|
||||
from backend.services.storage_service import StorageService
|
||||
storage = StorageService()
|
||||
|
||||
corrections_gcs_path = f"jobs/{job_id}/lyrics/corrections_updated.json"
|
||||
storage.upload_json(corrections_gcs_path, submission.corrections)
|
||||
job_manager.update_file_url(job_id, 'lyrics', 'corrections_updated', corrections_gcs_path)
|
||||
|
||||
logger.info(f"Job {job_id}: Corrections saved (review in progress)")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"job_status": "in_review",
|
||||
"message": "Corrections saved. Call /complete-review when done."
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error saving corrections for job {job_id}: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/{job_id}/complete-review")
|
||||
async def complete_review(
|
||||
job_id: str,
|
||||
background_tasks: BackgroundTasks
|
||||
) -> dict:
|
||||
"""
|
||||
Complete the human review and trigger video rendering.
|
||||
|
||||
This is the FIRST critical human-in-the-loop completion point.
|
||||
After this:
|
||||
1. Job transitions to REVIEW_COMPLETE
|
||||
2. Render video worker is triggered
|
||||
3. Worker uses OutputGenerator to create with_vocals.mkv
|
||||
4. Job transitions to AWAITING_INSTRUMENTAL_SELECTION
|
||||
"""
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
if job.status not in [JobStatus.AWAITING_REVIEW, JobStatus.IN_REVIEW]:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Job not in review state (current status: {job.status})"
|
||||
)
|
||||
|
||||
try:
|
||||
# Transition to REVIEW_COMPLETE
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.REVIEW_COMPLETE,
|
||||
progress=70,
|
||||
message="Review complete, rendering video with corrected lyrics"
|
||||
)
|
||||
|
||||
# Trigger render video worker
|
||||
background_tasks.add_task(worker_service.trigger_render_video_worker, job_id)
|
||||
|
||||
logger.info(f"Job {job_id}: Review complete, triggering render video worker")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"job_status": "review_complete",
|
||||
"message": "Review complete. Video rendering started."
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error completing review for job {job_id}: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/{job_id}/instrumental-options")
|
||||
async def get_instrumental_options(job_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Get instrumental audio options for user selection.
|
||||
|
||||
Returns signed URLs for both options:
|
||||
1. Clean instrumental (no backing vocals)
|
||||
2. Instrumental with backing vocals
|
||||
"""
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
if job.status != JobStatus.AWAITING_INSTRUMENTAL_SELECTION:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Job not ready for instrumental selection (current status: {job.status})"
|
||||
)
|
||||
|
||||
# Get stem URLs
|
||||
stems = job.file_urls.get('stems', {})
|
||||
clean_url = stems.get('instrumental_clean')
|
||||
backing_url = stems.get('instrumental_with_backing')
|
||||
|
||||
if not clean_url or not backing_url:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="Instrumental options not available"
|
||||
)
|
||||
|
||||
# Generate signed URLs
|
||||
from backend.services.storage_service import StorageService
|
||||
storage = StorageService()
|
||||
|
||||
return {
|
||||
"options": [
|
||||
{
|
||||
"id": "clean",
|
||||
"label": "Clean Instrumental (no backing vocals)",
|
||||
"audio_url": storage.generate_signed_url(clean_url, expiration_minutes=120),
|
||||
"duration_seconds": None # TODO: Extract from audio file
|
||||
},
|
||||
{
|
||||
"id": "with_backing",
|
||||
"label": "Instrumental with Backing Vocals",
|
||||
"audio_url": storage.generate_signed_url(backing_url, expiration_minutes=120),
|
||||
"duration_seconds": None # TODO: Extract from audio file
|
||||
}
|
||||
],
|
||||
"status": job.status,
|
||||
"artist": job.artist,
|
||||
"title": job.title
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{job_id}/select-instrumental")
|
||||
async def select_instrumental(
|
||||
job_id: str,
|
||||
selection: InstrumentalSelection,
|
||||
background_tasks: BackgroundTasks
|
||||
) -> dict:
|
||||
"""
|
||||
Submit instrumental selection.
|
||||
|
||||
This is the SECOND critical human-in-the-loop interaction point.
|
||||
After selection, the job proceeds to video generation.
|
||||
"""
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
if job.status != JobStatus.AWAITING_INSTRUMENTAL_SELECTION:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Job not ready for instrumental selection (current status: {job.status})"
|
||||
)
|
||||
|
||||
try:
|
||||
# Store selection in state_data
|
||||
job_manager.update_state_data(job_id, 'instrumental_selection', selection.selection)
|
||||
|
||||
# Transition to INSTRUMENTAL_SELECTED
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.INSTRUMENTAL_SELECTED,
|
||||
progress=65,
|
||||
message=f"Instrumental selected: {selection.selection}"
|
||||
)
|
||||
|
||||
# Trigger video generation worker
|
||||
background_tasks.add_task(worker_service.trigger_video_worker, job_id)
|
||||
|
||||
logger.info(f"Job {job_id}: Instrumental selected ({selection.selection}), triggering video generation")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"job_status": "instrumental_selected",
|
||||
"selection": selection.selection,
|
||||
"message": "Selection accepted, starting video generation"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error selecting instrumental for job {job_id}: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/{job_id}/download-urls")
|
||||
async def get_download_urls(job_id: str) -> dict:
|
||||
"""
|
||||
Get download URLs for all job output files.
|
||||
|
||||
Returns a dictionary mapping file types to download URLs.
|
||||
Uses the streaming download endpoint which proxies through the backend.
|
||||
"""
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
if job.status != JobStatus.COMPLETE:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Job not complete (current status: {job.status})"
|
||||
)
|
||||
|
||||
file_urls = job.file_urls or {}
|
||||
download_urls = {}
|
||||
|
||||
# Build download URLs using the streaming endpoint
|
||||
base_url = f"/api/jobs/{job_id}/download"
|
||||
|
||||
for category, files in file_urls.items():
|
||||
if isinstance(files, dict):
|
||||
download_urls[category] = {}
|
||||
for file_key, gcs_path in files.items():
|
||||
if gcs_path:
|
||||
download_urls[category][file_key] = f"{base_url}/{category}/{file_key}"
|
||||
elif isinstance(files, str) and files:
|
||||
# For single-file categories, use the category name as the file_key
|
||||
download_urls[category] = f"{base_url}/{category}/{category}"
|
||||
|
||||
return {
|
||||
"job_id": job_id,
|
||||
"artist": job.artist,
|
||||
"title": job.title,
|
||||
"download_urls": download_urls
|
||||
}
|
||||
|
||||
|
||||
@router.get("/{job_id}/download/{category}/{file_key}")
|
||||
async def download_file(job_id: str, category: str, file_key: str):
|
||||
"""
|
||||
Stream download a specific file from a completed job.
|
||||
|
||||
This endpoint proxies the file from GCS through the backend,
|
||||
so no client-side authentication is required.
|
||||
"""
|
||||
from fastapi.responses import StreamingResponse
|
||||
import tempfile
|
||||
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
if job.status != JobStatus.COMPLETE:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Job not complete (current status: {job.status})"
|
||||
)
|
||||
|
||||
file_urls = job.file_urls or {}
|
||||
category_files = file_urls.get(category)
|
||||
|
||||
if not category_files:
|
||||
raise HTTPException(status_code=404, detail=f"Category '{category}' not found")
|
||||
|
||||
if isinstance(category_files, dict):
|
||||
gcs_path = category_files.get(file_key)
|
||||
else:
|
||||
gcs_path = category_files if file_key == category else None
|
||||
|
||||
if not gcs_path:
|
||||
raise HTTPException(status_code=404, detail=f"File '{file_key}' not found in '{category}'")
|
||||
|
||||
# Determine content type based on file extension
|
||||
ext = gcs_path.split('.')[-1].lower()
|
||||
content_types = {
|
||||
'mp4': 'video/mp4',
|
||||
'mkv': 'video/x-matroska',
|
||||
'mov': 'video/quicktime',
|
||||
'flac': 'audio/flac',
|
||||
'mp3': 'audio/mpeg',
|
||||
'wav': 'audio/wav',
|
||||
'ass': 'text/plain',
|
||||
'lrc': 'text/plain',
|
||||
'txt': 'text/plain',
|
||||
'json': 'application/json',
|
||||
'zip': 'application/zip',
|
||||
'png': 'image/png',
|
||||
'jpg': 'image/jpeg',
|
||||
'jpeg': 'image/jpeg',
|
||||
}
|
||||
content_type = content_types.get(ext, 'application/octet-stream')
|
||||
|
||||
# Get filename for Content-Disposition
|
||||
filename = gcs_path.split('/')[-1]
|
||||
|
||||
try:
|
||||
# Download to temp file and stream
|
||||
storage = StorageService()
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=f'.{ext}') as tmp:
|
||||
tmp_path = tmp.name
|
||||
|
||||
storage.download_file(gcs_path, tmp_path)
|
||||
|
||||
def file_iterator():
|
||||
try:
|
||||
with open(tmp_path, 'rb') as f:
|
||||
while chunk := f.read(8192):
|
||||
yield chunk
|
||||
finally:
|
||||
import os
|
||||
os.unlink(tmp_path)
|
||||
|
||||
return StreamingResponse(
|
||||
file_iterator(),
|
||||
media_type=content_type,
|
||||
headers={
|
||||
'Content-Disposition': f'attachment; filename="{filename}"'
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error downloading {gcs_path}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Error downloading file: {e}")
|
||||
|
||||
|
||||
@router.post("/{job_id}/cancel")
|
||||
async def cancel_job(job_id: str, request: CancelJobRequest) -> dict:
|
||||
"""
|
||||
Cancel a job.
|
||||
|
||||
Jobs can be cancelled at any stage before completion.
|
||||
"""
|
||||
success = job_manager.cancel_job(job_id, reason=request.reason)
|
||||
|
||||
if not success:
|
||||
raise HTTPException(status_code=400, detail="Cannot cancel job")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"job_status": "cancelled",
|
||||
"message": "Job cancelled successfully"
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{job_id}/retry")
|
||||
async def retry_job(
|
||||
job_id: str,
|
||||
background_tasks: BackgroundTasks
|
||||
) -> dict:
|
||||
"""
|
||||
Retry a failed job from the last successful checkpoint.
|
||||
|
||||
This endpoint allows resuming jobs that failed during:
|
||||
- Video generation (re-runs video worker)
|
||||
- Encoding (re-runs video worker)
|
||||
- Packaging (re-runs video worker)
|
||||
|
||||
The retry logic determines the appropriate stage to resume from
|
||||
based on what files/state already exist.
|
||||
"""
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
if job.status != JobStatus.FAILED:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Only failed jobs can be retried (current status: {job.status})"
|
||||
)
|
||||
|
||||
try:
|
||||
# Determine retry point based on what's already complete
|
||||
error_details = job.error_details or {}
|
||||
error_stage = error_details.get('stage', 'unknown')
|
||||
|
||||
logger.info(f"Job {job_id}: Retrying from failed stage '{error_stage}'")
|
||||
|
||||
# Check what state exists to determine retry point
|
||||
file_urls = job.file_urls or {}
|
||||
state_data = job.state_data or {}
|
||||
|
||||
# If we have a video with vocals and instrumental selection, retry video generation
|
||||
if (file_urls.get('videos', {}).get('with_vocals') and
|
||||
state_data.get('instrumental_selection')):
|
||||
|
||||
logger.info(f"Job {job_id}: Has rendered video and instrumental selection, retrying video generation")
|
||||
|
||||
# Clear error state
|
||||
job_manager.update_job(job_id, {
|
||||
'error_message': None,
|
||||
'error_details': None,
|
||||
})
|
||||
|
||||
# Reset to INSTRUMENTAL_SELECTED and trigger video worker
|
||||
if not job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.INSTRUMENTAL_SELECTED,
|
||||
progress=65,
|
||||
message=f"Retrying video generation from failed state"
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="Failed to transition job status for retry"
|
||||
)
|
||||
|
||||
# Trigger video generation worker
|
||||
background_tasks.add_task(worker_service.trigger_video_worker, job_id)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"job_status": "instrumental_selected",
|
||||
"message": "Job retry started from video generation stage",
|
||||
"retry_stage": "video_generation"
|
||||
}
|
||||
|
||||
# If we have corrections and screens but no video, retry render
|
||||
elif (file_urls.get('lyrics', {}).get('corrections') and
|
||||
file_urls.get('screens', {}).get('title')):
|
||||
|
||||
logger.info(f"Job {job_id}: Has corrections and screens, retrying from render stage")
|
||||
|
||||
# Clear error state
|
||||
job_manager.update_job(job_id, {
|
||||
'error_message': None,
|
||||
'error_details': None,
|
||||
})
|
||||
|
||||
# Reset to REVIEW_COMPLETE and trigger render worker
|
||||
if not job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.REVIEW_COMPLETE,
|
||||
progress=70,
|
||||
message=f"Retrying video render from failed state"
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="Failed to transition job status for retry"
|
||||
)
|
||||
|
||||
# Trigger render video worker
|
||||
background_tasks.add_task(worker_service.trigger_render_video_worker, job_id)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"job_status": "review_complete",
|
||||
"message": "Job retry started from render stage",
|
||||
"retry_stage": "render_video"
|
||||
}
|
||||
|
||||
# If we have stems and corrections, retry from screens generation
|
||||
elif (file_urls.get('stems', {}).get('instrumental_clean') and
|
||||
file_urls.get('lyrics', {}).get('corrections')):
|
||||
|
||||
logger.info(f"Job {job_id}: Has stems and corrections, retrying from screens stage")
|
||||
|
||||
# Clear error state
|
||||
job_manager.update_job(job_id, {
|
||||
'error_message': None,
|
||||
'error_details': None,
|
||||
})
|
||||
|
||||
# Reset to a state before screens and trigger screens worker
|
||||
if not job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.LYRICS_COMPLETE,
|
||||
progress=45,
|
||||
message=f"Retrying from screens generation"
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="Failed to transition job status for retry"
|
||||
)
|
||||
|
||||
# Trigger screens worker
|
||||
background_tasks.add_task(worker_service.trigger_screens_worker, job_id)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"job_status": "lyrics_complete",
|
||||
"message": "Job retry started from screens generation",
|
||||
"retry_stage": "screens_generation"
|
||||
}
|
||||
|
||||
else:
|
||||
# Can't determine a safe retry point, need to restart from beginning
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Cannot determine safe retry point. Job may need to be resubmitted."
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error retrying job {job_id}: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/{job_id}/logs")
|
||||
async def get_worker_logs(
|
||||
job_id: str,
|
||||
since_index: int = 0,
|
||||
worker: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Get worker logs for debugging.
|
||||
|
||||
This endpoint returns worker logs stored in Firestore.
|
||||
Use `since_index` for efficient polling (returns only new logs).
|
||||
|
||||
Args:
|
||||
job_id: Job ID
|
||||
since_index: Return only logs after this index (for pagination/polling)
|
||||
worker: Filter by worker name (audio, lyrics, screens, video, render)
|
||||
|
||||
Returns:
|
||||
{
|
||||
"logs": [{"timestamp": "...", "level": "INFO", "worker": "audio", "message": "..."}],
|
||||
"next_index": 42, # Use this for next poll
|
||||
"total_logs": 42
|
||||
}
|
||||
"""
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
logs = job_manager.get_worker_logs(job_id, since_index=since_index, worker=worker)
|
||||
total = len(job.worker_logs) if job.worker_logs else 0
|
||||
|
||||
return {
|
||||
"logs": logs,
|
||||
"next_index": total,
|
||||
"total_logs": total
|
||||
}
|
||||
|
||||
@@ -0,0 +1,531 @@
|
||||
"""
|
||||
Review API routes - Compatible with LyricsTranscriber frontend.
|
||||
|
||||
These endpoints match the API that the LyricsTranscriber review frontend expects,
|
||||
allowing us to use the existing React review UI with our cloud backend.
|
||||
|
||||
Usage:
|
||||
Frontend URL: http://localhost:5173/?baseApiUrl=http://localhost:8000/api/review/{job_id}
|
||||
|
||||
The baseApiUrl includes the job_id, and all endpoints are relative to that.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Set
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.responses import FileResponse, StreamingResponse
|
||||
from starlette.background import BackgroundTask
|
||||
|
||||
from backend.models.job import JobStatus
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.storage_service import StorageService
|
||||
from backend.services.job_logging import job_log_context, JobLogger
|
||||
from backend.services.tracing import create_span, add_span_attribute, add_span_event
|
||||
|
||||
# LyricsTranscriber imports for preview generation
|
||||
from lyrics_transcriber.types import CorrectionResult
|
||||
from lyrics_transcriber.core.config import OutputConfig
|
||||
from lyrics_transcriber.correction.operations import CorrectionOperations
|
||||
|
||||
# Import from the unified style loader
|
||||
from karaoke_gen.style_loader import load_styles_from_gcs
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/review", tags=["review"])
|
||||
|
||||
# Store job context for the session
|
||||
# In production, this would be handled differently (e.g., session tokens)
|
||||
_job_contexts: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
# Store preview video paths for serving
|
||||
_preview_videos: Dict[str, Dict[str, str]] = {}
|
||||
|
||||
# Keep references to background tasks to prevent garbage collection
|
||||
_background_tasks: Set[asyncio.Task] = set()
|
||||
|
||||
|
||||
def _get_audio_hash(job_id: str) -> str:
|
||||
"""Generate a consistent audio hash for a job."""
|
||||
return hashlib.md5(job_id.encode()).hexdigest()
|
||||
|
||||
|
||||
@router.get("/{job_id}/ping")
|
||||
async def ping(job_id: str):
|
||||
"""Health check endpoint expected by frontend."""
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.get("/{job_id}/correction-data")
|
||||
async def get_correction_data(job_id: str):
|
||||
"""
|
||||
Get correction data for the review interface.
|
||||
|
||||
Returns the CorrectionResult data that the frontend needs to render
|
||||
the lyrics review UI.
|
||||
"""
|
||||
job_manager = JobManager()
|
||||
storage = StorageService()
|
||||
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
if job.status not in [JobStatus.AWAITING_REVIEW, JobStatus.IN_REVIEW]:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Job not ready for review (current status: {job.status})"
|
||||
)
|
||||
|
||||
# Get corrections URL from file_urls
|
||||
corrections_gcs = job.file_urls.get('lyrics', {}).get('corrections')
|
||||
if not corrections_gcs:
|
||||
# Try direct path
|
||||
corrections_gcs = f"jobs/{job_id}/lyrics/corrections.json"
|
||||
if not storage.file_exists(corrections_gcs):
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="Corrections data not found. Lyrics processing may not be complete."
|
||||
)
|
||||
|
||||
# Download and return corrections data
|
||||
try:
|
||||
corrections_data = storage.download_json(corrections_gcs)
|
||||
|
||||
# Add audio hash for the frontend
|
||||
audio_hash = _get_audio_hash(job_id)
|
||||
if 'metadata' not in corrections_data:
|
||||
corrections_data['metadata'] = {}
|
||||
corrections_data['metadata']['audio_hash'] = audio_hash
|
||||
corrections_data['metadata']['artist'] = job.artist
|
||||
corrections_data['metadata']['title'] = job.title
|
||||
|
||||
# Store context for audio serving
|
||||
_job_contexts[job_id] = {
|
||||
'audio_hash': audio_hash,
|
||||
'audio_gcs_path': job.input_media_gcs_path
|
||||
}
|
||||
|
||||
# Transition to IN_REVIEW if not already
|
||||
if job.status == JobStatus.AWAITING_REVIEW:
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.IN_REVIEW,
|
||||
message="User opened review interface"
|
||||
)
|
||||
|
||||
logger.info(f"Job {job_id}: Serving correction data for review")
|
||||
return corrections_data
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Job {job_id}: Error loading corrections: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Error loading corrections: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/{job_id}/audio/{audio_hash}")
|
||||
async def get_audio_with_hash(job_id: str, audio_hash: str):
|
||||
"""Stream the audio file for playback (with hash parameter)."""
|
||||
return await _stream_audio(job_id)
|
||||
|
||||
|
||||
@router.get("/{job_id}/audio/")
|
||||
@router.get("/{job_id}/audio")
|
||||
async def get_audio_no_hash(job_id: str):
|
||||
"""Stream the audio file for playback (without hash parameter)."""
|
||||
return await _stream_audio(job_id)
|
||||
|
||||
|
||||
async def _stream_audio(job_id: str):
|
||||
"""
|
||||
Stream the audio file for playback in the review interface.
|
||||
"""
|
||||
job_manager = JobManager()
|
||||
storage = StorageService()
|
||||
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
audio_gcs_path = job.input_media_gcs_path
|
||||
if not audio_gcs_path:
|
||||
raise HTTPException(status_code=404, detail="Audio file not found")
|
||||
|
||||
# Download to temp file and stream
|
||||
try:
|
||||
# Create temp file
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".flac") as tmp:
|
||||
tmp_path = tmp.name
|
||||
|
||||
storage.download_file(audio_gcs_path, tmp_path)
|
||||
|
||||
# Determine content type
|
||||
if audio_gcs_path.endswith('.flac'):
|
||||
media_type = "audio/flac"
|
||||
elif audio_gcs_path.endswith('.wav'):
|
||||
media_type = "audio/wav"
|
||||
elif audio_gcs_path.endswith('.mp3'):
|
||||
media_type = "audio/mpeg"
|
||||
else:
|
||||
media_type = "audio/mpeg"
|
||||
|
||||
logger.info(f"Job {job_id}: Streaming audio for review")
|
||||
|
||||
return FileResponse(
|
||||
tmp_path,
|
||||
media_type=media_type,
|
||||
filename=os.path.basename(audio_gcs_path),
|
||||
background=BackgroundTask(os.unlink, tmp_path),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Job {job_id}: Error streaming audio: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Error streaming audio: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/{job_id}/complete")
|
||||
async def complete_review(job_id: str, updated_data: Dict[str, Any]):
|
||||
"""
|
||||
Complete the review and save corrected lyrics.
|
||||
|
||||
This endpoint receives the updated correction data from the frontend
|
||||
and saves it, then triggers the render video worker.
|
||||
"""
|
||||
job_manager = JobManager()
|
||||
storage = StorageService()
|
||||
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
if job.status not in [JobStatus.AWAITING_REVIEW, JobStatus.IN_REVIEW]:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Job not in review state (current status: {job.status})"
|
||||
)
|
||||
|
||||
try:
|
||||
# Save updated corrections to GCS
|
||||
corrections_gcs_path = f"jobs/{job_id}/lyrics/corrections_updated.json"
|
||||
storage.upload_json(corrections_gcs_path, updated_data)
|
||||
job_manager.update_file_url(job_id, 'lyrics', 'corrections_updated', corrections_gcs_path)
|
||||
|
||||
logger.info(f"Job {job_id}: Saved updated corrections")
|
||||
|
||||
# Transition to REVIEW_COMPLETE
|
||||
job_manager.transition_to_state(
|
||||
job_id=job_id,
|
||||
new_status=JobStatus.REVIEW_COMPLETE,
|
||||
progress=70,
|
||||
message="Review complete, rendering video with corrected lyrics"
|
||||
)
|
||||
|
||||
# Trigger render video worker
|
||||
from backend.services.worker_service import get_worker_service
|
||||
worker_service = get_worker_service()
|
||||
|
||||
# Run in background, keep reference to prevent garbage collection
|
||||
task = asyncio.create_task(worker_service.trigger_render_video_worker(job_id))
|
||||
_background_tasks.add(task)
|
||||
task.add_done_callback(_background_tasks.discard)
|
||||
|
||||
logger.info(f"Job {job_id}: Review complete, triggered render video worker")
|
||||
|
||||
return {"status": "success"}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Job {job_id}: Error completing review: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Error completing review: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/{job_id}/handlers")
|
||||
async def update_handlers(job_id: str, enabled_handlers: list):
|
||||
"""
|
||||
Update enabled correction handlers (optional feature).
|
||||
|
||||
For now, just acknowledge the request - full handler support
|
||||
would require re-running correction.
|
||||
"""
|
||||
logger.info(f"Job {job_id}: Handler update requested (not implemented)")
|
||||
return {"status": "success", "message": "Handler updates not yet implemented"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/add-lyrics")
|
||||
async def add_lyrics(job_id: str, data: Dict[str, str]):
|
||||
"""
|
||||
Add custom lyrics source and rerun correction.
|
||||
|
||||
Uses the LyricsTranscriber's CorrectionOperations to add a new lyrics source
|
||||
and regenerate corrections with the new source included.
|
||||
"""
|
||||
job_manager = JobManager()
|
||||
storage = StorageService()
|
||||
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
# Job must be in review state to add lyrics
|
||||
if job.status not in [JobStatus.AWAITING_REVIEW, JobStatus.IN_REVIEW]:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Job not in review state (current status: {job.status})"
|
||||
)
|
||||
|
||||
source = data.get("source", "").strip()
|
||||
lyrics_text = data.get("lyrics", "").strip()
|
||||
|
||||
logger.info(f"Job {job_id}: Adding lyrics source '{source}' with {len(lyrics_text)} characters")
|
||||
|
||||
# Use tracing and job_log_context for full observability
|
||||
with create_span("add-lyrics", {"job_id": job_id, "source": source, "lyrics_length": len(lyrics_text)}) as span:
|
||||
with job_log_context(job_id, worker="add-lyrics"):
|
||||
try:
|
||||
# Create temp directory for this operation
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
# Download current corrections.json
|
||||
with create_span("download-corrections") as download_span:
|
||||
corrections_gcs = f"jobs/{job_id}/lyrics/corrections.json"
|
||||
corrections_path = os.path.join(temp_dir, "corrections.json")
|
||||
storage.download_file(corrections_gcs, corrections_path)
|
||||
download_span.set_attribute("gcs_path", corrections_gcs)
|
||||
|
||||
with open(corrections_path, 'r', encoding='utf-8') as f:
|
||||
original_data = json.load(f)
|
||||
|
||||
# Load as CorrectionResult
|
||||
correction_result = CorrectionResult.from_dict(original_data)
|
||||
add_span_event("corrections_loaded", {
|
||||
"segments": len(correction_result.corrected_segments) if correction_result.corrected_segments else 0,
|
||||
"reference_sources": len(correction_result.reference_lyrics) if correction_result.reference_lyrics else 0,
|
||||
})
|
||||
|
||||
# Set up cache directory
|
||||
cache_dir = os.path.join(temp_dir, "cache")
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
|
||||
# Add lyrics source using CorrectionOperations (this is the heavy operation)
|
||||
with create_span("correction-operations-add-lyrics") as correction_span:
|
||||
correction_span.set_attribute("source_name", source)
|
||||
updated_result = CorrectionOperations.add_lyrics_source(
|
||||
correction_result=correction_result,
|
||||
source=source,
|
||||
lyrics_text=lyrics_text,
|
||||
cache_dir=cache_dir,
|
||||
logger=logger
|
||||
)
|
||||
add_span_event("correction_complete", {
|
||||
"new_segments": len(updated_result.corrected_segments) if updated_result.corrected_segments else 0,
|
||||
})
|
||||
|
||||
# Add audio hash for the frontend
|
||||
audio_hash = _get_audio_hash(job_id)
|
||||
if not updated_result.metadata:
|
||||
updated_result.metadata = {}
|
||||
updated_result.metadata['audio_hash'] = audio_hash
|
||||
updated_result.metadata['artist'] = job.artist
|
||||
updated_result.metadata['title'] = job.title
|
||||
|
||||
# Upload updated corrections back to GCS
|
||||
with create_span("upload-corrections") as upload_span:
|
||||
updated_data = updated_result.to_dict()
|
||||
storage.upload_json(corrections_gcs, updated_data)
|
||||
upload_span.set_attribute("gcs_path", corrections_gcs)
|
||||
|
||||
logger.info(f"Job {job_id}: Successfully added lyrics source '{source}'")
|
||||
span.set_attribute("success", True)
|
||||
|
||||
return {"status": "success", "data": updated_data}
|
||||
|
||||
except ValueError as e:
|
||||
# ValueError from CorrectionOperations (e.g., duplicate source name)
|
||||
logger.warning(f"Job {job_id}: Invalid add lyrics request: {e}")
|
||||
span.set_attribute("error", str(e))
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"Job {job_id}: Failed to add lyrics: {e}", exc_info=True)
|
||||
span.set_attribute("error", str(e))
|
||||
raise HTTPException(status_code=500, detail=f"Failed to add lyrics: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/{job_id}/preview-video")
|
||||
async def generate_preview_video(job_id: str, updated_data: Dict[str, Any]):
|
||||
"""
|
||||
Generate a preview video with the current corrections.
|
||||
|
||||
Uses the LyricsTranscriber's CorrectionOperations to generate a 360p preview
|
||||
video with the user's current corrections applied.
|
||||
"""
|
||||
job_manager = JobManager()
|
||||
storage = StorageService()
|
||||
|
||||
job = job_manager.get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
# Job must be in review state to generate preview
|
||||
if job.status not in [JobStatus.AWAITING_REVIEW, JobStatus.IN_REVIEW]:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Job not in review state (current status: {job.status})"
|
||||
)
|
||||
|
||||
logger.info(f"Job {job_id}: Generating preview video")
|
||||
|
||||
# Use tracing and job_log_context for full observability
|
||||
with create_span("generate-preview-video", {"job_id": job_id}) as span:
|
||||
with job_log_context(job_id, worker="preview"):
|
||||
try:
|
||||
# Create temp directory for this preview operation
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
# 1. Download original corrections.json (has full structure)
|
||||
with create_span("download-corrections-and-audio") as download_span:
|
||||
corrections_gcs = f"jobs/{job_id}/lyrics/corrections.json"
|
||||
corrections_path = os.path.join(temp_dir, "corrections.json")
|
||||
storage.download_file(corrections_gcs, corrections_path)
|
||||
|
||||
with open(corrections_path, 'r', encoding='utf-8') as f:
|
||||
original_data = json.load(f)
|
||||
|
||||
# 2. Download input audio
|
||||
audio_path = os.path.join(temp_dir, "audio.flac")
|
||||
storage.download_file(job.input_media_gcs_path, audio_path)
|
||||
download_span.set_attribute("audio_gcs_path", job.input_media_gcs_path)
|
||||
|
||||
# 3. Load original as CorrectionResult
|
||||
correction_result = CorrectionResult.from_dict(original_data)
|
||||
add_span_event("corrections_loaded")
|
||||
|
||||
# 4. Get or create styles file for preview using unified style loader
|
||||
with create_span("load-styles") as styles_span:
|
||||
styles_path, _ = load_styles_from_gcs(
|
||||
style_params_gcs_path=job.style_params_gcs_path,
|
||||
style_assets=job.style_assets,
|
||||
temp_dir=temp_dir,
|
||||
download_func=storage.download_file,
|
||||
logger=logger,
|
||||
)
|
||||
styles_span.set_attribute("styles_path", styles_path)
|
||||
|
||||
# 5. Set up output config for preview
|
||||
output_dir = os.path.join(temp_dir, "output")
|
||||
cache_dir = os.path.join(temp_dir, "cache")
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
|
||||
output_config = OutputConfig(
|
||||
output_styles_json=styles_path,
|
||||
output_dir=output_dir,
|
||||
cache_dir=cache_dir,
|
||||
video_resolution="360p",
|
||||
)
|
||||
|
||||
# 6. Generate preview video using CorrectionOperations (heavy operation)
|
||||
with create_span("render-preview-video") as render_span:
|
||||
render_span.set_attribute("resolution", "360p")
|
||||
result = CorrectionOperations.generate_preview_video(
|
||||
correction_result=correction_result,
|
||||
updated_data=updated_data,
|
||||
output_config=output_config,
|
||||
audio_filepath=audio_path,
|
||||
artist=job.artist,
|
||||
title=job.title,
|
||||
logger=logger
|
||||
)
|
||||
add_span_event("render_complete")
|
||||
|
||||
# 7. Upload preview video to GCS for serving
|
||||
preview_hash = result["preview_hash"]
|
||||
video_path = result["video_path"]
|
||||
|
||||
with create_span("upload-preview-video") as upload_span:
|
||||
preview_gcs_path = f"jobs/{job_id}/previews/{preview_hash}.mp4"
|
||||
storage.upload_file(video_path, preview_gcs_path)
|
||||
upload_span.set_attribute("gcs_path", preview_gcs_path)
|
||||
|
||||
# Store the GCS path for serving
|
||||
if job_id not in _preview_videos:
|
||||
_preview_videos[job_id] = {}
|
||||
_preview_videos[job_id][preview_hash] = preview_gcs_path
|
||||
|
||||
logger.info(f"Job {job_id}: Preview video generated: {preview_hash}")
|
||||
span.set_attribute("preview_hash", preview_hash)
|
||||
span.set_attribute("success", True)
|
||||
|
||||
return {"status": "success", "preview_hash": preview_hash}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Job {job_id}: Failed to generate preview video: {e}", exc_info=True)
|
||||
span.set_attribute("error", str(e))
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Failed to generate preview video: {e}"
|
||||
) from e
|
||||
|
||||
|
||||
@router.get("/{job_id}/preview-video/{preview_hash}")
|
||||
async def get_preview_video(job_id: str, preview_hash: str):
|
||||
"""Stream the generated preview video."""
|
||||
storage = StorageService()
|
||||
|
||||
# Check in-memory cache first
|
||||
preview_gcs_path = None
|
||||
if job_id in _preview_videos and preview_hash in _preview_videos[job_id]:
|
||||
preview_gcs_path = _preview_videos[job_id][preview_hash]
|
||||
else:
|
||||
# Try standard path
|
||||
preview_gcs_path = f"jobs/{job_id}/previews/{preview_hash}.mp4"
|
||||
if not storage.file_exists(preview_gcs_path):
|
||||
raise HTTPException(status_code=404, detail="Preview video not found")
|
||||
|
||||
try:
|
||||
# Download to temp file and stream
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as tmp:
|
||||
tmp_path = tmp.name
|
||||
|
||||
storage.download_file(preview_gcs_path, tmp_path)
|
||||
|
||||
logger.info(f"Job {job_id}: Streaming preview video {preview_hash}")
|
||||
|
||||
return FileResponse(
|
||||
tmp_path,
|
||||
media_type="video/mp4",
|
||||
filename=f"preview_{preview_hash}.mp4",
|
||||
headers={
|
||||
"Accept-Ranges": "bytes",
|
||||
"Content-Disposition": "inline",
|
||||
"Cache-Control": "no-cache",
|
||||
},
|
||||
background=BackgroundTask(os.unlink, tmp_path),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Job {job_id}: Error streaming preview video: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Error streaming preview video: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/{job_id}/v1/annotations")
|
||||
async def submit_annotation(job_id: str, annotation: Dict[str, Any]):
|
||||
"""
|
||||
Submit a correction annotation for ML training data.
|
||||
|
||||
For now, just log and acknowledge - full annotation support
|
||||
would require a database.
|
||||
"""
|
||||
logger.info(f"Job {job_id}: Annotation submitted (logged but not stored)")
|
||||
return {"status": "success", "annotation_id": "stub"}
|
||||
|
||||
|
||||
@router.get("/{job_id}/v1/annotations/stats")
|
||||
async def get_annotation_stats(job_id: str):
|
||||
"""Get annotation statistics."""
|
||||
return {
|
||||
"total_annotations": 0,
|
||||
"by_type": {},
|
||||
"message": "Annotation stats not yet implemented"
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
"""
|
||||
Configuration management for the karaoke generation backend.
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from typing import Optional, Dict
|
||||
from pydantic_settings import BaseSettings
|
||||
from google.cloud import secretmanager
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""Application settings."""
|
||||
|
||||
# Google Cloud
|
||||
google_cloud_project: str = os.getenv("GOOGLE_CLOUD_PROJECT", "")
|
||||
gcs_bucket_name: str = os.getenv("GCS_BUCKET_NAME", "karaoke-gen-storage")
|
||||
gcs_temp_bucket: str = os.getenv("GCS_TEMP_BUCKET", "karaoke-gen-temp")
|
||||
gcs_output_bucket: str = os.getenv("GCS_OUTPUT_BUCKET", "karaoke-gen-outputs")
|
||||
firestore_collection: str = os.getenv("FIRESTORE_COLLECTION", "jobs")
|
||||
|
||||
# Audio Separator API (for GPU processing)
|
||||
audio_separator_api_url: Optional[str] = os.getenv("AUDIO_SEPARATOR_API_URL")
|
||||
|
||||
# External APIs (can be set via env or Secret Manager)
|
||||
audioshake_api_key: Optional[str] = os.getenv("AUDIOSHAKE_API_KEY")
|
||||
genius_api_key: Optional[str] = os.getenv("GENIUS_API_KEY")
|
||||
spotify_cookie: Optional[str] = os.getenv("SPOTIFY_COOKIE_SP_DC")
|
||||
rapidapi_key: Optional[str] = os.getenv("RAPIDAPI_KEY")
|
||||
|
||||
# Authentication
|
||||
admin_tokens: Optional[str] = os.getenv("ADMIN_TOKENS") # Comma-separated list
|
||||
|
||||
# Application
|
||||
environment: str = os.getenv("ENVIRONMENT", "development")
|
||||
log_level: str = os.getenv("LOG_LEVEL", "INFO")
|
||||
|
||||
# Processing
|
||||
max_concurrent_jobs: int = int(os.getenv("MAX_CONCURRENT_JOBS", "5"))
|
||||
job_timeout_seconds: int = int(os.getenv("JOB_TIMEOUT_SECONDS", "3600"))
|
||||
|
||||
# Cloud Tasks (for scalable worker coordination)
|
||||
# When enabled, workers are triggered via Cloud Tasks for guaranteed delivery
|
||||
# When disabled (default), workers are triggered via direct HTTP (for development)
|
||||
enable_cloud_tasks: bool = os.getenv("ENABLE_CLOUD_TASKS", "false").lower() in ("true", "1", "yes")
|
||||
gcp_region: str = os.getenv("GCP_REGION", "us-central1")
|
||||
|
||||
# Cloud Run Jobs (for long-running video encoding)
|
||||
# When enabled AND enable_cloud_tasks is true, video worker uses Cloud Run Jobs
|
||||
# instead of Cloud Tasks. This supports encoding times >30 minutes (up to 24 hours).
|
||||
# Default is false - Cloud Tasks is sufficient for most videos (15-20 min).
|
||||
use_cloud_run_jobs_for_video: bool = os.getenv("USE_CLOUD_RUN_JOBS_FOR_VIDEO", "false").lower() in ("true", "1", "yes")
|
||||
|
||||
# Storage paths
|
||||
temp_dir: str = os.getenv("TEMP_DIR", "/tmp/karaoke-gen")
|
||||
|
||||
# Default distribution settings (can be overridden per-request)
|
||||
default_dropbox_path: Optional[str] = os.getenv("DEFAULT_DROPBOX_PATH")
|
||||
default_gdrive_folder_id: Optional[str] = os.getenv("DEFAULT_GDRIVE_FOLDER_ID")
|
||||
# Strip whitespace/newlines from webhook URL - common issue when env vars are set with trailing newlines
|
||||
default_discord_webhook_url: Optional[str] = (
|
||||
os.getenv("DEFAULT_DISCORD_WEBHOOK_URL", "").strip() or None
|
||||
)
|
||||
|
||||
# Secret Manager cache
|
||||
_secret_cache: Dict[str, str] = {}
|
||||
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
case_sensitive = False
|
||||
|
||||
def get_secret(self, secret_id: str) -> Optional[str]:
|
||||
"""
|
||||
Get a secret from Google Secret Manager.
|
||||
|
||||
Caches secrets in memory to avoid repeated API calls.
|
||||
Falls back to environment variables if Secret Manager unavailable.
|
||||
|
||||
Args:
|
||||
secret_id: Secret name (e.g., "audioshake-api-key")
|
||||
|
||||
Returns:
|
||||
Secret value or None if not found
|
||||
"""
|
||||
# Check cache first
|
||||
if secret_id in self._secret_cache:
|
||||
return self._secret_cache[secret_id]
|
||||
|
||||
# Check environment variable (development mode)
|
||||
env_var = secret_id.upper().replace('-', '_')
|
||||
env_value = os.getenv(env_var)
|
||||
if env_value:
|
||||
logger.debug(f"Using {secret_id} from environment variable")
|
||||
self._secret_cache[secret_id] = env_value
|
||||
return env_value
|
||||
|
||||
# Try Secret Manager (production mode)
|
||||
if not self.google_cloud_project:
|
||||
logger.warning(f"Cannot fetch secret {secret_id}: No GCP project configured")
|
||||
return None
|
||||
|
||||
try:
|
||||
client = secretmanager.SecretManagerServiceClient()
|
||||
name = f"projects/{self.google_cloud_project}/secrets/{secret_id}/versions/latest"
|
||||
response = client.access_secret_version(request={"name": name})
|
||||
# Strip whitespace/newlines - common issue when secrets are created with trailing newlines
|
||||
secret_value = response.payload.data.decode('UTF-8').strip()
|
||||
|
||||
# Cache it
|
||||
self._secret_cache[secret_id] = secret_value
|
||||
logger.info(f"Loaded secret {secret_id} from Secret Manager")
|
||||
return secret_value
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load secret {secret_id} from Secret Manager: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# Global settings instance
|
||||
settings = Settings()
|
||||
|
||||
|
||||
def get_settings() -> Settings:
|
||||
"""Get the global settings instance."""
|
||||
return settings
|
||||
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
"""
|
||||
FastAPI application entry point for karaoke generation backend.
|
||||
"""
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from backend.config import settings
|
||||
from backend.api.routes import health, jobs, internal, file_upload, review, auth
|
||||
from backend.services.tracing import setup_tracing, instrument_app, get_current_trace_id
|
||||
|
||||
|
||||
from backend.version import VERSION
|
||||
|
||||
|
||||
# Configure logging with trace ID support
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, settings.log_level.upper()),
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Initialize OpenTelemetry tracing (must happen before app creation)
|
||||
tracing_enabled = setup_tracing(
|
||||
service_name="karaoke-backend",
|
||||
service_version=VERSION,
|
||||
enable_in_dev=False, # Set to True to enable tracing locally
|
||||
)
|
||||
|
||||
|
||||
async def validate_credentials_on_startup():
|
||||
"""Validate OAuth credentials on startup and send alerts if needed."""
|
||||
try:
|
||||
from backend.services.credential_manager import get_credential_manager, CredentialStatus
|
||||
|
||||
manager = get_credential_manager()
|
||||
results = manager.check_all_credentials()
|
||||
|
||||
invalid_services = [
|
||||
result for result in results.values()
|
||||
if result.status in (CredentialStatus.INVALID, CredentialStatus.EXPIRED)
|
||||
]
|
||||
|
||||
if invalid_services:
|
||||
logger.warning(f"Some OAuth credentials need attention:")
|
||||
for result in invalid_services:
|
||||
logger.warning(f" - {result.service}: {result.message}")
|
||||
|
||||
# Try to send Discord alert
|
||||
discord_url = settings.get_secret("discord-alert-webhook") if hasattr(settings, 'get_secret') else None
|
||||
if discord_url:
|
||||
manager.send_credential_alert(invalid_services, discord_url)
|
||||
logger.info("Sent credential alert to Discord")
|
||||
else:
|
||||
logger.info("All OAuth credentials validated successfully")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to validate credentials on startup: {e}")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""Lifespan event handler for startup and shutdown."""
|
||||
# Startup
|
||||
logger.info("Starting karaoke generation backend")
|
||||
logger.info(f"Environment: {settings.environment}")
|
||||
logger.info(f"GCS Bucket: {settings.gcs_bucket_name}")
|
||||
logger.info(f"Tracing enabled: {tracing_enabled}")
|
||||
|
||||
# Validate OAuth credentials (non-blocking)
|
||||
try:
|
||||
await validate_credentials_on_startup()
|
||||
except Exception as e:
|
||||
logger.error(f"Credential validation failed: {e}")
|
||||
|
||||
yield
|
||||
|
||||
# Shutdown
|
||||
logger.info("Shutting down karaoke generation backend")
|
||||
|
||||
|
||||
# Create FastAPI app
|
||||
app = FastAPI(
|
||||
title="Karaoke Generator API",
|
||||
description="Backend API for web-based karaoke video generation",
|
||||
version=VERSION,
|
||||
lifespan=lifespan
|
||||
)
|
||||
|
||||
# Instrument FastAPI with OpenTelemetry (adds automatic spans for all requests)
|
||||
if tracing_enabled:
|
||||
instrument_app(app)
|
||||
|
||||
# Configure CORS
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"], # Configure this properly for production
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Include routers
|
||||
app.include_router(health.router, prefix="/api")
|
||||
app.include_router(jobs.router, prefix="/api")
|
||||
app.include_router(file_upload.router, prefix="/api") # File upload endpoint
|
||||
app.include_router(internal.router, prefix="/api") # Internal worker endpoints
|
||||
app.include_router(review.router, prefix="/api") # Review UI compatibility endpoints
|
||||
app.include_router(auth.router, prefix="/api") # OAuth credential management
|
||||
|
||||
|
||||
@app.get("/")
|
||||
async def root():
|
||||
"""Root endpoint."""
|
||||
return {
|
||||
"service": "karaoke-gen-backend",
|
||||
"version": VERSION,
|
||||
"status": "running"
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run(app, host="0.0.0.0", port=8080)
|
||||
|
||||
Whitespace-only changes.
@@ -0,0 +1,429 @@
|
||||
"""
|
||||
Job data models for karaoke generation.
|
||||
|
||||
This module defines the complete state machine for karaoke generation jobs,
|
||||
mirroring the 8-stage CLI workflow with human-in-the-loop interaction points.
|
||||
"""
|
||||
from typing import Optional, Dict, Any, List
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from pydantic import BaseModel, Field, validator
|
||||
|
||||
|
||||
class JobStatus(str, Enum):
|
||||
"""
|
||||
Job status enumeration - Complete state machine.
|
||||
|
||||
The workflow has 8 main stages with 2 human interaction points:
|
||||
1. Input & Setup
|
||||
2. Parallel Processing (audio + lyrics)
|
||||
3. Title/End Screen Generation
|
||||
4. Countdown Padding Synchronization
|
||||
5. Human Review (BLOCKING)
|
||||
6. Instrumental Selection (BLOCKING)
|
||||
7. Video Finalization
|
||||
8. Distribution
|
||||
"""
|
||||
# Initial states
|
||||
PENDING = "pending" # Job created, queued for processing
|
||||
DOWNLOADING = "downloading" # Downloading from URL or processing upload
|
||||
|
||||
# Stage 2a: Audio separation (parallel track 1)
|
||||
SEPARATING_STAGE1 = "separating_stage1" # Clean instrumental separation (Modal API)
|
||||
SEPARATING_STAGE2 = "separating_stage2" # Backing vocals separation (Modal API)
|
||||
AUDIO_COMPLETE = "audio_complete" # All audio stems ready
|
||||
|
||||
# Stage 2b: Lyrics processing (parallel track 2)
|
||||
TRANSCRIBING = "transcribing" # AudioShake API transcription
|
||||
CORRECTING = "correcting" # Automatic lyrics correction
|
||||
LYRICS_COMPLETE = "lyrics_complete" # Corrections JSON ready
|
||||
|
||||
# Stage 3: Title/End screens
|
||||
GENERATING_SCREENS = "generating_screens" # Creating title and end screen videos
|
||||
|
||||
# Stage 4: Countdown padding (automatic)
|
||||
APPLYING_PADDING = "applying_padding" # Synchronizing countdown padding
|
||||
|
||||
# Stage 5: Human review (BLOCKING)
|
||||
AWAITING_REVIEW = "awaiting_review" # ⚠️ WAITING FOR USER - lyrics review needed
|
||||
IN_REVIEW = "in_review" # User is actively reviewing lyrics
|
||||
REVIEW_COMPLETE = "review_complete" # User submitted corrected lyrics
|
||||
|
||||
# Stage 5.5: Render video with corrected lyrics (post-review)
|
||||
RENDERING_VIDEO = "rendering_video" # Using OutputGenerator to create with_vocals.mkv
|
||||
|
||||
# Stage 6: Instrumental selection (BLOCKING)
|
||||
AWAITING_INSTRUMENTAL_SELECTION = "awaiting_instrumental_selection" # ⚠️ WAITING FOR USER
|
||||
INSTRUMENTAL_SELECTED = "instrumental_selected" # User made selection
|
||||
|
||||
# Stage 7: Video generation and finalization
|
||||
GENERATING_VIDEO = "generating_video" # Creating initial karaoke video
|
||||
ENCODING = "encoding" # Multi-format video encoding (Cloud Build)
|
||||
PACKAGING = "packaging" # CDG/TXT ZIP generation
|
||||
|
||||
# Stage 8: Distribution (optional)
|
||||
UPLOADING = "uploading" # YouTube/Dropbox upload
|
||||
NOTIFYING = "notifying" # Discord/Email notifications
|
||||
|
||||
# Terminal states
|
||||
COMPLETE = "complete" # All processing finished successfully
|
||||
FAILED = "failed" # Unrecoverable error occurred
|
||||
CANCELLED = "cancelled" # User cancelled the job
|
||||
|
||||
# Legacy compatibility (will be removed)
|
||||
QUEUED = "queued" # Deprecated: use PENDING
|
||||
PROCESSING = "processing" # Deprecated: use specific states
|
||||
READY_FOR_FINALIZATION = "ready_for_finalization" # Deprecated
|
||||
FINALIZING = "finalizing" # Deprecated: use ENCODING/PACKAGING
|
||||
ERROR = "error" # Deprecated: use FAILED
|
||||
|
||||
|
||||
# Valid state transitions
|
||||
STATE_TRANSITIONS = {
|
||||
JobStatus.PENDING: [JobStatus.DOWNLOADING, JobStatus.FAILED, JobStatus.CANCELLED],
|
||||
# DOWNLOADING allows parallel processing (audio + lyrics) and then screens when both complete
|
||||
JobStatus.DOWNLOADING: [JobStatus.SEPARATING_STAGE1, JobStatus.TRANSCRIBING, JobStatus.GENERATING_SCREENS, JobStatus.FAILED],
|
||||
|
||||
# Audio separation flow
|
||||
JobStatus.SEPARATING_STAGE1: [JobStatus.SEPARATING_STAGE2, JobStatus.FAILED],
|
||||
JobStatus.SEPARATING_STAGE2: [JobStatus.AUDIO_COMPLETE, JobStatus.FAILED],
|
||||
JobStatus.AUDIO_COMPLETE: [JobStatus.GENERATING_SCREENS, JobStatus.FAILED],
|
||||
|
||||
# Lyrics flow
|
||||
JobStatus.TRANSCRIBING: [JobStatus.CORRECTING, JobStatus.FAILED],
|
||||
JobStatus.CORRECTING: [JobStatus.LYRICS_COMPLETE, JobStatus.FAILED],
|
||||
JobStatus.LYRICS_COMPLETE: [JobStatus.GENERATING_SCREENS, JobStatus.FAILED],
|
||||
|
||||
# Post-parallel processing
|
||||
JobStatus.GENERATING_SCREENS: [JobStatus.APPLYING_PADDING, JobStatus.AWAITING_REVIEW, JobStatus.AWAITING_INSTRUMENTAL_SELECTION, JobStatus.FAILED],
|
||||
JobStatus.APPLYING_PADDING: [JobStatus.AWAITING_REVIEW, JobStatus.FAILED],
|
||||
|
||||
# Human review flow
|
||||
# AWAITING_REVIEW can go directly to REVIEW_COMPLETE (quick review) or to IN_REVIEW (editing)
|
||||
JobStatus.AWAITING_REVIEW: [JobStatus.IN_REVIEW, JobStatus.REVIEW_COMPLETE, JobStatus.FAILED, JobStatus.CANCELLED],
|
||||
JobStatus.IN_REVIEW: [JobStatus.REVIEW_COMPLETE, JobStatus.AWAITING_REVIEW, JobStatus.FAILED],
|
||||
JobStatus.REVIEW_COMPLETE: [JobStatus.RENDERING_VIDEO, JobStatus.FAILED],
|
||||
|
||||
# Video rendering (post-review)
|
||||
JobStatus.RENDERING_VIDEO: [JobStatus.AWAITING_INSTRUMENTAL_SELECTION, JobStatus.FAILED],
|
||||
|
||||
# Instrumental selection flow
|
||||
JobStatus.AWAITING_INSTRUMENTAL_SELECTION: [JobStatus.INSTRUMENTAL_SELECTED, JobStatus.FAILED, JobStatus.CANCELLED],
|
||||
JobStatus.INSTRUMENTAL_SELECTED: [JobStatus.GENERATING_VIDEO, JobStatus.FAILED],
|
||||
|
||||
# Video generation flow
|
||||
JobStatus.GENERATING_VIDEO: [JobStatus.ENCODING, JobStatus.FAILED],
|
||||
JobStatus.ENCODING: [JobStatus.PACKAGING, JobStatus.COMPLETE, JobStatus.FAILED],
|
||||
JobStatus.PACKAGING: [JobStatus.UPLOADING, JobStatus.COMPLETE, JobStatus.FAILED],
|
||||
|
||||
# Distribution flow
|
||||
JobStatus.UPLOADING: [JobStatus.NOTIFYING, JobStatus.COMPLETE, JobStatus.FAILED],
|
||||
JobStatus.NOTIFYING: [JobStatus.COMPLETE, JobStatus.FAILED],
|
||||
|
||||
# Terminal states - COMPLETE and CANCELLED have no transitions
|
||||
# FAILED allows retry transitions to resume from checkpoints
|
||||
JobStatus.COMPLETE: [],
|
||||
JobStatus.FAILED: [
|
||||
JobStatus.INSTRUMENTAL_SELECTED, # Retry from video generation
|
||||
JobStatus.REVIEW_COMPLETE, # Retry from render stage
|
||||
JobStatus.LYRICS_COMPLETE, # Retry from screens generation
|
||||
],
|
||||
JobStatus.CANCELLED: [],
|
||||
|
||||
# Legacy states (for backward compatibility)
|
||||
JobStatus.QUEUED: [JobStatus.PENDING],
|
||||
JobStatus.PROCESSING: [JobStatus.SEPARATING_STAGE1, JobStatus.TRANSCRIBING],
|
||||
JobStatus.READY_FOR_FINALIZATION: [JobStatus.GENERATING_VIDEO],
|
||||
JobStatus.FINALIZING: [JobStatus.ENCODING],
|
||||
JobStatus.ERROR: [JobStatus.FAILED],
|
||||
}
|
||||
|
||||
|
||||
class TimelineEvent(BaseModel):
|
||||
"""Timeline event for job progress tracking."""
|
||||
status: str
|
||||
timestamp: str
|
||||
progress: Optional[int] = None
|
||||
message: Optional[str] = None
|
||||
|
||||
|
||||
class LogEntry(BaseModel):
|
||||
"""Worker log entry for debugging and monitoring."""
|
||||
timestamp: str
|
||||
level: str # DEBUG, INFO, WARNING, ERROR
|
||||
worker: str # audio, lyrics, screens, video, render
|
||||
message: str
|
||||
|
||||
|
||||
class Job(BaseModel):
|
||||
"""
|
||||
Complete job data model.
|
||||
|
||||
Tracks the full lifecycle of a karaoke generation job from submission
|
||||
through all 8 stages to completion or failure.
|
||||
"""
|
||||
job_id: str
|
||||
status: JobStatus
|
||||
progress: int = 0 # 0-100 percentage for UI display
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
# Input data
|
||||
url: Optional[str] = None # YouTube URL (if provided)
|
||||
artist: Optional[str] = None
|
||||
title: Optional[str] = None
|
||||
filename: Optional[str] = None # Original uploaded filename
|
||||
input_media_gcs_path: Optional[str] = None # GCS path to uploaded file
|
||||
|
||||
# User preferences
|
||||
enable_cdg: bool = True # Generate CDG+MP3 package
|
||||
enable_txt: bool = True # Generate TXT+MP3 package
|
||||
enable_youtube_upload: bool = False # Upload to YouTube
|
||||
youtube_description: Optional[str] = None # YouTube video description
|
||||
webhook_url: Optional[str] = None # Webhook for notifications
|
||||
user_email: Optional[str] = None # Email for notifications
|
||||
|
||||
# Style configuration (uploaded files)
|
||||
style_params_gcs_path: Optional[str] = None # GCS path to style_params.json
|
||||
style_assets: Dict[str, str] = Field(default_factory=dict)
|
||||
"""
|
||||
GCS paths to style asset files:
|
||||
{
|
||||
"intro_background": "gs://bucket/jobs/{job_id}/style/intro_bg.png",
|
||||
"karaoke_background": "gs://bucket/jobs/{job_id}/style/karaoke_bg.png",
|
||||
"end_background": "gs://bucket/jobs/{job_id}/style/end_bg.png",
|
||||
"font": "gs://bucket/jobs/{job_id}/style/font.ttf",
|
||||
"cdg_instrumental_background": "gs://bucket/jobs/{job_id}/style/cdg_instr.png",
|
||||
"cdg_title_background": "gs://bucket/jobs/{job_id}/style/cdg_title.png",
|
||||
"cdg_outro_background": "gs://bucket/jobs/{job_id}/style/cdg_outro.png"
|
||||
}
|
||||
"""
|
||||
|
||||
# Finalisation configuration
|
||||
brand_prefix: Optional[str] = None # Brand code prefix (e.g., "NOMAD")
|
||||
discord_webhook_url: Optional[str] = None # Discord notification webhook
|
||||
youtube_description_template: Optional[str] = None # YouTube description template text
|
||||
|
||||
# Distribution configuration (native API - for remote CLI)
|
||||
dropbox_path: Optional[str] = None # Dropbox folder path for organized output (e.g., "/Karaoke/Tracks-Organized")
|
||||
gdrive_folder_id: Optional[str] = None # Google Drive folder ID for public share uploads
|
||||
|
||||
# Legacy distribution configuration (rclone - for local CLI backward compat)
|
||||
organised_dir_rclone_root: Optional[str] = None # Deprecated: use dropbox_path instead
|
||||
|
||||
# Lyrics configuration (overrides for search/transcription)
|
||||
lyrics_artist: Optional[str] = None # Override artist name for lyrics search
|
||||
lyrics_title: Optional[str] = None # Override title for lyrics search
|
||||
lyrics_file_gcs_path: Optional[str] = None # GCS path to user-provided lyrics file
|
||||
subtitle_offset_ms: int = 0 # Offset for subtitle timing (positive = delay)
|
||||
|
||||
# Audio separation model configuration
|
||||
clean_instrumental_model: Optional[str] = None # Model for clean instrumental separation
|
||||
backing_vocals_models: Optional[List[str]] = None # Models for backing vocals separation
|
||||
other_stems_models: Optional[List[str]] = None # Models for other stems (bass, drums, etc.)
|
||||
|
||||
# Existing instrumental configuration (Batch 3)
|
||||
existing_instrumental_gcs_path: Optional[str] = None # GCS path to user-provided instrumental file
|
||||
|
||||
# Processing state
|
||||
track_output_dir: Optional[str] = None # Local output directory (temp)
|
||||
audio_hash: Optional[str] = None # Hash for deduplication
|
||||
|
||||
# State-specific data (JSON field for stage-specific metadata)
|
||||
state_data: Dict[str, Any] = Field(default_factory=dict)
|
||||
"""
|
||||
Stage-specific metadata. Examples:
|
||||
- audio_complete: {"stems": {"clean": "gs://...", "backing": "gs://..."}}
|
||||
- lyrics_complete: {"corrections_url": "gs://...", "audio_url": "gs://..."}
|
||||
- review_complete: {"corrected_lyrics": {...}}
|
||||
- instrumental_selected: {"selection": "clean" | "with_backing"}
|
||||
- encoding: {"build_id": "...", "progress": 45}
|
||||
"""
|
||||
|
||||
# Timeline tracking
|
||||
timeline: List[TimelineEvent] = Field(default_factory=list)
|
||||
|
||||
# Worker logs for debugging (limited to last N entries to avoid document size issues)
|
||||
worker_logs: List[LogEntry] = Field(default_factory=list)
|
||||
|
||||
# File URLs (GCS storage)
|
||||
file_urls: Dict[str, Any] = Field(default_factory=dict)
|
||||
"""
|
||||
File storage URLs. Structure:
|
||||
{
|
||||
"input": "gs://bucket/jobs/{job_id}/input.flac",
|
||||
"stems": {
|
||||
"instrumental_clean": "gs://...",
|
||||
"instrumental_with_backing": "gs://...",
|
||||
"vocals": "gs://...",
|
||||
"backing_vocals": "gs://...",
|
||||
"lead_vocals": "gs://...",
|
||||
"bass": "gs://...",
|
||||
"drums": "gs://...",
|
||||
"guitar": "gs://...",
|
||||
"piano": "gs://...",
|
||||
"other": "gs://..."
|
||||
},
|
||||
"lyrics": {
|
||||
"corrections": "gs://bucket/jobs/{job_id}/lyrics/corrections.json",
|
||||
"audio": "gs://bucket/jobs/{job_id}/lyrics/audio.flac",
|
||||
"lrc": "gs://...",
|
||||
"ass": "gs://..."
|
||||
},
|
||||
"screens": {
|
||||
"title": "gs://bucket/jobs/{job_id}/screens/title.mov",
|
||||
"end": "gs://bucket/jobs/{job_id}/screens/end.mov"
|
||||
},
|
||||
"videos": {
|
||||
"with_vocals": "gs://bucket/jobs/{job_id}/videos/with_vocals.mkv"
|
||||
},
|
||||
"finals": {
|
||||
"lossless_4k_mp4": "gs://...",
|
||||
"lossless_4k_mkv": "gs://...",
|
||||
"lossy_4k_mp4": "gs://...",
|
||||
"lossy_720p_mp4": "gs://..."
|
||||
},
|
||||
"packages": {
|
||||
"cdg_zip": "gs://...",
|
||||
"txt_zip": "gs://..."
|
||||
},
|
||||
"youtube": {
|
||||
"url": "https://youtube.com/watch?v=...",
|
||||
"video_id": "..."
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
# Results (for backward compatibility, will be deprecated)
|
||||
output_files: Dict[str, str] = Field(default_factory=dict)
|
||||
download_urls: Dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
# Error handling
|
||||
error_message: Optional[str] = None
|
||||
error_details: Optional[Dict[str, Any]] = None # Structured error information
|
||||
retry_count: int = 0 # Number of retry attempts
|
||||
|
||||
# Worker tracking
|
||||
worker_ids: Dict[str, str] = Field(default_factory=dict)
|
||||
"""
|
||||
IDs of background workers/jobs:
|
||||
{
|
||||
"audio_worker": "cloud-run-request-id",
|
||||
"lyrics_worker": "cloud-run-request-id",
|
||||
"video_encoder": "cloud-build-id",
|
||||
"distribution": "cloud-run-request-id"
|
||||
}
|
||||
"""
|
||||
|
||||
# Request metadata (captured at job creation for tracking and filtering)
|
||||
request_metadata: Dict[str, Any] = Field(default_factory=dict)
|
||||
"""
|
||||
Metadata captured from the original API request.
|
||||
Used for tracking, filtering, and operational management.
|
||||
|
||||
Standard fields:
|
||||
{
|
||||
"client_ip": "192.168.1.1", # IP address of the client
|
||||
"user_agent": "karaoke-gen-remote/0.71.0", # User-Agent header
|
||||
"environment": "test", # From X-Environment header (test/production/development)
|
||||
"client_id": "cli-user-123", # From X-Client-ID header (customer/user identifier)
|
||||
"server_version": "0.71.0", # Server version at job creation
|
||||
"created_from": "upload", # "upload" (file) or "url" (YouTube URL)
|
||||
"custom_headers": { # All X-* headers for extensibility
|
||||
"X-Environment": "test",
|
||||
"X-Client-ID": "cli-user-123",
|
||||
"X-Request-ID": "abc-123"
|
||||
}
|
||||
}
|
||||
|
||||
Use cases:
|
||||
- Filter test vs production jobs
|
||||
- Track jobs by customer/client
|
||||
- Debug issues with specific clients
|
||||
- Bulk cleanup of test jobs
|
||||
"""
|
||||
|
||||
# Note: Status transition validation is handled by JobManager.validate_state_transition()
|
||||
# which is called before status updates. The Job model does not validate transitions
|
||||
# because Firestore updates happen directly without reconstructing the model.
|
||||
|
||||
class Config:
|
||||
use_enum_values = True
|
||||
|
||||
|
||||
class JobCreate(BaseModel):
|
||||
"""
|
||||
Job creation request.
|
||||
|
||||
Either `url` OR file upload is required (file upload handled separately).
|
||||
Artist and title are optional - will be auto-detected from YouTube if not provided.
|
||||
"""
|
||||
url: Optional[str] = None
|
||||
artist: Optional[str] = None
|
||||
title: Optional[str] = None
|
||||
filename: Optional[str] = None # Original uploaded filename
|
||||
|
||||
# Optional preferences
|
||||
enable_cdg: bool = True
|
||||
enable_txt: bool = True
|
||||
enable_youtube_upload: bool = False
|
||||
youtube_description: Optional[str] = None
|
||||
webhook_url: Optional[str] = None
|
||||
user_email: Optional[str] = None
|
||||
|
||||
# Style configuration (will be populated after file upload)
|
||||
style_params_gcs_path: Optional[str] = None
|
||||
style_assets: Dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
# Finalisation configuration
|
||||
brand_prefix: Optional[str] = None
|
||||
discord_webhook_url: Optional[str] = None
|
||||
youtube_description_template: Optional[str] = None
|
||||
|
||||
# Distribution configuration (native API - for remote CLI)
|
||||
dropbox_path: Optional[str] = None # Dropbox folder path for organized output
|
||||
gdrive_folder_id: Optional[str] = None # Google Drive folder ID for public share uploads
|
||||
|
||||
# Legacy (rclone - deprecated, use dropbox_path instead)
|
||||
organised_dir_rclone_root: Optional[str] = None
|
||||
|
||||
# Lyrics configuration (overrides for search/transcription)
|
||||
lyrics_artist: Optional[str] = None # Override artist name for lyrics search
|
||||
lyrics_title: Optional[str] = None # Override title for lyrics search
|
||||
lyrics_file_gcs_path: Optional[str] = None # GCS path to user-provided lyrics file
|
||||
subtitle_offset_ms: int = 0 # Offset for subtitle timing (positive = delay)
|
||||
|
||||
# Audio separation model configuration
|
||||
clean_instrumental_model: Optional[str] = None # Model for clean instrumental separation
|
||||
backing_vocals_models: Optional[List[str]] = None # Models for backing vocals separation
|
||||
other_stems_models: Optional[List[str]] = None # Models for other stems (bass, drums, etc.)
|
||||
|
||||
# Existing instrumental configuration (Batch 3)
|
||||
existing_instrumental_gcs_path: Optional[str] = None # GCS path to user-provided instrumental file
|
||||
|
||||
# Request metadata (set by API endpoint from request headers)
|
||||
request_metadata: Dict[str, Any] = Field(default_factory=dict)
|
||||
"""
|
||||
Populated by the API endpoint with request context:
|
||||
- client_ip: Client IP address
|
||||
- user_agent: User-Agent header
|
||||
- environment: From X-Environment header (test/production/development)
|
||||
- client_id: From X-Client-ID header
|
||||
- server_version: Current server version
|
||||
- custom_headers: All X-* headers
|
||||
"""
|
||||
|
||||
@validator('url', 'artist', 'title')
|
||||
def validate_inputs(cls, v):
|
||||
"""Validate string inputs are not empty."""
|
||||
if v is not None and isinstance(v, str) and not v.strip():
|
||||
raise ValueError("Field cannot be empty string")
|
||||
return v.strip() if isinstance(v, str) else v
|
||||
|
||||
|
||||
class JobResponse(BaseModel):
|
||||
"""Job response model."""
|
||||
status: str
|
||||
job_id: str
|
||||
message: str
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
"""
|
||||
API request models for karaoke generation endpoints.
|
||||
"""
|
||||
from typing import Optional, Dict, Any, List
|
||||
from pydantic import BaseModel, HttpUrl, validator
|
||||
|
||||
|
||||
class URLSubmissionRequest(BaseModel):
|
||||
"""Request to submit a job from a URL (YouTube, etc.)."""
|
||||
url: HttpUrl
|
||||
artist: Optional[str] = None # Auto-detected if not provided
|
||||
title: Optional[str] = None # Auto-detected if not provided
|
||||
|
||||
# Optional preferences
|
||||
enable_cdg: bool = True
|
||||
enable_txt: bool = True
|
||||
enable_youtube_upload: bool = False
|
||||
youtube_description: Optional[str] = None
|
||||
webhook_url: Optional[str] = None
|
||||
user_email: Optional[str] = None
|
||||
|
||||
|
||||
class UploadSubmissionRequest(BaseModel):
|
||||
"""Request to submit a job from an uploaded file."""
|
||||
artist: str
|
||||
title: str
|
||||
|
||||
# Optional preferences
|
||||
enable_cdg: bool = True
|
||||
enable_txt: bool = True
|
||||
enable_youtube_upload: bool = False
|
||||
youtube_description: Optional[str] = None
|
||||
webhook_url: Optional[str] = None
|
||||
user_email: Optional[str] = None
|
||||
|
||||
|
||||
class CorrectionsSubmission(BaseModel):
|
||||
"""
|
||||
Request to submit corrected lyrics after human review.
|
||||
|
||||
This is the critical human-in-the-loop interaction point.
|
||||
The corrections data comes from the lyrics-transcriber review interface.
|
||||
"""
|
||||
corrections: Dict[str, Any] # Full corrections JSON from frontend
|
||||
user_notes: Optional[str] = None # Optional notes from reviewer
|
||||
|
||||
@validator('corrections')
|
||||
def validate_corrections_format(cls, v):
|
||||
"""Validate corrections has required fields."""
|
||||
required_fields = ['lines', 'metadata']
|
||||
for field in required_fields:
|
||||
if field not in v:
|
||||
raise ValueError(f"Corrections must include '{field}' field")
|
||||
return v
|
||||
|
||||
|
||||
class InstrumentalSelection(BaseModel):
|
||||
"""
|
||||
Request to select instrumental audio option.
|
||||
|
||||
This is the second critical human-in-the-loop interaction point.
|
||||
User chooses between clean instrumental or instrumental with backing vocals.
|
||||
"""
|
||||
selection: str # "clean" or "with_backing"
|
||||
|
||||
@validator('selection')
|
||||
def validate_selection(cls, v):
|
||||
"""Validate selection is a valid option."""
|
||||
valid_options = ['clean', 'with_backing']
|
||||
if v not in valid_options:
|
||||
raise ValueError(f"Selection must be one of: {valid_options}")
|
||||
return v
|
||||
|
||||
|
||||
class StartReviewRequest(BaseModel):
|
||||
"""Request to mark job as in-review (user opened interface)."""
|
||||
pass # No body needed, just triggers state transition
|
||||
|
||||
|
||||
class CancelJobRequest(BaseModel):
|
||||
"""Request to cancel a job."""
|
||||
reason: Optional[str] = None
|
||||
|
||||
|
||||
class RetryJobRequest(BaseModel):
|
||||
"""Request to retry a failed job."""
|
||||
from_stage: Optional[str] = None # Optional: restart from specific stage
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
python_files = ["test_*.py"]
|
||||
python_classes = ["Test*"]
|
||||
python_functions = ["test_*"]
|
||||
addopts = [
|
||||
"-v",
|
||||
"--tb=short",
|
||||
"--strict-markers",
|
||||
"-ra",
|
||||
]
|
||||
|
||||
markers = [
|
||||
"slow: marks tests as slow (deselect with '-m \"not slow\"')",
|
||||
"integration: marks tests as integration tests requiring live service",
|
||||
]
|
||||
|
||||
# Timeout for slow tests (in seconds)
|
||||
timeout = 300
|
||||
|
||||
[tool.coverage.run]
|
||||
source = ["backend"]
|
||||
omit = ["*/tests/*", "*/test_*"]
|
||||
|
||||
[tool.coverage.report]
|
||||
precision = 2
|
||||
show_missing = true
|
||||
skip_covered = false
|
||||
|
||||
Executable
+93
@@ -0,0 +1,93 @@
|
||||
#!/bin/bash
|
||||
# Quick syntax and structure check (no dependencies required)
|
||||
# Run this before every deployment!
|
||||
|
||||
set -e
|
||||
|
||||
echo "🔍 Quick Backend Check (no dependencies required)"
|
||||
echo "=================================================="
|
||||
echo ""
|
||||
|
||||
# Change to backend directory
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
NC='\033[0m'
|
||||
|
||||
ERRORS=0
|
||||
|
||||
# Check 1: Python syntax
|
||||
echo "1. Checking Python syntax..."
|
||||
for file in $(find . -name "*.py" -not -path "./__pycache__/*" -not -path "./venv/*"); do
|
||||
if ! python3 -m py_compile "$file" 2>/dev/null; then
|
||||
echo -e " ${RED}❌ Syntax error in $file${NC}"
|
||||
python3 -m py_compile "$file"
|
||||
ERRORS=$((ERRORS+1))
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $ERRORS -eq 0 ]; then
|
||||
echo -e " ${GREEN}✅ All Python files have valid syntax${NC}"
|
||||
fi
|
||||
|
||||
# Check 2: No missing imports (basic check)
|
||||
echo ""
|
||||
echo "2. Checking for obvious import issues..."
|
||||
|
||||
# Check that we don't import deleted modules
|
||||
if grep -r "from backend.services.processing_service import" --include="*.py" . 2>/dev/null; then
|
||||
echo -e " ${RED}❌ Found import of deleted processing_service${NC}"
|
||||
ERRORS=$((ERRORS+1))
|
||||
else
|
||||
echo -e " ${GREEN}✅ No imports of deleted modules${NC}"
|
||||
fi
|
||||
|
||||
# Check 3: Required files exist
|
||||
echo ""
|
||||
echo "3. Checking required files exist..."
|
||||
REQUIRED_FILES=(
|
||||
"main.py"
|
||||
"config.py"
|
||||
"requirements.txt"
|
||||
"Dockerfile"
|
||||
"api/routes/health.py"
|
||||
"api/routes/jobs.py"
|
||||
"api/routes/internal.py"
|
||||
"api/routes/file_upload.py"
|
||||
"services/job_manager.py"
|
||||
"services/firestore_service.py"
|
||||
"services/storage_service.py"
|
||||
"services/worker_service.py"
|
||||
"services/auth_service.py"
|
||||
"workers/audio_worker.py"
|
||||
"workers/lyrics_worker.py"
|
||||
"workers/screens_worker.py"
|
||||
"workers/video_worker.py"
|
||||
"models/job.py"
|
||||
"models/requests.py"
|
||||
)
|
||||
|
||||
for file in "${REQUIRED_FILES[@]}"; do
|
||||
if [ ! -f "$file" ]; then
|
||||
echo -e " ${RED}❌ Missing required file: $file${NC}"
|
||||
ERRORS=$((ERRORS+1))
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $ERRORS -eq 0 ]; then
|
||||
echo -e " ${GREEN}✅ All required files exist${NC}"
|
||||
fi
|
||||
|
||||
# Summary
|
||||
echo ""
|
||||
echo "=================================================="
|
||||
if [ $ERRORS -eq 0 ]; then
|
||||
echo -e "${GREEN}✅ All checks passed! Safe to deploy.${NC}"
|
||||
exit 0
|
||||
else
|
||||
echo -e "${RED}❌ $ERRORS error(s) found. Fix before deploying!${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
# DEPRECATED: This file is no longer used for Docker builds or CI.
|
||||
# Docker and CI both use poetry/pyproject.toml directly.
|
||||
# This file is kept for reference only.
|
||||
|
||||
fastapi>=0.104.0,<1.0.0
|
||||
uvicorn[standard]>=0.24.0,<1.0.0
|
||||
pydantic>=2.5.0,<3.0.0
|
||||
pydantic-settings>=2.1.0,<3.0.0
|
||||
python-multipart>=0.0.6,<1.0.0
|
||||
httpx>=0.25.0,<1.0.0
|
||||
google-cloud-firestore>=2.14.0,<3.0.0
|
||||
google-cloud-storage>=2.14.0,<3.0.0
|
||||
google-cloud-secret-manager>=2.18.0,<3.0.0
|
||||
google-cloud-tasks>=2.14.0,<3.0.0
|
||||
google-cloud-run>=0.10.0,<1.0.0
|
||||
python-dotenv>=1.0.0,<2.0.0
|
||||
|
||||
# Audio processing (reuses karaoke_gen library)
|
||||
audio-separator[remote]>=0.18.0 # Remote Modal API client
|
||||
pydub>=0.25.1 # Audio manipulation (used by karaoke_gen)
|
||||
yt-dlp>=2024.0.0 # YouTube downloads (future use)
|
||||
|
||||
# Cloud storage native APIs
|
||||
dropbox>=11.0.0 # Dropbox Python SDK for native API access
|
||||
# google-api-python-client already installed via karaoke_gen for YouTube
|
||||
|
||||
# karaoke-gen dependencies (will be installed from parent dir in Dockerfile)
|
||||
# Using explicit dependencies instead of -e ../
|
||||
|
||||
Executable
+60
@@ -0,0 +1,60 @@
|
||||
#!/bin/bash
|
||||
# Backend Test Runner
|
||||
# Runs integration tests against the deployed Cloud Run service
|
||||
|
||||
set -e
|
||||
|
||||
echo "🧪 Backend Integration Test Suite"
|
||||
echo "=================================="
|
||||
echo ""
|
||||
|
||||
# Check prerequisites
|
||||
echo "Checking prerequisites..."
|
||||
if ! command -v gcloud &> /dev/null; then
|
||||
echo "❌ gcloud CLI not found. Please install Google Cloud SDK."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v pytest &> /dev/null; then
|
||||
echo "❌ pytest not found. Installing test dependencies..."
|
||||
pip install -r tests/requirements.txt
|
||||
fi
|
||||
|
||||
# Verify authentication
|
||||
echo "Verifying authentication..."
|
||||
if ! gcloud auth print-identity-token &> /dev/null; then
|
||||
echo "❌ Not authenticated with gcloud. Run: gcloud auth login"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ Prerequisites met"
|
||||
echo ""
|
||||
|
||||
# Run tests
|
||||
echo "Running integration tests..."
|
||||
echo ""
|
||||
|
||||
# Run fast tests (no slow marker)
|
||||
echo "📝 Running fast tests..."
|
||||
pytest tests/test_api_integration.py -v -m "not slow" --tb=short
|
||||
|
||||
echo ""
|
||||
echo "✅ Fast tests complete!"
|
||||
echo ""
|
||||
|
||||
# Ask if user wants to run slow tests
|
||||
read -p "Run slow/integration tests? These test actual job processing (5-10 min). [y/N] " -n 1 -r
|
||||
echo
|
||||
if [[ $REPLY =~ ^[Yy]$ ]]; then
|
||||
echo "🐌 Running slow integration tests..."
|
||||
pytest tests/test_api_integration.py -v -m "slow" --tb=short
|
||||
echo ""
|
||||
echo "✅ All tests complete!"
|
||||
else
|
||||
echo "⏭️ Skipping slow tests"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=================================="
|
||||
echo "Test run complete! 🎉"
|
||||
|
||||
Whitespace-only changes.
@@ -0,0 +1,285 @@
|
||||
"""
|
||||
Authentication service for karaoke backend.
|
||||
|
||||
Provides token-based authentication with support for:
|
||||
- Admin tokens (hardcoded in environment)
|
||||
- User tokens (stored in Firestore)
|
||||
- Usage tracking and limits
|
||||
- Token management API
|
||||
|
||||
This is designed to be extensible for future Stripe integration
|
||||
and admin dashboard functionality.
|
||||
"""
|
||||
import logging
|
||||
import hashlib
|
||||
import time
|
||||
from enum import Enum
|
||||
from typing import Optional, Tuple, Dict, Any
|
||||
from datetime import datetime
|
||||
|
||||
from backend.services.firestore_service import FirestoreService
|
||||
from backend.config import get_settings
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class UserType(str, Enum):
|
||||
"""User access levels."""
|
||||
ADMIN = "admin" # Unlimited access, can manage tokens
|
||||
UNLIMITED = "unlimited" # Unlimited karaoke generation
|
||||
LIMITED = "limited" # Limited number of uses
|
||||
STRIPE = "stripe" # Paid access via Stripe (future)
|
||||
|
||||
|
||||
class AuthService:
|
||||
"""Service for authentication and token management."""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize auth service."""
|
||||
self.firestore = FirestoreService()
|
||||
self.settings = get_settings()
|
||||
self._init_admin_tokens()
|
||||
|
||||
def _init_admin_tokens(self):
|
||||
"""Load admin tokens from environment variables."""
|
||||
# Get admin tokens from secret manager or env vars
|
||||
admin_tokens_str = self.settings.admin_tokens or ""
|
||||
self.admin_tokens = [t.strip() for t in admin_tokens_str.split(",") if t.strip()]
|
||||
|
||||
if self.admin_tokens:
|
||||
logger.info(f"Loaded {len(self.admin_tokens)} admin token(s)")
|
||||
else:
|
||||
logger.warning("No admin tokens configured! Set ADMIN_TOKENS environment variable.")
|
||||
|
||||
def validate_token(self, token: str) -> Tuple[bool, UserType, int, str]:
|
||||
"""
|
||||
Validate an access token.
|
||||
|
||||
Returns:
|
||||
(is_valid, user_type, remaining_uses, message)
|
||||
remaining_uses: -1 = unlimited, 0 = exhausted, >0 = remaining count
|
||||
"""
|
||||
if not token:
|
||||
return False, UserType.LIMITED, 0, "No token provided"
|
||||
|
||||
# Check for admin tokens (highest priority)
|
||||
if token in self.admin_tokens:
|
||||
logger.info("Admin token validated")
|
||||
return True, UserType.ADMIN, -1, "Admin access granted"
|
||||
|
||||
# Check stored tokens in Firestore
|
||||
token_data = self.firestore.get_token(token)
|
||||
|
||||
if not token_data:
|
||||
return False, UserType.LIMITED, 0, "Invalid token"
|
||||
|
||||
# Check if token is active
|
||||
if not token_data.get("active", True):
|
||||
return False, UserType(token_data["type"]), 0, "Token has been revoked"
|
||||
|
||||
token_type = UserType(token_data["type"])
|
||||
max_uses = token_data.get("max_uses", -1)
|
||||
|
||||
# UNLIMITED tokens: no usage limits
|
||||
if token_type == UserType.UNLIMITED:
|
||||
return True, token_type, -1, "Unlimited access granted"
|
||||
|
||||
# LIMITED tokens: check usage count
|
||||
if token_type == UserType.LIMITED:
|
||||
if max_uses <= 0: # -1 means unlimited
|
||||
return True, token_type, -1, "Limited token with unlimited uses"
|
||||
|
||||
current_uses = token_data.get("usage_count", 0)
|
||||
remaining = max_uses - current_uses
|
||||
|
||||
if remaining <= 0:
|
||||
return False, token_type, 0, "Token usage limit exceeded"
|
||||
|
||||
return True, token_type, remaining, f"Limited token: {remaining} uses remaining"
|
||||
|
||||
# STRIPE tokens: check expiration and usage
|
||||
if token_type == UserType.STRIPE:
|
||||
expires_at = token_data.get("expires_at")
|
||||
|
||||
if expires_at:
|
||||
# expires_at is stored as timestamp
|
||||
if time.time() > expires_at:
|
||||
return False, token_type, 0, "Token has expired"
|
||||
|
||||
if max_uses > 0:
|
||||
current_uses = token_data.get("usage_count", 0)
|
||||
remaining = max_uses - current_uses
|
||||
|
||||
if remaining <= 0:
|
||||
return False, token_type, 0, "Token usage limit exceeded"
|
||||
|
||||
return True, token_type, remaining, f"Stripe token: {remaining} uses remaining"
|
||||
|
||||
return True, token_type, -1, "Stripe access granted"
|
||||
|
||||
return False, UserType.LIMITED, 0, "Unknown token type"
|
||||
|
||||
def increment_token_usage(self, token: str, job_id: str) -> bool:
|
||||
"""
|
||||
Increment usage count for a token and track the job.
|
||||
|
||||
Args:
|
||||
token: The access token
|
||||
job_id: The job ID being created
|
||||
|
||||
Returns:
|
||||
True if usage was tracked, False otherwise
|
||||
"""
|
||||
# Validate token first
|
||||
is_valid, user_type, remaining_uses, message = self.validate_token(token)
|
||||
|
||||
if not is_valid:
|
||||
logger.warning(f"Cannot increment usage for invalid token: {message}")
|
||||
return False
|
||||
|
||||
# Don't track usage for admin or unlimited tokens
|
||||
if user_type in [UserType.ADMIN, UserType.UNLIMITED]:
|
||||
logger.debug(f"Skipping usage tracking for {user_type} token")
|
||||
return True
|
||||
|
||||
# For admin tokens (not in Firestore), no tracking needed
|
||||
if token in self.admin_tokens:
|
||||
return True
|
||||
|
||||
# Increment usage in Firestore
|
||||
try:
|
||||
self.firestore.increment_token_usage(token, job_id)
|
||||
logger.info(f"Incremented usage for token (remaining: {remaining_uses - 1})")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to increment token usage: {e}")
|
||||
return False
|
||||
|
||||
def create_token(
|
||||
self,
|
||||
token_value: str,
|
||||
token_type: UserType,
|
||||
max_uses: int = -1,
|
||||
expires_at: Optional[float] = None,
|
||||
created_by: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Create a new access token.
|
||||
|
||||
Args:
|
||||
token_value: The actual token string (should be secure random string)
|
||||
token_type: Type of access (admin, unlimited, limited, stripe)
|
||||
max_uses: Maximum number of uses (-1 = unlimited)
|
||||
expires_at: Expiration timestamp (None = no expiration)
|
||||
created_by: Admin token that created this token
|
||||
|
||||
Returns:
|
||||
Token data dictionary
|
||||
"""
|
||||
token_data = {
|
||||
"token": token_value,
|
||||
"type": token_type.value,
|
||||
"max_uses": max_uses,
|
||||
"usage_count": 0,
|
||||
"active": True,
|
||||
"created_at": time.time(),
|
||||
"created_by": created_by,
|
||||
"expires_at": expires_at,
|
||||
"jobs": []
|
||||
}
|
||||
|
||||
self.firestore.create_token(token_value, token_data)
|
||||
logger.info(f"Created new {token_type} token with max_uses={max_uses}")
|
||||
|
||||
return token_data
|
||||
|
||||
def revoke_token(self, token: str, revoked_by: Optional[str] = None) -> bool:
|
||||
"""
|
||||
Revoke an access token.
|
||||
|
||||
Args:
|
||||
token: The token to revoke
|
||||
revoked_by: Admin token that revoked this token
|
||||
|
||||
Returns:
|
||||
True if revoked, False if token not found
|
||||
"""
|
||||
try:
|
||||
self.firestore.update_token(token, {
|
||||
"active": False,
|
||||
"revoked_at": time.time(),
|
||||
"revoked_by": revoked_by
|
||||
})
|
||||
logger.info(f"Revoked token")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to revoke token: {e}")
|
||||
return False
|
||||
|
||||
def list_tokens(self, include_inactive: bool = False) -> list:
|
||||
"""
|
||||
List all tokens (admin only).
|
||||
|
||||
Args:
|
||||
include_inactive: Whether to include revoked tokens
|
||||
|
||||
Returns:
|
||||
List of token data dictionaries (with sensitive data masked)
|
||||
"""
|
||||
try:
|
||||
tokens = self.firestore.list_tokens()
|
||||
|
||||
# Filter inactive if requested
|
||||
if not include_inactive:
|
||||
tokens = [t for t in tokens if t.get("active", True)]
|
||||
|
||||
# Mask token values for security (show only first 8 chars)
|
||||
for token in tokens:
|
||||
if "token" in token and len(token["token"]) > 8:
|
||||
token["token"] = token["token"][:8] + "..."
|
||||
|
||||
return tokens
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to list tokens: {e}")
|
||||
return []
|
||||
|
||||
def get_token_info(self, token: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Get information about a token (for admin dashboard).
|
||||
|
||||
Returns:
|
||||
Token data or None if not found
|
||||
"""
|
||||
# Check if it's an admin token (not stored in Firestore)
|
||||
if token in self.admin_tokens:
|
||||
return {
|
||||
"token": token[:8] + "...",
|
||||
"type": UserType.ADMIN.value,
|
||||
"max_uses": -1,
|
||||
"usage_count": 0,
|
||||
"active": True,
|
||||
"source": "environment"
|
||||
}
|
||||
|
||||
# Get from Firestore
|
||||
token_data = self.firestore.get_token(token)
|
||||
|
||||
if token_data and "token" in token_data:
|
||||
# Mask the full token value
|
||||
token_data["token"] = token_data["token"][:8] + "..."
|
||||
|
||||
return token_data
|
||||
|
||||
|
||||
# Global instance
|
||||
_auth_service = None
|
||||
|
||||
|
||||
def get_auth_service() -> AuthService:
|
||||
"""Get the global auth service instance."""
|
||||
global _auth_service
|
||||
if _auth_service is None:
|
||||
_auth_service = AuthService()
|
||||
return _auth_service
|
||||
|
||||
@@ -0,0 +1,792 @@
|
||||
"""
|
||||
OAuth Credential Manager for validating and refreshing credentials.
|
||||
|
||||
This service provides:
|
||||
1. Credential validation for YouTube, Dropbox, and Google Drive
|
||||
2. Device Authorization Flow for re-authentication
|
||||
3. Proactive monitoring with alerts when credentials expire
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from typing import Optional, Dict, Any, Tuple
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from datetime import datetime
|
||||
|
||||
from google.oauth2.credentials import Credentials as GoogleCredentials
|
||||
from google.auth.transport.requests import Request as GoogleAuthRequest
|
||||
|
||||
from backend.config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CredentialStatus(str, Enum):
|
||||
"""Status of OAuth credentials."""
|
||||
VALID = "valid"
|
||||
EXPIRED = "expired" # Token expired but refresh may work
|
||||
INVALID = "invalid" # Refresh failed, re-auth needed
|
||||
NOT_CONFIGURED = "not_configured" # No credentials stored
|
||||
ERROR = "error" # Unknown error during validation
|
||||
|
||||
|
||||
@dataclass
|
||||
class CredentialCheckResult:
|
||||
"""Result of a credential validation check."""
|
||||
service: str
|
||||
status: CredentialStatus
|
||||
message: str
|
||||
last_checked: datetime
|
||||
expires_at: Optional[datetime] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeviceAuthInfo:
|
||||
"""Information for device authorization flow."""
|
||||
device_code: str
|
||||
user_code: str
|
||||
verification_url: str
|
||||
expires_in: int
|
||||
interval: int
|
||||
started_at: datetime
|
||||
|
||||
|
||||
class CredentialManager:
|
||||
"""
|
||||
Manages OAuth credentials for all external services.
|
||||
|
||||
Provides validation, refresh, and device authorization flow for:
|
||||
- YouTube (Google OAuth)
|
||||
- Google Drive (Google OAuth)
|
||||
- Dropbox
|
||||
"""
|
||||
|
||||
# Secret names in Secret Manager
|
||||
YOUTUBE_SECRET = "youtube-oauth-credentials"
|
||||
GDRIVE_SECRET = "gdrive-oauth-credentials"
|
||||
DROPBOX_SECRET = "dropbox-oauth-credentials"
|
||||
|
||||
# OAuth client credentials (for device auth flow)
|
||||
YOUTUBE_CLIENT_SECRET = "youtube-client-credentials"
|
||||
GDRIVE_CLIENT_SECRET = "gdrive-client-credentials"
|
||||
|
||||
# Google OAuth endpoints
|
||||
GOOGLE_TOKEN_URI = "https://oauth2.googleapis.com/token"
|
||||
GOOGLE_DEVICE_AUTH_URI = "https://oauth2.googleapis.com/device/code"
|
||||
|
||||
# Dropbox OAuth endpoints
|
||||
DROPBOX_TOKEN_URI = "https://api.dropboxapi.com/oauth2/token"
|
||||
|
||||
# Scopes for each service
|
||||
YOUTUBE_SCOPES = ["https://www.googleapis.com/auth/youtube"]
|
||||
GDRIVE_SCOPES = ["https://www.googleapis.com/auth/drive.file"]
|
||||
|
||||
def __init__(self):
|
||||
self.settings = get_settings()
|
||||
self._pending_device_auths: Dict[str, DeviceAuthInfo] = {}
|
||||
|
||||
def _get_client_credentials(self, secret_name: str) -> Optional[Dict[str, str]]:
|
||||
"""
|
||||
Load OAuth client credentials from Secret Manager.
|
||||
|
||||
The secret should contain:
|
||||
{
|
||||
"client_id": "...",
|
||||
"client_secret": "..."
|
||||
}
|
||||
"""
|
||||
try:
|
||||
creds_json = self.settings.get_secret(secret_name)
|
||||
if not creds_json:
|
||||
return None
|
||||
|
||||
creds = json.loads(creds_json)
|
||||
if creds.get("client_id") and creds.get("client_secret"):
|
||||
return creds
|
||||
|
||||
logger.warning(f"{secret_name} missing client_id or client_secret")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load {secret_name}: {e}")
|
||||
return None
|
||||
|
||||
def get_youtube_client_credentials(self) -> Optional[Dict[str, str]]:
|
||||
"""Get YouTube OAuth client credentials from Secret Manager."""
|
||||
return self._get_client_credentials(self.YOUTUBE_CLIENT_SECRET)
|
||||
|
||||
def get_gdrive_client_credentials(self) -> Optional[Dict[str, str]]:
|
||||
"""Get Google Drive OAuth client credentials from Secret Manager."""
|
||||
return self._get_client_credentials(self.GDRIVE_CLIENT_SECRET)
|
||||
|
||||
# =========================================================================
|
||||
# Credential Validation
|
||||
# =========================================================================
|
||||
|
||||
def check_all_credentials(self) -> Dict[str, CredentialCheckResult]:
|
||||
"""
|
||||
Check validity of all OAuth credentials.
|
||||
|
||||
Returns:
|
||||
Dictionary mapping service name to check result
|
||||
"""
|
||||
results = {}
|
||||
|
||||
results["youtube"] = self.check_youtube_credentials()
|
||||
results["gdrive"] = self.check_gdrive_credentials()
|
||||
results["dropbox"] = self.check_dropbox_credentials()
|
||||
|
||||
return results
|
||||
|
||||
def check_youtube_credentials(self) -> CredentialCheckResult:
|
||||
"""Check if YouTube credentials are valid and can be refreshed."""
|
||||
return self._check_google_credentials(
|
||||
secret_name=self.YOUTUBE_SECRET,
|
||||
service_name="youtube",
|
||||
scopes=self.YOUTUBE_SCOPES,
|
||||
test_api_call=self._test_youtube_api
|
||||
)
|
||||
|
||||
def check_gdrive_credentials(self) -> CredentialCheckResult:
|
||||
"""Check if Google Drive credentials are valid and can be refreshed."""
|
||||
return self._check_google_credentials(
|
||||
secret_name=self.GDRIVE_SECRET,
|
||||
service_name="gdrive",
|
||||
scopes=self.GDRIVE_SCOPES,
|
||||
test_api_call=self._test_gdrive_api
|
||||
)
|
||||
|
||||
def check_dropbox_credentials(self) -> CredentialCheckResult:
|
||||
"""Check if Dropbox credentials are valid."""
|
||||
try:
|
||||
creds_json = self.settings.get_secret(self.DROPBOX_SECRET)
|
||||
|
||||
if not creds_json:
|
||||
return CredentialCheckResult(
|
||||
service="dropbox",
|
||||
status=CredentialStatus.NOT_CONFIGURED,
|
||||
message="Dropbox credentials not configured",
|
||||
last_checked=datetime.utcnow()
|
||||
)
|
||||
|
||||
creds = json.loads(creds_json)
|
||||
|
||||
if not creds.get("access_token"):
|
||||
return CredentialCheckResult(
|
||||
service="dropbox",
|
||||
status=CredentialStatus.INVALID,
|
||||
message="Dropbox credentials missing access_token",
|
||||
last_checked=datetime.utcnow()
|
||||
)
|
||||
|
||||
# Test the credentials with a simple API call
|
||||
if self._test_dropbox_api(creds):
|
||||
return CredentialCheckResult(
|
||||
service="dropbox",
|
||||
status=CredentialStatus.VALID,
|
||||
message="Dropbox credentials are valid",
|
||||
last_checked=datetime.utcnow()
|
||||
)
|
||||
else:
|
||||
# Try to refresh if we have refresh token
|
||||
if creds.get("refresh_token") and creds.get("app_key") and creds.get("app_secret"):
|
||||
if self._refresh_dropbox_token(creds):
|
||||
return CredentialCheckResult(
|
||||
service="dropbox",
|
||||
status=CredentialStatus.VALID,
|
||||
message="Dropbox credentials refreshed successfully",
|
||||
last_checked=datetime.utcnow()
|
||||
)
|
||||
|
||||
return CredentialCheckResult(
|
||||
service="dropbox",
|
||||
status=CredentialStatus.INVALID,
|
||||
message="Dropbox credentials invalid and refresh failed",
|
||||
last_checked=datetime.utcnow()
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking Dropbox credentials: {e}")
|
||||
return CredentialCheckResult(
|
||||
service="dropbox",
|
||||
status=CredentialStatus.ERROR,
|
||||
message=f"Error checking credentials: {str(e)}",
|
||||
last_checked=datetime.utcnow()
|
||||
)
|
||||
|
||||
def _check_google_credentials(
|
||||
self,
|
||||
secret_name: str,
|
||||
service_name: str,
|
||||
scopes: list,
|
||||
test_api_call: callable
|
||||
) -> CredentialCheckResult:
|
||||
"""Generic Google OAuth credential check."""
|
||||
try:
|
||||
creds_json = self.settings.get_secret(secret_name)
|
||||
|
||||
if not creds_json:
|
||||
return CredentialCheckResult(
|
||||
service=service_name,
|
||||
status=CredentialStatus.NOT_CONFIGURED,
|
||||
message=f"{service_name} credentials not configured",
|
||||
last_checked=datetime.utcnow()
|
||||
)
|
||||
|
||||
creds_data = json.loads(creds_json)
|
||||
|
||||
# Check for required fields
|
||||
required = ["refresh_token", "client_id", "client_secret"]
|
||||
missing = [f for f in required if not creds_data.get(f)]
|
||||
if missing:
|
||||
return CredentialCheckResult(
|
||||
service=service_name,
|
||||
status=CredentialStatus.INVALID,
|
||||
message=f"Missing required fields: {missing}",
|
||||
last_checked=datetime.utcnow()
|
||||
)
|
||||
|
||||
# Create credentials object
|
||||
credentials = GoogleCredentials(
|
||||
token=creds_data.get("token"),
|
||||
refresh_token=creds_data.get("refresh_token"),
|
||||
token_uri=creds_data.get("token_uri", self.GOOGLE_TOKEN_URI),
|
||||
client_id=creds_data.get("client_id"),
|
||||
client_secret=creds_data.get("client_secret"),
|
||||
scopes=creds_data.get("scopes", scopes)
|
||||
)
|
||||
|
||||
# Try to refresh if expired
|
||||
if credentials.expired or not credentials.token:
|
||||
try:
|
||||
credentials.refresh(GoogleAuthRequest())
|
||||
# Update stored credentials with new token
|
||||
self._update_google_credentials(secret_name, creds_data, credentials)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to refresh {service_name} credentials: {e}")
|
||||
return CredentialCheckResult(
|
||||
service=service_name,
|
||||
status=CredentialStatus.INVALID,
|
||||
message=f"Token refresh failed: {str(e)}",
|
||||
last_checked=datetime.utcnow()
|
||||
)
|
||||
|
||||
# Test with API call
|
||||
if test_api_call(credentials):
|
||||
return CredentialCheckResult(
|
||||
service=service_name,
|
||||
status=CredentialStatus.VALID,
|
||||
message=f"{service_name} credentials are valid",
|
||||
last_checked=datetime.utcnow(),
|
||||
expires_at=credentials.expiry
|
||||
)
|
||||
else:
|
||||
return CredentialCheckResult(
|
||||
service=service_name,
|
||||
status=CredentialStatus.INVALID,
|
||||
message=f"{service_name} API test failed",
|
||||
last_checked=datetime.utcnow()
|
||||
)
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
return CredentialCheckResult(
|
||||
service=service_name,
|
||||
status=CredentialStatus.INVALID,
|
||||
message=f"Invalid JSON in credentials: {str(e)}",
|
||||
last_checked=datetime.utcnow()
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking {service_name} credentials: {e}")
|
||||
return CredentialCheckResult(
|
||||
service=service_name,
|
||||
status=CredentialStatus.ERROR,
|
||||
message=f"Error: {str(e)}",
|
||||
last_checked=datetime.utcnow()
|
||||
)
|
||||
|
||||
def _update_google_credentials(
|
||||
self,
|
||||
secret_name: str,
|
||||
original_data: dict,
|
||||
credentials: GoogleCredentials
|
||||
) -> bool:
|
||||
"""Update stored Google credentials with refreshed token."""
|
||||
try:
|
||||
from google.cloud import secretmanager
|
||||
|
||||
updated_data = original_data.copy()
|
||||
updated_data["token"] = credentials.token
|
||||
if credentials.expiry:
|
||||
updated_data["expiry"] = credentials.expiry.isoformat()
|
||||
|
||||
# Add new secret version
|
||||
client = secretmanager.SecretManagerServiceClient()
|
||||
project_id = self.settings.google_cloud_project
|
||||
secret_path = f"projects/{project_id}/secrets/{secret_name}"
|
||||
|
||||
client.add_secret_version(
|
||||
parent=secret_path,
|
||||
payload={"data": json.dumps(updated_data).encode("utf-8")}
|
||||
)
|
||||
|
||||
logger.info(f"Updated {secret_name} with refreshed token")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to update {secret_name}: {e}")
|
||||
return False
|
||||
|
||||
# =========================================================================
|
||||
# API Test Methods
|
||||
# =========================================================================
|
||||
|
||||
def _test_youtube_api(self, credentials: GoogleCredentials) -> bool:
|
||||
"""Test YouTube credentials with a simple API call."""
|
||||
try:
|
||||
from googleapiclient.discovery import build
|
||||
|
||||
youtube = build("youtube", "v3", credentials=credentials)
|
||||
# Just get channel info - minimal API call
|
||||
request = youtube.channels().list(part="id", mine=True)
|
||||
response = request.execute()
|
||||
|
||||
return "items" in response
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"YouTube API test failed: {e}")
|
||||
return False
|
||||
|
||||
def _test_gdrive_api(self, credentials: GoogleCredentials) -> bool:
|
||||
"""Test Google Drive credentials with a simple API call."""
|
||||
try:
|
||||
from googleapiclient.discovery import build
|
||||
|
||||
drive = build("drive", "v3", credentials=credentials)
|
||||
# Just get about info - minimal API call
|
||||
request = drive.about().get(fields="user")
|
||||
response = request.execute()
|
||||
|
||||
return "user" in response
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Google Drive API test failed: {e}")
|
||||
return False
|
||||
|
||||
def _test_dropbox_api(self, creds: dict) -> bool:
|
||||
"""Test Dropbox credentials with a simple API call."""
|
||||
try:
|
||||
import dropbox
|
||||
|
||||
dbx = dropbox.Dropbox(creds["access_token"])
|
||||
# Just get account info - minimal API call
|
||||
dbx.users_get_current_account()
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Dropbox API test failed: {e}")
|
||||
return False
|
||||
|
||||
def _refresh_dropbox_token(self, creds: dict) -> bool:
|
||||
"""Try to refresh Dropbox token."""
|
||||
try:
|
||||
import requests
|
||||
|
||||
response = requests.post(
|
||||
self.DROPBOX_TOKEN_URI,
|
||||
data={
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": creds["refresh_token"],
|
||||
"client_id": creds["app_key"],
|
||||
"client_secret": creds["app_secret"],
|
||||
},
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
token_data = response.json()
|
||||
creds["access_token"] = token_data["access_token"]
|
||||
|
||||
# Update in Secret Manager
|
||||
self._update_dropbox_credentials(creds)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Dropbox token refresh failed: {e}")
|
||||
return False
|
||||
|
||||
def _update_dropbox_credentials(self, creds: dict) -> bool:
|
||||
"""Update stored Dropbox credentials."""
|
||||
try:
|
||||
from google.cloud import secretmanager
|
||||
|
||||
client = secretmanager.SecretManagerServiceClient()
|
||||
project_id = self.settings.google_cloud_project
|
||||
secret_path = f"projects/{project_id}/secrets/{self.DROPBOX_SECRET}"
|
||||
|
||||
client.add_secret_version(
|
||||
parent=secret_path,
|
||||
payload={"data": json.dumps(creds).encode("utf-8")}
|
||||
)
|
||||
|
||||
logger.info("Updated Dropbox credentials with refreshed token")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to update Dropbox credentials: {e}")
|
||||
return False
|
||||
|
||||
# =========================================================================
|
||||
# Device Authorization Flow
|
||||
# =========================================================================
|
||||
|
||||
def start_youtube_device_auth(
|
||||
self,
|
||||
client_id: Optional[str] = None,
|
||||
client_secret: Optional[str] = None
|
||||
) -> DeviceAuthInfo:
|
||||
"""
|
||||
Start YouTube device authorization flow.
|
||||
|
||||
Args:
|
||||
client_id: Google OAuth client ID (optional, reads from Secret Manager if not provided)
|
||||
client_secret: Google OAuth client secret (optional, reads from Secret Manager if not provided)
|
||||
|
||||
Returns:
|
||||
DeviceAuthInfo with user code and verification URL
|
||||
"""
|
||||
# Load from Secret Manager if not provided
|
||||
if not client_id or not client_secret:
|
||||
stored_creds = self.get_youtube_client_credentials()
|
||||
if not stored_creds:
|
||||
raise Exception(
|
||||
"YouTube client credentials not found. Either pass client_id/client_secret "
|
||||
"or create the 'youtube-client-credentials' secret in Secret Manager."
|
||||
)
|
||||
client_id = stored_creds["client_id"]
|
||||
client_secret = stored_creds["client_secret"]
|
||||
|
||||
return self._start_google_device_auth(
|
||||
client_id=client_id,
|
||||
client_secret=client_secret,
|
||||
scopes=self.YOUTUBE_SCOPES,
|
||||
service_name="youtube"
|
||||
)
|
||||
|
||||
def start_gdrive_device_auth(
|
||||
self,
|
||||
client_id: Optional[str] = None,
|
||||
client_secret: Optional[str] = None
|
||||
) -> DeviceAuthInfo:
|
||||
"""
|
||||
Start Google Drive device authorization flow.
|
||||
|
||||
Args:
|
||||
client_id: Google OAuth client ID (optional, reads from Secret Manager if not provided)
|
||||
client_secret: Google OAuth client secret (optional, reads from Secret Manager if not provided)
|
||||
|
||||
Returns:
|
||||
DeviceAuthInfo with user code and verification URL
|
||||
"""
|
||||
# Load from Secret Manager if not provided
|
||||
if not client_id or not client_secret:
|
||||
stored_creds = self.get_gdrive_client_credentials()
|
||||
if not stored_creds:
|
||||
raise Exception(
|
||||
"Google Drive client credentials not found. Either pass client_id/client_secret "
|
||||
"or create the 'gdrive-client-credentials' secret in Secret Manager."
|
||||
)
|
||||
client_id = stored_creds["client_id"]
|
||||
client_secret = stored_creds["client_secret"]
|
||||
|
||||
return self._start_google_device_auth(
|
||||
client_id=client_id,
|
||||
client_secret=client_secret,
|
||||
scopes=self.GDRIVE_SCOPES,
|
||||
service_name="gdrive"
|
||||
)
|
||||
|
||||
def _start_google_device_auth(
|
||||
self,
|
||||
client_id: str,
|
||||
client_secret: str,
|
||||
scopes: list,
|
||||
service_name: str
|
||||
) -> DeviceAuthInfo:
|
||||
"""Start Google device authorization flow."""
|
||||
import requests
|
||||
|
||||
logger.info(f"[{service_name}] Starting device auth flow with scopes: {scopes}")
|
||||
|
||||
response = requests.post(
|
||||
self.GOOGLE_DEVICE_AUTH_URI,
|
||||
data={
|
||||
"client_id": client_id,
|
||||
"scope": " ".join(scopes),
|
||||
},
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
logger.error(f"[{service_name}] Device auth request failed: {response.status_code} - {response.text}")
|
||||
raise Exception(f"Device auth request failed: {response.text}")
|
||||
|
||||
logger.info(f"[{service_name}] Device auth initiated successfully")
|
||||
|
||||
data = response.json()
|
||||
|
||||
# Google uses 'verification_uri' but some docs show 'verification_url'
|
||||
verification_url = data.get("verification_uri") or data.get("verification_url")
|
||||
|
||||
device_info = DeviceAuthInfo(
|
||||
device_code=data["device_code"],
|
||||
user_code=data["user_code"],
|
||||
verification_url=verification_url,
|
||||
expires_in=data["expires_in"],
|
||||
interval=data.get("interval", 5),
|
||||
started_at=datetime.utcnow()
|
||||
)
|
||||
|
||||
# Store for polling, include client secret for token exchange
|
||||
self._pending_device_auths[f"{service_name}:{data['device_code']}"] = {
|
||||
"info": device_info,
|
||||
"client_id": client_id,
|
||||
"client_secret": client_secret,
|
||||
"scopes": scopes,
|
||||
"service_name": service_name
|
||||
}
|
||||
|
||||
return device_info
|
||||
|
||||
def poll_device_auth(self, service_name: str, device_code: str) -> Tuple[str, Optional[dict]]:
|
||||
"""
|
||||
Poll for device authorization completion.
|
||||
|
||||
This method is STATELESS - it fetches client credentials from Secret Manager
|
||||
and polls Google directly. This works correctly in serverless environments
|
||||
like Cloud Run where in-memory state is not preserved between requests.
|
||||
|
||||
Args:
|
||||
service_name: "youtube" or "gdrive"
|
||||
device_code: The device code from start_*_device_auth
|
||||
|
||||
Returns:
|
||||
Tuple of (status, token_data)
|
||||
status: "pending", "complete", "expired", "error"
|
||||
token_data: Token data if complete, None otherwise
|
||||
"""
|
||||
import requests
|
||||
|
||||
logger.info(f"[{service_name}] Polling device auth for code: {device_code[:20]}...")
|
||||
|
||||
# Get client credentials from Secret Manager (stateless approach)
|
||||
if service_name == "youtube":
|
||||
client_creds = self.get_youtube_client_credentials()
|
||||
scopes = self.YOUTUBE_SCOPES
|
||||
secret_name = self.YOUTUBE_SECRET
|
||||
elif service_name == "gdrive":
|
||||
client_creds = self.get_gdrive_client_credentials()
|
||||
scopes = self.GDRIVE_SCOPES
|
||||
secret_name = self.GDRIVE_SECRET
|
||||
else:
|
||||
logger.error(f"[{service_name}] Unknown service")
|
||||
return ("error", {"message": f"Unknown service: {service_name}"})
|
||||
|
||||
if not client_creds:
|
||||
logger.error(f"[{service_name}] Client credentials not found in Secret Manager")
|
||||
return ("error", {"message": f"Client credentials not found in Secret Manager for {service_name}"})
|
||||
|
||||
client_id = client_creds.get("client_id")
|
||||
client_secret = client_creds.get("client_secret")
|
||||
|
||||
if not client_id or not client_secret:
|
||||
logger.error(f"[{service_name}] Invalid client credentials (missing client_id or client_secret)")
|
||||
return ("error", {"message": f"Invalid client credentials for {service_name}"})
|
||||
|
||||
logger.info(f"[{service_name}] Got client credentials, polling Google token endpoint...")
|
||||
|
||||
# Poll Google token endpoint
|
||||
response = requests.post(
|
||||
self.GOOGLE_TOKEN_URI,
|
||||
data={
|
||||
"client_id": client_id,
|
||||
"client_secret": client_secret,
|
||||
"device_code": device_code,
|
||||
"grant_type": "urn:ietf:params:oauth:grant-type:device_code",
|
||||
},
|
||||
timeout=30
|
||||
)
|
||||
|
||||
data = response.json()
|
||||
logger.info(f"[{service_name}] Token endpoint response: status={response.status_code}")
|
||||
|
||||
if response.status_code == 200:
|
||||
logger.info(f"[{service_name}] Token exchange successful! Got access token and refresh_token={bool(data.get('refresh_token'))}")
|
||||
|
||||
# Success! Store the credentials
|
||||
token_data = {
|
||||
"token": data["access_token"],
|
||||
"refresh_token": data.get("refresh_token"),
|
||||
"token_uri": self.GOOGLE_TOKEN_URI,
|
||||
"client_id": client_id,
|
||||
"client_secret": client_secret,
|
||||
"scopes": scopes,
|
||||
}
|
||||
|
||||
# Save to Secret Manager
|
||||
logger.info(f"[{service_name}] Saving credentials to secret: {secret_name}")
|
||||
saved = self._save_credentials_to_secret(secret_name, token_data)
|
||||
if not saved:
|
||||
logger.error(f"[{service_name}] Failed to save credentials to Secret Manager!")
|
||||
return ("error", {"message": "Token exchange succeeded but failed to save credentials to Secret Manager"})
|
||||
|
||||
logger.info(f"[{service_name}] Device auth flow COMPLETE - credentials saved successfully")
|
||||
return ("complete", token_data)
|
||||
|
||||
elif "error" in data:
|
||||
error = data["error"]
|
||||
logger.info(f"[{service_name}] Token endpoint returned error: {error}")
|
||||
|
||||
if error == "authorization_pending":
|
||||
return ("pending", {"message": "Waiting for user to authorize. Please visit the verification URL and enter the code."})
|
||||
elif error == "slow_down":
|
||||
return ("pending", {"message": "Please wait a few more seconds before polling again."})
|
||||
elif error == "expired_token":
|
||||
logger.warning(f"[{service_name}] Device code expired")
|
||||
return ("expired", {"message": "Device code expired. Please start a new device auth flow."})
|
||||
elif error == "access_denied":
|
||||
logger.warning(f"[{service_name}] User denied access")
|
||||
return ("error", {"message": "User denied access"})
|
||||
else:
|
||||
logger.error(f"[{service_name}] Unexpected error: {data}")
|
||||
return ("error", {"message": data.get("error_description", error)})
|
||||
|
||||
logger.error(f"[{service_name}] Unknown response from token endpoint: {data}")
|
||||
return ("error", {"message": "Unknown response from token endpoint"})
|
||||
|
||||
def _save_credentials_to_secret(self, secret_name: str, token_data: dict) -> bool:
|
||||
"""Save credentials to Secret Manager, creating the secret if needed."""
|
||||
logger.info(f"[SecretManager] Attempting to save credentials to: {secret_name}")
|
||||
|
||||
try:
|
||||
from google.cloud import secretmanager
|
||||
from google.api_core import exceptions as gcp_exceptions
|
||||
|
||||
client = secretmanager.SecretManagerServiceClient()
|
||||
project_id = self.settings.google_cloud_project
|
||||
|
||||
if not project_id:
|
||||
logger.error(f"[SecretManager] No GCP project configured!")
|
||||
return False
|
||||
|
||||
parent = f"projects/{project_id}"
|
||||
secret_path = f"{parent}/secrets/{secret_name}"
|
||||
|
||||
logger.info(f"[SecretManager] Using project: {project_id}, secret path: {secret_path}")
|
||||
|
||||
# Try to add a version to existing secret
|
||||
try:
|
||||
client.add_secret_version(
|
||||
parent=secret_path,
|
||||
payload={"data": json.dumps(token_data).encode("utf-8")}
|
||||
)
|
||||
logger.info(f"[SecretManager] SUCCESS - Added new version to existing secret: {secret_name}")
|
||||
return True
|
||||
|
||||
except gcp_exceptions.NotFound:
|
||||
# Secret doesn't exist, create it first
|
||||
logger.info(f"[SecretManager] Secret {secret_name} not found, creating new secret...")
|
||||
client.create_secret(
|
||||
parent=parent,
|
||||
secret_id=secret_name,
|
||||
secret={"replication": {"automatic": {}}}
|
||||
)
|
||||
logger.info(f"[SecretManager] Created secret: {secret_name}")
|
||||
|
||||
# Now add the version
|
||||
client.add_secret_version(
|
||||
parent=secret_path,
|
||||
payload={"data": json.dumps(token_data).encode("utf-8")}
|
||||
)
|
||||
logger.info(f"[SecretManager] SUCCESS - Created secret and saved credentials to: {secret_name}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[SecretManager] FAILED to save credentials to {secret_name}: {type(e).__name__}: {e}")
|
||||
import traceback
|
||||
logger.error(f"[SecretManager] Traceback: {traceback.format_exc()}")
|
||||
return False
|
||||
|
||||
# =========================================================================
|
||||
# Alerts
|
||||
# =========================================================================
|
||||
|
||||
def send_credential_alert(
|
||||
self,
|
||||
invalid_services: list[CredentialCheckResult],
|
||||
discord_webhook_url: Optional[str] = None
|
||||
) -> bool:
|
||||
"""
|
||||
Send alert about invalid credentials.
|
||||
|
||||
Args:
|
||||
invalid_services: List of services with invalid credentials
|
||||
discord_webhook_url: Discord webhook URL for notifications
|
||||
|
||||
Returns:
|
||||
True if alert was sent successfully
|
||||
"""
|
||||
if not discord_webhook_url:
|
||||
logger.warning("No Discord webhook URL configured for alerts")
|
||||
return False
|
||||
|
||||
try:
|
||||
import requests
|
||||
|
||||
services_list = "\n".join([
|
||||
f"• **{r.service}**: {r.message}"
|
||||
for r in invalid_services
|
||||
])
|
||||
|
||||
# Get the API base URL for re-auth links
|
||||
api_url = self.settings.api_base_url if hasattr(self.settings, 'api_base_url') else "https://your-api-url"
|
||||
|
||||
message = {
|
||||
"embeds": [{
|
||||
"title": "⚠️ OAuth Credentials Need Attention",
|
||||
"description": f"The following service credentials need re-authorization:\n\n{services_list}",
|
||||
"color": 16744256, # Orange
|
||||
"fields": [
|
||||
{
|
||||
"name": "Re-authorize",
|
||||
"value": f"Visit `{api_url}/api/auth/status` to start re-authorization flow",
|
||||
"inline": False
|
||||
}
|
||||
],
|
||||
"footer": {
|
||||
"text": "karaoke-gen backend"
|
||||
},
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}]
|
||||
}
|
||||
|
||||
response = requests.post(discord_webhook_url, json=message, timeout=30)
|
||||
return response.status_code in (200, 204)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send credential alert: {e}")
|
||||
return False
|
||||
|
||||
|
||||
# Singleton instance
|
||||
_credential_manager: Optional[CredentialManager] = None
|
||||
|
||||
|
||||
def get_credential_manager() -> CredentialManager:
|
||||
"""Get the singleton credential manager instance."""
|
||||
global _credential_manager
|
||||
if _credential_manager is None:
|
||||
_credential_manager = CredentialManager()
|
||||
return _credential_manager
|
||||
@@ -0,0 +1,262 @@
|
||||
"""Native Dropbox API service for cloud backend.
|
||||
|
||||
This service provides Dropbox operations using the native Python SDK,
|
||||
replacing rclone for server-side operations. It handles:
|
||||
- Folder listing for brand code calculation
|
||||
- File and folder uploads
|
||||
- Shared link generation
|
||||
|
||||
Credentials are loaded from Google Cloud Secret Manager.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
from google.cloud import secretmanager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DropboxService:
|
||||
"""Dropbox operations using native Python SDK."""
|
||||
|
||||
def __init__(self):
|
||||
self._client = None
|
||||
self._is_configured = False
|
||||
|
||||
def _load_credentials(self) -> Optional[dict]:
|
||||
"""Load OAuth credentials from Secret Manager."""
|
||||
try:
|
||||
client = secretmanager.SecretManagerServiceClient()
|
||||
# Try to get the project ID from environment or use default
|
||||
project_id = os.environ.get("GOOGLE_CLOUD_PROJECT", "karaoke-gen")
|
||||
name = f"projects/{project_id}/secrets/dropbox-oauth-credentials/versions/latest"
|
||||
response = client.access_secret_version(request={"name": name})
|
||||
return json.loads(response.payload.data.decode("UTF-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load Dropbox credentials from Secret Manager: {e}")
|
||||
return None
|
||||
|
||||
@property
|
||||
def is_configured(self) -> bool:
|
||||
"""Check if Dropbox credentials are available."""
|
||||
if self._is_configured:
|
||||
return True
|
||||
creds = self._load_credentials()
|
||||
self._is_configured = creds is not None and "access_token" in creds
|
||||
return self._is_configured
|
||||
|
||||
@property
|
||||
def client(self):
|
||||
"""Get or create Dropbox client."""
|
||||
if self._client is None:
|
||||
# Import here to avoid import errors if dropbox is not installed
|
||||
try:
|
||||
from dropbox import Dropbox
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"dropbox package is not installed. "
|
||||
"Install it with: pip install dropbox"
|
||||
)
|
||||
|
||||
creds = self._load_credentials()
|
||||
if not creds:
|
||||
raise RuntimeError("Dropbox credentials not configured in Secret Manager")
|
||||
|
||||
self._client = Dropbox(
|
||||
oauth2_access_token=creds["access_token"],
|
||||
oauth2_refresh_token=creds.get("refresh_token"),
|
||||
app_key=creds.get("app_key"),
|
||||
app_secret=creds.get("app_secret"),
|
||||
)
|
||||
return self._client
|
||||
|
||||
def list_folders(self, path: str) -> list[str]:
|
||||
"""
|
||||
List folder names at path for brand code calculation.
|
||||
|
||||
Args:
|
||||
path: Dropbox path to list (e.g., "/Karaoke/Tracks-Organized")
|
||||
|
||||
Returns:
|
||||
List of folder names in the path
|
||||
"""
|
||||
from dropbox.files import FolderMetadata
|
||||
|
||||
# Ensure path starts with /
|
||||
if not path.startswith("/"):
|
||||
path = f"/{path}"
|
||||
|
||||
logger.info(f"Listing folders at Dropbox path: {path}")
|
||||
|
||||
result = self.client.files_list_folder(path)
|
||||
folders = []
|
||||
|
||||
# Get all entries (handling pagination)
|
||||
while True:
|
||||
for entry in result.entries:
|
||||
if isinstance(entry, FolderMetadata):
|
||||
folders.append(entry.name)
|
||||
|
||||
if not result.has_more:
|
||||
break
|
||||
result = self.client.files_list_folder_continue(result.cursor)
|
||||
|
||||
logger.info(f"Found {len(folders)} folders in {path}")
|
||||
return folders
|
||||
|
||||
def get_next_brand_code(self, path: str, brand_prefix: str) -> str:
|
||||
"""
|
||||
Calculate next brand code from existing folders.
|
||||
|
||||
Scans the folder for existing brand codes like "NOMAD-0001" and
|
||||
returns the next sequential code.
|
||||
|
||||
Args:
|
||||
path: Dropbox path containing organized folders
|
||||
brand_prefix: Brand prefix (e.g., "NOMAD")
|
||||
|
||||
Returns:
|
||||
Next brand code (e.g., "NOMAD-1164")
|
||||
"""
|
||||
folders = self.list_folders(path)
|
||||
pattern = re.compile(rf"^{re.escape(brand_prefix)}-(\d{{4}})")
|
||||
|
||||
max_num = 0
|
||||
for folder in folders:
|
||||
match = pattern.match(folder)
|
||||
if match:
|
||||
num = int(match.group(1))
|
||||
max_num = max(max_num, num)
|
||||
|
||||
next_code = f"{brand_prefix}-{max_num + 1:04d}"
|
||||
logger.info(f"Next brand code for {brand_prefix}: {next_code} (max existing: {max_num})")
|
||||
return next_code
|
||||
|
||||
def upload_file(self, local_path: str, remote_path: str) -> None:
|
||||
"""
|
||||
Upload a single file to Dropbox.
|
||||
|
||||
Args:
|
||||
local_path: Local file path
|
||||
remote_path: Dropbox destination path (must start with /)
|
||||
"""
|
||||
from dropbox.files import WriteMode
|
||||
|
||||
# Ensure remote path starts with /
|
||||
if not remote_path.startswith("/"):
|
||||
remote_path = f"/{remote_path}"
|
||||
|
||||
file_size = os.path.getsize(local_path)
|
||||
logger.info(f"Uploading {local_path} ({file_size / 1024 / 1024:.1f} MB) to {remote_path}")
|
||||
|
||||
# For large files (>150MB), use upload sessions
|
||||
CHUNK_SIZE = 150 * 1024 * 1024 # 150 MB
|
||||
|
||||
with open(local_path, "rb") as f:
|
||||
if file_size <= CHUNK_SIZE:
|
||||
# Simple upload for smaller files
|
||||
self.client.files_upload(f.read(), remote_path, mode=WriteMode.overwrite)
|
||||
else:
|
||||
# Chunked upload for large files
|
||||
self._upload_large_file(f, remote_path, file_size)
|
||||
|
||||
logger.info(f"Successfully uploaded to {remote_path}")
|
||||
|
||||
def _upload_large_file(self, file_obj, remote_path: str, file_size: int) -> None:
|
||||
"""Upload a large file using upload sessions."""
|
||||
from dropbox.files import CommitInfo, UploadSessionCursor, WriteMode
|
||||
|
||||
CHUNK_SIZE = 150 * 1024 * 1024 # 150 MB
|
||||
|
||||
# Start upload session
|
||||
session_start = self.client.files_upload_session_start(file_obj.read(CHUNK_SIZE))
|
||||
cursor = UploadSessionCursor(
|
||||
session_id=session_start.session_id,
|
||||
offset=file_obj.tell(),
|
||||
)
|
||||
commit = CommitInfo(path=remote_path, mode=WriteMode.overwrite)
|
||||
|
||||
# Upload remaining chunks
|
||||
while file_obj.tell() < file_size:
|
||||
remaining = file_size - file_obj.tell()
|
||||
if remaining <= CHUNK_SIZE:
|
||||
# Final chunk
|
||||
self.client.files_upload_session_finish(
|
||||
file_obj.read(CHUNK_SIZE),
|
||||
cursor,
|
||||
commit,
|
||||
)
|
||||
else:
|
||||
# Intermediate chunk
|
||||
self.client.files_upload_session_append_v2(
|
||||
file_obj.read(CHUNK_SIZE),
|
||||
cursor,
|
||||
)
|
||||
cursor.offset = file_obj.tell()
|
||||
|
||||
def upload_folder(self, local_dir: str, remote_path: str) -> None:
|
||||
"""
|
||||
Upload all files in a directory to Dropbox folder.
|
||||
|
||||
Args:
|
||||
local_dir: Local directory to upload
|
||||
remote_path: Dropbox destination folder path
|
||||
"""
|
||||
# Ensure remote path starts with /
|
||||
if not remote_path.startswith("/"):
|
||||
remote_path = f"/{remote_path}"
|
||||
|
||||
logger.info(f"Uploading folder {local_dir} to {remote_path}")
|
||||
|
||||
uploaded_count = 0
|
||||
for filename in os.listdir(local_dir):
|
||||
local_file = os.path.join(local_dir, filename)
|
||||
if os.path.isfile(local_file):
|
||||
remote_file = f"{remote_path}/{filename}"
|
||||
self.upload_file(local_file, remote_file)
|
||||
uploaded_count += 1
|
||||
|
||||
logger.info(f"Uploaded {uploaded_count} files to {remote_path}")
|
||||
|
||||
def create_shared_link(self, path: str) -> str:
|
||||
"""
|
||||
Create and return a shared link for the path.
|
||||
|
||||
Args:
|
||||
path: Dropbox path to share
|
||||
|
||||
Returns:
|
||||
Shared link URL
|
||||
"""
|
||||
from dropbox.exceptions import ApiError
|
||||
from dropbox.sharing import SharedLinkSettings
|
||||
|
||||
# Ensure path starts with /
|
||||
if not path.startswith("/"):
|
||||
path = f"/{path}"
|
||||
|
||||
logger.info(f"Creating shared link for: {path}")
|
||||
|
||||
try:
|
||||
# Try to create a new shared link
|
||||
settings = SharedLinkSettings(requested_visibility=None)
|
||||
link = self.client.sharing_create_shared_link_with_settings(path, settings)
|
||||
logger.info(f"Created new shared link: {link.url}")
|
||||
return link.url
|
||||
except ApiError as e:
|
||||
# Link may already exist - try to get existing link
|
||||
if e.error.is_shared_link_already_exists():
|
||||
logger.info("Shared link already exists, retrieving existing link")
|
||||
links = self.client.sharing_list_shared_links(path=path, direct_only=True)
|
||||
if links.links:
|
||||
logger.info(f"Found existing shared link: {links.links[0].url}")
|
||||
return links.links[0].url
|
||||
raise
|
||||
|
||||
|
||||
def get_dropbox_service() -> DropboxService:
|
||||
"""Factory function to get a DropboxService instance."""
|
||||
return DropboxService()
|
||||
@@ -0,0 +1,343 @@
|
||||
"""
|
||||
Firestore database operations for job management.
|
||||
"""
|
||||
import logging
|
||||
from typing import Optional, Dict, Any, List
|
||||
from datetime import datetime
|
||||
from google.cloud import firestore
|
||||
from google.cloud.firestore_v1 import FieldFilter
|
||||
|
||||
from backend.config import settings
|
||||
from backend.models.job import Job, JobStatus, TimelineEvent
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FirestoreService:
|
||||
"""Service for Firestore database operations."""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize Firestore client."""
|
||||
self.db = firestore.Client(project=settings.google_cloud_project)
|
||||
self.collection = settings.firestore_collection
|
||||
self.tokens_collection = "auth_tokens" # Collection for access tokens
|
||||
|
||||
def create_job(self, job: Job) -> None:
|
||||
"""Create a new job in Firestore."""
|
||||
try:
|
||||
doc_ref = self.db.collection(self.collection).document(job.job_id)
|
||||
doc_ref.set(job.model_dump(mode='json'))
|
||||
logger.info(f"Created job {job.job_id} in Firestore")
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating job {job.job_id}: {e}")
|
||||
raise
|
||||
|
||||
def get_job(self, job_id: str) -> Optional[Job]:
|
||||
"""Get a job by ID."""
|
||||
try:
|
||||
doc_ref = self.db.collection(self.collection).document(job_id)
|
||||
doc = doc_ref.get()
|
||||
|
||||
logger.debug(f"Fetching job {job_id} from collection {self.collection}")
|
||||
logger.debug(f"Document exists: {doc.exists}")
|
||||
|
||||
if not doc.exists:
|
||||
return None
|
||||
|
||||
data = doc.to_dict()
|
||||
logger.debug(f"Document data keys: {list(data.keys()) if data else 'None'}")
|
||||
logger.debug(f"Document data: {data}")
|
||||
|
||||
return Job(**data)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting job {job_id}: {e}")
|
||||
raise
|
||||
|
||||
def update_job(self, job_id: str, updates: Dict[str, Any]) -> None:
|
||||
"""Update a job with partial data."""
|
||||
try:
|
||||
doc_ref = self.db.collection(self.collection).document(job_id)
|
||||
|
||||
# Add updated_at timestamp
|
||||
updates['updated_at'] = datetime.utcnow()
|
||||
|
||||
doc_ref.update(updates)
|
||||
logger.info(f"Updated job {job_id} in Firestore")
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating job {job_id}: {e}")
|
||||
raise
|
||||
|
||||
def update_job_status(
|
||||
self,
|
||||
job_id: str,
|
||||
status: JobStatus,
|
||||
progress: Optional[int] = None,
|
||||
message: Optional[str] = None,
|
||||
**additional_fields
|
||||
) -> None:
|
||||
"""Update job status and add timeline event."""
|
||||
try:
|
||||
doc_ref = self.db.collection(self.collection).document(job_id)
|
||||
|
||||
# Create timeline event
|
||||
timeline_event = TimelineEvent(
|
||||
status=status.value,
|
||||
timestamp=datetime.utcnow().isoformat(),
|
||||
progress=progress,
|
||||
message=message
|
||||
)
|
||||
|
||||
# Prepare updates
|
||||
updates = {
|
||||
'status': status.value,
|
||||
'updated_at': datetime.utcnow(),
|
||||
'timeline': firestore.ArrayUnion([timeline_event.model_dump(mode='json')])
|
||||
}
|
||||
|
||||
if progress is not None:
|
||||
updates['progress'] = progress
|
||||
|
||||
# Add any additional fields
|
||||
updates.update(additional_fields)
|
||||
|
||||
doc_ref.update(updates)
|
||||
logger.info(f"Updated job {job_id} status to {status.value}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating job status {job_id}: {e}")
|
||||
raise
|
||||
|
||||
def list_jobs(
|
||||
self,
|
||||
status: Optional[JobStatus] = None,
|
||||
environment: Optional[str] = None,
|
||||
client_id: Optional[str] = None,
|
||||
created_after: Optional[datetime] = None,
|
||||
created_before: Optional[datetime] = None,
|
||||
limit: int = 100
|
||||
) -> List[Job]:
|
||||
"""
|
||||
List jobs with optional filters.
|
||||
|
||||
Args:
|
||||
status: Filter by job status
|
||||
environment: Filter by request_metadata.environment (test/production/development)
|
||||
client_id: Filter by request_metadata.client_id
|
||||
created_after: Filter jobs created after this datetime
|
||||
created_before: Filter jobs created before this datetime
|
||||
limit: Maximum number of jobs to return
|
||||
|
||||
Returns:
|
||||
List of Job objects matching filters, ordered by created_at descending
|
||||
"""
|
||||
try:
|
||||
query = self.db.collection(self.collection)
|
||||
|
||||
if status:
|
||||
query = query.where(filter=FieldFilter('status', '==', status.value))
|
||||
|
||||
# Filter by request_metadata fields using dot notation
|
||||
if environment:
|
||||
query = query.where(filter=FieldFilter('request_metadata.environment', '==', environment))
|
||||
|
||||
if client_id:
|
||||
query = query.where(filter=FieldFilter('request_metadata.client_id', '==', client_id))
|
||||
|
||||
# Date range filters
|
||||
if created_after:
|
||||
query = query.where(filter=FieldFilter('created_at', '>=', created_after))
|
||||
|
||||
if created_before:
|
||||
query = query.where(filter=FieldFilter('created_at', '<=', created_before))
|
||||
|
||||
query = query.order_by('created_at', direction=firestore.Query.DESCENDING).limit(limit)
|
||||
|
||||
docs = query.stream()
|
||||
jobs = [Job(**doc.to_dict()) for doc in docs]
|
||||
|
||||
return jobs
|
||||
except Exception as e:
|
||||
logger.error(f"Error listing jobs: {e}")
|
||||
raise
|
||||
|
||||
def delete_jobs_by_filter(
|
||||
self,
|
||||
environment: Optional[str] = None,
|
||||
client_id: Optional[str] = None,
|
||||
status: Optional[JobStatus] = None,
|
||||
created_before: Optional[datetime] = None,
|
||||
) -> int:
|
||||
"""
|
||||
Delete multiple jobs matching filter criteria.
|
||||
|
||||
CAUTION: This is a destructive operation. Use carefully.
|
||||
|
||||
Args:
|
||||
environment: Delete jobs with this environment (e.g., "test")
|
||||
client_id: Delete jobs from this client
|
||||
status: Delete jobs with this status
|
||||
created_before: Delete jobs created before this datetime
|
||||
|
||||
Returns:
|
||||
Number of jobs deleted
|
||||
"""
|
||||
try:
|
||||
query = self.db.collection(self.collection)
|
||||
|
||||
if environment:
|
||||
query = query.where(filter=FieldFilter('request_metadata.environment', '==', environment))
|
||||
|
||||
if client_id:
|
||||
query = query.where(filter=FieldFilter('request_metadata.client_id', '==', client_id))
|
||||
|
||||
if status:
|
||||
query = query.where(filter=FieldFilter('status', '==', status.value))
|
||||
|
||||
if created_before:
|
||||
query = query.where(filter=FieldFilter('created_at', '<=', created_before))
|
||||
|
||||
# Get matching documents
|
||||
docs = list(query.stream())
|
||||
deleted_count = 0
|
||||
|
||||
# Delete in batches
|
||||
batch = self.db.batch()
|
||||
batch_count = 0
|
||||
|
||||
for doc in docs:
|
||||
batch.delete(doc.reference)
|
||||
batch_count += 1
|
||||
deleted_count += 1
|
||||
|
||||
# Firestore batch limit is 500
|
||||
if batch_count >= 500:
|
||||
batch.commit()
|
||||
batch = self.db.batch()
|
||||
batch_count = 0
|
||||
|
||||
# Commit any remaining deletes
|
||||
if batch_count > 0:
|
||||
batch.commit()
|
||||
|
||||
logger.info(f"Deleted {deleted_count} jobs matching filter criteria")
|
||||
return deleted_count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error deleting jobs by filter: {e}")
|
||||
raise
|
||||
|
||||
def delete_job(self, job_id: str) -> None:
|
||||
"""Delete a job from Firestore."""
|
||||
try:
|
||||
doc_ref = self.db.collection(self.collection).document(job_id)
|
||||
doc_ref.delete()
|
||||
logger.info(f"Deleted job {job_id} from Firestore")
|
||||
except Exception as e:
|
||||
logger.error(f"Error deleting job {job_id}: {e}")
|
||||
raise
|
||||
|
||||
def append_worker_log(self, job_id: str, log_entry: Dict[str, Any]) -> None:
|
||||
"""
|
||||
Atomically append a log entry to worker_logs using ArrayUnion.
|
||||
|
||||
This avoids the race condition of read-modify-write when multiple
|
||||
workers are logging concurrently.
|
||||
|
||||
Args:
|
||||
job_id: Job ID
|
||||
log_entry: Log entry dict with timestamp, level, worker, message
|
||||
"""
|
||||
try:
|
||||
doc_ref = self.db.collection(self.collection).document(job_id)
|
||||
doc_ref.update({
|
||||
'worker_logs': firestore.ArrayUnion([log_entry]),
|
||||
'updated_at': datetime.utcnow()
|
||||
})
|
||||
# Don't log every append - too spammy
|
||||
except Exception as e:
|
||||
# Log but don't raise - logging shouldn't break workers
|
||||
logger.debug(f"Error appending worker log for job {job_id}: {e}")
|
||||
|
||||
# ============================================
|
||||
# Token Management Methods
|
||||
# ============================================
|
||||
|
||||
def create_token(self, token: str, token_data: Dict[str, Any]) -> None:
|
||||
"""Create a new access token in Firestore."""
|
||||
try:
|
||||
doc_ref = self.db.collection(self.tokens_collection).document(token)
|
||||
doc_ref.set(token_data)
|
||||
logger.info(f"Created token in Firestore")
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating token: {e}")
|
||||
raise
|
||||
|
||||
def get_token(self, token: str) -> Optional[Dict[str, Any]]:
|
||||
"""Get token data by token string."""
|
||||
try:
|
||||
doc_ref = self.db.collection(self.tokens_collection).document(token)
|
||||
doc = doc_ref.get()
|
||||
|
||||
if not doc.exists:
|
||||
return None
|
||||
|
||||
return doc.to_dict()
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting token: {e}")
|
||||
return None
|
||||
|
||||
def update_token(self, token: str, updates: Dict[str, Any]) -> None:
|
||||
"""Update token data."""
|
||||
try:
|
||||
doc_ref = self.db.collection(self.tokens_collection).document(token)
|
||||
doc_ref.update(updates)
|
||||
logger.info(f"Updated token in Firestore")
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating token: {e}")
|
||||
raise
|
||||
|
||||
def increment_token_usage(self, token: str, job_id: str) -> None:
|
||||
"""Increment token usage count and add job to history."""
|
||||
try:
|
||||
doc_ref = self.db.collection(self.tokens_collection).document(token)
|
||||
|
||||
# Use Firestore transaction to ensure atomic increment
|
||||
@firestore.transactional
|
||||
def update_in_transaction(transaction, doc_ref):
|
||||
snapshot = doc_ref.get(transaction=transaction)
|
||||
if not snapshot.exists:
|
||||
raise ValueError("Token not found")
|
||||
|
||||
data = snapshot.to_dict()
|
||||
current_usage = data.get("usage_count", 0)
|
||||
jobs = data.get("jobs", [])
|
||||
|
||||
# Increment usage
|
||||
transaction.update(doc_ref, {
|
||||
"usage_count": current_usage + 1,
|
||||
"last_used": datetime.utcnow(),
|
||||
"jobs": firestore.ArrayUnion([{
|
||||
"job_id": job_id,
|
||||
"created_at": datetime.utcnow()
|
||||
}])
|
||||
})
|
||||
|
||||
transaction = self.db.transaction()
|
||||
update_in_transaction(transaction, doc_ref)
|
||||
|
||||
logger.info(f"Incremented token usage for job {job_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error incrementing token usage: {e}")
|
||||
raise
|
||||
|
||||
def list_tokens(self) -> List[Dict[str, Any]]:
|
||||
"""List all tokens (admin only)."""
|
||||
try:
|
||||
docs = self.db.collection(self.tokens_collection).stream()
|
||||
tokens = [doc.to_dict() for doc in docs]
|
||||
logger.info(f"Retrieved {len(tokens)} tokens from Firestore")
|
||||
return tokens
|
||||
except Exception as e:
|
||||
logger.error(f"Error listing tokens: {e}")
|
||||
return []
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
"""Native Google Drive API service for cloud backend.
|
||||
|
||||
This service provides Google Drive operations using the native API,
|
||||
for uploading files to public share folders. It handles:
|
||||
- Folder creation and lookup
|
||||
- File uploads with resumable upload support
|
||||
- Uploading to organized folder structure (MP4/, MP4-720p/, CDG/)
|
||||
|
||||
Credentials are loaded from Google Cloud Secret Manager and can be
|
||||
shared with YouTube credentials if scopes include drive.file.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from backend.config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GoogleDriveService:
|
||||
"""Google Drive operations using native API."""
|
||||
|
||||
# Secret Manager secret name for Google Drive credentials
|
||||
# Can be same as YouTube if scopes include drive.file
|
||||
GDRIVE_CREDENTIALS_SECRET = "gdrive-oauth-credentials"
|
||||
|
||||
def __init__(self):
|
||||
self.settings = get_settings()
|
||||
self._service = None
|
||||
self._credentials_data: Optional[Dict[str, Any]] = None
|
||||
self._loaded = False
|
||||
|
||||
def _load_credentials(self) -> Optional[Dict[str, Any]]:
|
||||
"""Load OAuth credentials from Secret Manager."""
|
||||
if self._loaded:
|
||||
return self._credentials_data
|
||||
|
||||
try:
|
||||
creds_json = self.settings.get_secret(self.GDRIVE_CREDENTIALS_SECRET)
|
||||
|
||||
if not creds_json:
|
||||
# Try falling back to YouTube credentials (may have drive scope)
|
||||
logger.info(
|
||||
"Google Drive credentials not found, trying YouTube credentials"
|
||||
)
|
||||
creds_json = self.settings.get_secret("youtube-oauth-credentials")
|
||||
|
||||
if not creds_json:
|
||||
logger.warning("Google Drive credentials not found in Secret Manager")
|
||||
self._loaded = True
|
||||
return None
|
||||
|
||||
self._credentials_data = json.loads(creds_json)
|
||||
|
||||
# Validate required fields
|
||||
required_fields = ["refresh_token", "client_id", "client_secret"]
|
||||
missing = [f for f in required_fields if not self._credentials_data.get(f)]
|
||||
|
||||
if missing:
|
||||
logger.error(f"Google Drive credentials missing required fields: {missing}")
|
||||
self._credentials_data = None
|
||||
self._loaded = True
|
||||
return None
|
||||
|
||||
logger.info("Google Drive credentials loaded successfully from Secret Manager")
|
||||
self._loaded = True
|
||||
return self._credentials_data
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"Failed to parse Google Drive credentials JSON: {e}")
|
||||
self._loaded = True
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load Google Drive credentials: {e}")
|
||||
self._loaded = True
|
||||
return None
|
||||
|
||||
@property
|
||||
def is_configured(self) -> bool:
|
||||
"""Check if Google Drive credentials are available."""
|
||||
creds = self._load_credentials()
|
||||
return creds is not None
|
||||
|
||||
@property
|
||||
def service(self):
|
||||
"""Get or create Google Drive service."""
|
||||
if self._service is None:
|
||||
# Import here to avoid import errors if google packages not installed
|
||||
try:
|
||||
from google.oauth2.credentials import Credentials
|
||||
from googleapiclient.discovery import build
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"google-api-python-client and google-auth packages are required. "
|
||||
"Install them with: pip install google-api-python-client google-auth"
|
||||
)
|
||||
|
||||
creds_data = self._load_credentials()
|
||||
if not creds_data:
|
||||
raise RuntimeError(
|
||||
"Google Drive credentials not configured in Secret Manager"
|
||||
)
|
||||
|
||||
# Create credentials object
|
||||
credentials = Credentials(
|
||||
token=creds_data.get("token"),
|
||||
refresh_token=creds_data.get("refresh_token"),
|
||||
token_uri=creds_data.get(
|
||||
"token_uri", "https://oauth2.googleapis.com/token"
|
||||
),
|
||||
client_id=creds_data.get("client_id"),
|
||||
client_secret=creds_data.get("client_secret"),
|
||||
scopes=creds_data.get(
|
||||
"scopes", ["https://www.googleapis.com/auth/drive.file"]
|
||||
),
|
||||
)
|
||||
|
||||
self._service = build("drive", "v3", credentials=credentials)
|
||||
logger.info("Google Drive service initialized successfully")
|
||||
|
||||
return self._service
|
||||
|
||||
def get_or_create_folder(self, parent_id: str, folder_name: str) -> str:
|
||||
"""
|
||||
Get existing folder or create new one, return folder ID.
|
||||
|
||||
Args:
|
||||
parent_id: Parent folder ID
|
||||
folder_name: Name of folder to find or create
|
||||
|
||||
Returns:
|
||||
Folder ID
|
||||
"""
|
||||
logger.info(f"Looking for folder '{folder_name}' in parent {parent_id}")
|
||||
|
||||
# Search for existing folder
|
||||
query = (
|
||||
f"name='{folder_name}' and '{parent_id}' in parents "
|
||||
f"and mimeType='application/vnd.google-apps.folder' and trashed=false"
|
||||
)
|
||||
results = self.service.files().list(q=query, fields="files(id, name)").execute()
|
||||
|
||||
if results.get("files"):
|
||||
folder_id = results["files"][0]["id"]
|
||||
logger.info(f"Found existing folder '{folder_name}': {folder_id}")
|
||||
return folder_id
|
||||
|
||||
# Create folder
|
||||
logger.info(f"Creating new folder '{folder_name}'")
|
||||
metadata = {
|
||||
"name": folder_name,
|
||||
"mimeType": "application/vnd.google-apps.folder",
|
||||
"parents": [parent_id],
|
||||
}
|
||||
folder = self.service.files().create(body=metadata, fields="id").execute()
|
||||
folder_id = folder["id"]
|
||||
logger.info(f"Created folder '{folder_name}': {folder_id}")
|
||||
return folder_id
|
||||
|
||||
def upload_file(
|
||||
self,
|
||||
local_path: str,
|
||||
parent_id: str,
|
||||
filename: str,
|
||||
replace_existing: bool = True,
|
||||
) -> str:
|
||||
"""
|
||||
Upload a file to a specific Drive folder.
|
||||
|
||||
Args:
|
||||
local_path: Local file path
|
||||
parent_id: Parent folder ID in Google Drive
|
||||
filename: Name for the file in Drive
|
||||
replace_existing: If True, delete existing file with same name first
|
||||
|
||||
Returns:
|
||||
File ID of uploaded file
|
||||
"""
|
||||
from googleapiclient.http import MediaFileUpload
|
||||
|
||||
file_size = os.path.getsize(local_path)
|
||||
logger.info(
|
||||
f"Uploading {local_path} ({file_size / 1024 / 1024:.1f} MB) "
|
||||
f"as '{filename}' to folder {parent_id}"
|
||||
)
|
||||
|
||||
# Determine MIME type
|
||||
ext = os.path.splitext(local_path)[1].lower()
|
||||
mime_types = {
|
||||
".mp4": "video/mp4",
|
||||
".mkv": "video/x-matroska",
|
||||
".zip": "application/zip",
|
||||
".flac": "audio/flac",
|
||||
".mp3": "audio/mpeg",
|
||||
".wav": "audio/wav",
|
||||
".png": "image/png",
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
}
|
||||
mime_type = mime_types.get(ext, "application/octet-stream")
|
||||
|
||||
# Check for existing file with same name
|
||||
if replace_existing:
|
||||
query = (
|
||||
f"name='{filename}' and '{parent_id}' in parents and trashed=false"
|
||||
)
|
||||
results = self.service.files().list(q=query, fields="files(id)").execute()
|
||||
for existing_file in results.get("files", []):
|
||||
logger.info(f"Deleting existing file: {existing_file['id']}")
|
||||
self.service.files().delete(fileId=existing_file["id"]).execute()
|
||||
|
||||
# Upload file
|
||||
metadata = {"name": filename, "parents": [parent_id]}
|
||||
media = MediaFileUpload(local_path, mimetype=mime_type, resumable=True)
|
||||
|
||||
file_result = (
|
||||
self.service.files().create(body=metadata, media_body=media, fields="id").execute()
|
||||
)
|
||||
file_id = file_result["id"]
|
||||
logger.info(f"Successfully uploaded '{filename}': {file_id}")
|
||||
return file_id
|
||||
|
||||
def upload_to_public_share(
|
||||
self,
|
||||
root_folder_id: str,
|
||||
brand_code: str,
|
||||
base_name: str,
|
||||
output_files: dict,
|
||||
) -> Dict[str, str]:
|
||||
"""
|
||||
Upload final files to public share folder structure.
|
||||
|
||||
Creates/uses subfolders:
|
||||
- MP4/{brand_code} - {base_name}.mp4 (lossy 4k)
|
||||
- MP4-720p/{brand_code} - {base_name}.mp4
|
||||
- CDG/{brand_code} - {base_name}.zip
|
||||
|
||||
Args:
|
||||
root_folder_id: Google Drive folder ID for public share root
|
||||
brand_code: Brand code (e.g., "NOMAD-1163")
|
||||
base_name: Base filename (e.g., "Artist - Title")
|
||||
output_files: Dictionary with output file paths:
|
||||
- final_karaoke_lossy_mp4: 4K lossy MP4
|
||||
- final_karaoke_lossy_720p_mp4: 720p lossy MP4
|
||||
- final_karaoke_cdg_zip: CDG package ZIP
|
||||
|
||||
Returns:
|
||||
Dictionary mapping category to uploaded file ID
|
||||
"""
|
||||
filename_base = f"{brand_code} - {base_name}"
|
||||
uploaded_files = {}
|
||||
|
||||
logger.info(
|
||||
f"Uploading public share files to Google Drive folder {root_folder_id}"
|
||||
)
|
||||
logger.info(f"Filename base: {filename_base}")
|
||||
|
||||
# Upload lossy 4k to MP4/
|
||||
lossy_mp4_path = output_files.get("final_karaoke_lossy_mp4")
|
||||
if lossy_mp4_path and os.path.exists(lossy_mp4_path):
|
||||
mp4_folder_id = self.get_or_create_folder(root_folder_id, "MP4")
|
||||
file_id = self.upload_file(
|
||||
lossy_mp4_path,
|
||||
mp4_folder_id,
|
||||
f"{filename_base}.mp4",
|
||||
)
|
||||
uploaded_files["mp4"] = file_id
|
||||
logger.info(f"Uploaded 4K MP4 to MP4/ folder")
|
||||
|
||||
# Upload 720p to MP4-720p/
|
||||
mp4_720p_path = output_files.get("final_karaoke_lossy_720p_mp4")
|
||||
if mp4_720p_path and os.path.exists(mp4_720p_path):
|
||||
mp4_720_folder_id = self.get_or_create_folder(root_folder_id, "MP4-720p")
|
||||
file_id = self.upload_file(
|
||||
mp4_720p_path,
|
||||
mp4_720_folder_id,
|
||||
f"{filename_base}.mp4",
|
||||
)
|
||||
uploaded_files["mp4_720p"] = file_id
|
||||
logger.info(f"Uploaded 720p MP4 to MP4-720p/ folder")
|
||||
|
||||
# Upload CDG ZIP to CDG/
|
||||
cdg_zip_path = output_files.get("final_karaoke_cdg_zip")
|
||||
if cdg_zip_path and os.path.exists(cdg_zip_path):
|
||||
cdg_folder_id = self.get_or_create_folder(root_folder_id, "CDG")
|
||||
file_id = self.upload_file(
|
||||
cdg_zip_path,
|
||||
cdg_folder_id,
|
||||
f"{filename_base}.zip",
|
||||
)
|
||||
uploaded_files["cdg"] = file_id
|
||||
logger.info(f"Uploaded CDG ZIP to CDG/ folder")
|
||||
|
||||
logger.info(f"Public share upload complete: {len(uploaded_files)} files uploaded")
|
||||
return uploaded_files
|
||||
|
||||
|
||||
# Singleton instance
|
||||
_gdrive_service: Optional[GoogleDriveService] = None
|
||||
|
||||
|
||||
def get_gdrive_service() -> GoogleDriveService:
|
||||
"""Get the singleton Google Drive service instance."""
|
||||
global _gdrive_service
|
||||
if _gdrive_service is None:
|
||||
_gdrive_service = GoogleDriveService()
|
||||
return _gdrive_service
|
||||
@@ -0,0 +1,258 @@
|
||||
"""
|
||||
Job-aware logging utilities.
|
||||
|
||||
This module provides a context manager and logging handler that captures
|
||||
Python logging output and forwards it to Firestore for real-time streaming
|
||||
to the CLI.
|
||||
|
||||
Usage:
|
||||
with JobLogContext(job_id, worker="review") as log_context:
|
||||
# Any logging within this block will be captured and sent to Firestore
|
||||
logger.info("Processing...") # This will be stored and streamed to CLI
|
||||
"""
|
||||
import logging
|
||||
import threading
|
||||
from typing import Optional
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
|
||||
from backend.services.firestore_service import FirestoreService
|
||||
|
||||
|
||||
class JobLogHandler(logging.Handler):
|
||||
"""
|
||||
A logging handler that forwards log records to Firestore for job tracking.
|
||||
|
||||
This enables real-time log streaming to the CLI during long-running operations
|
||||
like lyrics correction, preview generation, etc.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
job_id: str,
|
||||
worker: str = "review",
|
||||
firestore: Optional[FirestoreService] = None,
|
||||
min_level: int = logging.INFO,
|
||||
batch_size: int = 1, # Send immediately for real-time experience
|
||||
):
|
||||
"""
|
||||
Initialize the job log handler.
|
||||
|
||||
Args:
|
||||
job_id: The job ID to associate logs with
|
||||
worker: Worker name (e.g., "review", "add-lyrics", "preview")
|
||||
firestore: FirestoreService instance (creates one if not provided)
|
||||
min_level: Minimum log level to capture (default INFO)
|
||||
batch_size: Number of logs to batch before sending (1 = immediate)
|
||||
"""
|
||||
super().__init__()
|
||||
self.job_id = job_id
|
||||
self.worker = worker
|
||||
self.firestore = firestore or FirestoreService()
|
||||
self.min_level = min_level
|
||||
self.batch_size = batch_size
|
||||
self._log_buffer = []
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# Set the handler's level
|
||||
self.setLevel(min_level)
|
||||
|
||||
# Create a formatter that extracts the useful parts
|
||||
self.setFormatter(logging.Formatter('%(message)s'))
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
"""
|
||||
Process a log record and send it to Firestore.
|
||||
|
||||
This method is called by the logging framework for each log message.
|
||||
"""
|
||||
try:
|
||||
# Skip if below minimum level
|
||||
if record.levelno < self.min_level:
|
||||
return
|
||||
|
||||
# Format the message
|
||||
message = self.format(record)
|
||||
|
||||
# Truncate very long messages
|
||||
if len(message) > 1000:
|
||||
message = message[:997] + "..."
|
||||
|
||||
# Create log entry
|
||||
log_entry = {
|
||||
'timestamp': datetime.utcnow().isoformat() + 'Z',
|
||||
'level': record.levelname,
|
||||
'worker': self.worker,
|
||||
'message': message,
|
||||
}
|
||||
|
||||
# Add to buffer
|
||||
with self._lock:
|
||||
self._log_buffer.append(log_entry)
|
||||
|
||||
# Flush if batch size reached
|
||||
if len(self._log_buffer) >= self.batch_size:
|
||||
self._flush_buffer()
|
||||
|
||||
except Exception:
|
||||
# Never let logging errors propagate
|
||||
self.handleError(record)
|
||||
|
||||
def _flush_buffer(self) -> None:
|
||||
"""Flush buffered logs to Firestore."""
|
||||
if not self._log_buffer:
|
||||
return
|
||||
|
||||
try:
|
||||
for log_entry in self._log_buffer:
|
||||
self.firestore.append_worker_log(self.job_id, log_entry)
|
||||
self._log_buffer = []
|
||||
except Exception:
|
||||
# If Firestore fails, just clear buffer and continue
|
||||
self._log_buffer = []
|
||||
|
||||
def flush(self) -> None:
|
||||
"""Flush any remaining buffered logs."""
|
||||
with self._lock:
|
||||
self._flush_buffer()
|
||||
|
||||
def close(self) -> None:
|
||||
"""Clean up the handler."""
|
||||
self.flush()
|
||||
super().close()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def job_log_context(
|
||||
job_id: str,
|
||||
worker: str = "review",
|
||||
logger_names: Optional[list] = None,
|
||||
min_level: int = logging.INFO,
|
||||
):
|
||||
"""
|
||||
Context manager that captures logs and forwards them to Firestore.
|
||||
|
||||
This allows the CLI to see real-time logs from operations like
|
||||
add-lyrics, preview generation, etc.
|
||||
|
||||
Args:
|
||||
job_id: Job ID to associate logs with
|
||||
worker: Worker name for log categorization
|
||||
logger_names: List of logger names to capture (None = capture common loggers)
|
||||
min_level: Minimum log level to capture
|
||||
|
||||
Usage:
|
||||
with job_log_context(job_id, worker="add-lyrics"):
|
||||
# All logging in this block will be captured
|
||||
correction_operations.add_lyrics_source(...)
|
||||
"""
|
||||
# Default loggers to capture - these cover most of our processing
|
||||
# Note: We intentionally DON'T include the root logger ('') to avoid
|
||||
# duplicate log entries since logs propagate up the logger hierarchy.
|
||||
# Only capture at the top-level package loggers.
|
||||
if logger_names is None:
|
||||
logger_names = [
|
||||
'backend.api.routes.review',
|
||||
'lyrics_transcriber',
|
||||
'karaoke_gen',
|
||||
]
|
||||
|
||||
# Create the handler
|
||||
handler = JobLogHandler(
|
||||
job_id=job_id,
|
||||
worker=worker,
|
||||
min_level=min_level,
|
||||
)
|
||||
|
||||
# Add handler to all specified loggers
|
||||
loggers = []
|
||||
for name in logger_names:
|
||||
log = logging.getLogger(name) if name else logging.getLogger()
|
||||
log.addHandler(handler)
|
||||
loggers.append(log)
|
||||
|
||||
try:
|
||||
# Log start
|
||||
handler.emit(logging.LogRecord(
|
||||
name='job_log_context',
|
||||
level=logging.INFO,
|
||||
pathname='',
|
||||
lineno=0,
|
||||
msg=f"🚀 Starting {worker} operation for job {job_id}",
|
||||
args=(),
|
||||
exc_info=None,
|
||||
))
|
||||
|
||||
yield handler
|
||||
|
||||
finally:
|
||||
# Log completion
|
||||
handler.emit(logging.LogRecord(
|
||||
name='job_log_context',
|
||||
level=logging.INFO,
|
||||
pathname='',
|
||||
lineno=0,
|
||||
msg=f"✅ Completed {worker} operation for job {job_id}",
|
||||
args=(),
|
||||
exc_info=None,
|
||||
))
|
||||
|
||||
# Flush remaining logs
|
||||
handler.flush()
|
||||
|
||||
# Remove handler from all loggers
|
||||
for log in loggers:
|
||||
log.removeHandler(handler)
|
||||
|
||||
# Close handler
|
||||
handler.close()
|
||||
|
||||
|
||||
class JobLogger:
|
||||
"""
|
||||
A convenience class for direct logging to a job's log stream.
|
||||
|
||||
Use this when you want to log specific messages without capturing
|
||||
all logging output.
|
||||
"""
|
||||
|
||||
def __init__(self, job_id: str, worker: str = "review"):
|
||||
"""
|
||||
Initialize job logger.
|
||||
|
||||
Args:
|
||||
job_id: Job ID to log to
|
||||
worker: Worker name for log categorization
|
||||
"""
|
||||
self.job_id = job_id
|
||||
self.worker = worker
|
||||
self.firestore = FirestoreService()
|
||||
|
||||
def _log(self, level: str, message: str) -> None:
|
||||
"""Internal logging method."""
|
||||
log_entry = {
|
||||
'timestamp': datetime.utcnow().isoformat() + 'Z',
|
||||
'level': level,
|
||||
'worker': self.worker,
|
||||
'message': message[:1000],
|
||||
}
|
||||
try:
|
||||
self.firestore.append_worker_log(self.job_id, log_entry)
|
||||
except Exception:
|
||||
pass # Don't let logging errors propagate
|
||||
|
||||
def info(self, message: str) -> None:
|
||||
"""Log an INFO message."""
|
||||
self._log('INFO', message)
|
||||
|
||||
def warning(self, message: str) -> None:
|
||||
"""Log a WARNING message."""
|
||||
self._log('WARNING', message)
|
||||
|
||||
def error(self, message: str) -> None:
|
||||
"""Log an ERROR message."""
|
||||
self._log('ERROR', message)
|
||||
|
||||
def debug(self, message: str) -> None:
|
||||
"""Log a DEBUG message."""
|
||||
self._log('DEBUG', message)
|
||||
@@ -0,0 +1,592 @@
|
||||
"""
|
||||
Job management and queue operations.
|
||||
|
||||
This module handles the complete job lifecycle including:
|
||||
- Job creation and initialization
|
||||
- State transitions and validation
|
||||
- Worker coordination (parallel audio + lyrics processing)
|
||||
- Progress tracking and timeline events
|
||||
- Error handling and retries
|
||||
"""
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Optional, Dict, Any, List
|
||||
|
||||
from backend.models.job import Job, JobStatus, JobCreate, STATE_TRANSITIONS
|
||||
from backend.services.firestore_service import FirestoreService
|
||||
from backend.services.storage_service import StorageService
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class JobManager:
|
||||
"""Manager for job lifecycle and state."""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize job manager with required services."""
|
||||
self.firestore = FirestoreService()
|
||||
self.storage = StorageService()
|
||||
|
||||
def create_job(self, job_create: JobCreate) -> Job:
|
||||
"""
|
||||
Create a new job with initial state PENDING.
|
||||
|
||||
Jobs start in PENDING state and transition to DOWNLOADING
|
||||
when a worker picks them up.
|
||||
"""
|
||||
job_id = str(uuid.uuid4())[:8]
|
||||
|
||||
now = datetime.utcnow()
|
||||
job = Job(
|
||||
job_id=job_id,
|
||||
status=JobStatus.PENDING, # New state machine starts with PENDING
|
||||
progress=0,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
url=str(job_create.url) if job_create.url else None,
|
||||
artist=job_create.artist,
|
||||
title=job_create.title,
|
||||
# User preferences
|
||||
enable_cdg=job_create.enable_cdg,
|
||||
enable_txt=job_create.enable_txt,
|
||||
enable_youtube_upload=job_create.enable_youtube_upload,
|
||||
youtube_description=job_create.youtube_description,
|
||||
webhook_url=job_create.webhook_url,
|
||||
user_email=job_create.user_email,
|
||||
# Request metadata (for tracking and filtering)
|
||||
request_metadata=job_create.request_metadata,
|
||||
)
|
||||
|
||||
self.firestore.create_job(job)
|
||||
logger.info(f"Created new job {job_id} with status PENDING")
|
||||
|
||||
return job
|
||||
|
||||
def get_job(self, job_id: str) -> Optional[Job]:
|
||||
"""Get a job by ID."""
|
||||
return self.firestore.get_job(job_id)
|
||||
|
||||
def update_job_status(
|
||||
self,
|
||||
job_id: str,
|
||||
status: JobStatus,
|
||||
progress: Optional[int] = None,
|
||||
message: Optional[str] = None,
|
||||
**kwargs
|
||||
) -> None:
|
||||
"""Update job status with timeline tracking."""
|
||||
self.firestore.update_job_status(
|
||||
job_id=job_id,
|
||||
status=status,
|
||||
progress=progress,
|
||||
message=message,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
def update_job(self, job_id: str, updates: Dict[str, Any]) -> None:
|
||||
"""Update job with arbitrary fields."""
|
||||
self.firestore.update_job(job_id, updates)
|
||||
|
||||
def list_jobs(
|
||||
self,
|
||||
status: Optional[JobStatus] = None,
|
||||
environment: Optional[str] = None,
|
||||
client_id: Optional[str] = None,
|
||||
created_after: Optional[datetime] = None,
|
||||
created_before: Optional[datetime] = None,
|
||||
limit: int = 100
|
||||
) -> List[Job]:
|
||||
"""
|
||||
List jobs with optional filtering.
|
||||
|
||||
Args:
|
||||
status: Filter by job status
|
||||
environment: Filter by request_metadata.environment (test/production/development)
|
||||
client_id: Filter by request_metadata.client_id (customer identifier)
|
||||
created_after: Filter jobs created after this datetime
|
||||
created_before: Filter jobs created before this datetime
|
||||
limit: Maximum number of jobs to return
|
||||
|
||||
Returns:
|
||||
List of Job objects matching filters
|
||||
"""
|
||||
return self.firestore.list_jobs(
|
||||
status=status,
|
||||
environment=environment,
|
||||
client_id=client_id,
|
||||
created_after=created_after,
|
||||
created_before=created_before,
|
||||
limit=limit
|
||||
)
|
||||
|
||||
def delete_jobs_by_filter(
|
||||
self,
|
||||
environment: Optional[str] = None,
|
||||
client_id: Optional[str] = None,
|
||||
status: Optional[JobStatus] = None,
|
||||
created_before: Optional[datetime] = None,
|
||||
delete_files: bool = True
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Delete multiple jobs matching filter criteria.
|
||||
|
||||
CAUTION: This is a destructive operation. Use carefully.
|
||||
|
||||
Args:
|
||||
environment: Delete jobs with this environment (e.g., "test")
|
||||
client_id: Delete jobs from this client
|
||||
status: Delete jobs with this status
|
||||
created_before: Delete jobs created before this datetime
|
||||
delete_files: Also delete GCS files (default True)
|
||||
|
||||
Returns:
|
||||
Dict with deletion statistics
|
||||
"""
|
||||
# First get matching jobs to delete their files
|
||||
if delete_files:
|
||||
jobs = self.firestore.list_jobs(
|
||||
status=status,
|
||||
environment=environment,
|
||||
client_id=client_id,
|
||||
created_before=created_before,
|
||||
limit=10000 # High limit for deletion
|
||||
)
|
||||
|
||||
files_deleted = 0
|
||||
for job in jobs:
|
||||
# Delete files from various locations
|
||||
try:
|
||||
# Delete uploads folder
|
||||
self.storage.delete_folder(f"uploads/{job.job_id}/")
|
||||
# Delete jobs folder
|
||||
self.storage.delete_folder(f"jobs/{job.job_id}/")
|
||||
files_deleted += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"Error deleting files for job {job.job_id}: {e}")
|
||||
|
||||
# Delete the jobs from Firestore
|
||||
deleted_count = self.firestore.delete_jobs_by_filter(
|
||||
environment=environment,
|
||||
client_id=client_id,
|
||||
status=status,
|
||||
created_before=created_before
|
||||
)
|
||||
|
||||
return {
|
||||
'jobs_deleted': deleted_count,
|
||||
'files_deleted': files_deleted if delete_files else 0
|
||||
}
|
||||
|
||||
def mark_job_error(self, job_id: str, error_message: str) -> None:
|
||||
"""Mark a job as errored."""
|
||||
self.firestore.update_job_status(
|
||||
job_id=job_id,
|
||||
status=JobStatus.ERROR,
|
||||
progress=0,
|
||||
message=error_message,
|
||||
error_message=error_message
|
||||
)
|
||||
logger.error(f"Job {job_id} marked as error: {error_message}")
|
||||
|
||||
def get_output_urls(self, job_id: str) -> Dict[str, str]:
|
||||
"""Generate signed URLs for job output files."""
|
||||
job = self.get_job(job_id)
|
||||
if not job or not job.output_files:
|
||||
return {}
|
||||
|
||||
urls = {}
|
||||
for file_type, gcs_path in job.output_files.items():
|
||||
try:
|
||||
urls[file_type] = self.storage.generate_signed_url(gcs_path, expiration_minutes=120)
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating URL for {file_type}: {e}")
|
||||
|
||||
return urls
|
||||
|
||||
def delete_job(self, job_id: str, delete_files: bool = True) -> None:
|
||||
"""Delete a job and optionally its files."""
|
||||
if delete_files:
|
||||
job = self.get_job(job_id)
|
||||
if job and job.output_files:
|
||||
for gcs_path in job.output_files.values():
|
||||
try:
|
||||
self.storage.delete_file(gcs_path)
|
||||
except Exception as e:
|
||||
logger.error(f"Error deleting file {gcs_path}: {e}")
|
||||
|
||||
self.firestore.delete_job(job_id)
|
||||
logger.info(f"Deleted job {job_id}")
|
||||
|
||||
def validate_state_transition(self, job_id: str, new_status: JobStatus) -> bool:
|
||||
"""
|
||||
Validate that a state transition is legal.
|
||||
|
||||
Returns:
|
||||
True if transition is valid, False otherwise
|
||||
"""
|
||||
job = self.get_job(job_id)
|
||||
if not job:
|
||||
logger.error(f"Job {job_id} not found")
|
||||
return False
|
||||
|
||||
current_status = job.status
|
||||
valid_transitions = STATE_TRANSITIONS.get(current_status, [])
|
||||
|
||||
if new_status not in valid_transitions:
|
||||
logger.error(
|
||||
f"Invalid state transition for job {job_id}: "
|
||||
f"{current_status} -> {new_status}. "
|
||||
f"Valid transitions: {valid_transitions}"
|
||||
)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def transition_to_state(
|
||||
self,
|
||||
job_id: str,
|
||||
new_status: JobStatus,
|
||||
progress: Optional[int] = None,
|
||||
message: Optional[str] = None,
|
||||
state_data_updates: Optional[Dict[str, Any]] = None
|
||||
) -> bool:
|
||||
"""
|
||||
Transition job to new state with validation.
|
||||
|
||||
Args:
|
||||
job_id: Job ID
|
||||
new_status: Target state
|
||||
progress: Progress percentage (0-100)
|
||||
message: Timeline message
|
||||
state_data_updates: Updates to state_data field
|
||||
|
||||
Returns:
|
||||
True if transition succeeded, False otherwise
|
||||
"""
|
||||
if not self.validate_state_transition(job_id, new_status):
|
||||
return False
|
||||
|
||||
updates = {
|
||||
'status': new_status,
|
||||
'updated_at': datetime.utcnow()
|
||||
}
|
||||
|
||||
if progress is not None:
|
||||
updates['progress'] = progress
|
||||
|
||||
# If we have state_data_updates, merge them with existing state_data
|
||||
merged_state_data = None
|
||||
if state_data_updates:
|
||||
job = self.get_job(job_id)
|
||||
if job:
|
||||
merged_state_data = {**job.state_data, **state_data_updates}
|
||||
|
||||
# Update job status (includes timeline event), passing state_data if present
|
||||
if merged_state_data is not None:
|
||||
self.update_job_status(
|
||||
job_id=job_id,
|
||||
status=new_status,
|
||||
progress=progress,
|
||||
message=message,
|
||||
state_data=merged_state_data
|
||||
)
|
||||
else:
|
||||
self.update_job_status(
|
||||
job_id=job_id,
|
||||
status=new_status,
|
||||
progress=progress,
|
||||
message=message
|
||||
)
|
||||
|
||||
logger.info(f"Job {job_id} transitioned to {new_status}")
|
||||
return True
|
||||
|
||||
def update_state_data(self, job_id: str, key: str, value: Any) -> None:
|
||||
"""
|
||||
Update a specific key in the job's state_data field.
|
||||
|
||||
This is used by workers to store stage-specific metadata.
|
||||
"""
|
||||
job = self.get_job(job_id)
|
||||
if not job:
|
||||
logger.error(f"Job {job_id} not found")
|
||||
return
|
||||
|
||||
state_data = job.state_data.copy()
|
||||
state_data[key] = value
|
||||
|
||||
self.update_job(job_id, {'state_data': state_data})
|
||||
logger.debug(f"Job {job_id} state_data updated: {key} = {value}")
|
||||
|
||||
def fail_job(self, job_id: str, error_message: str, error_details: Optional[Dict[str, Any]] = None) -> bool:
|
||||
"""
|
||||
Mark a job as failed with error information.
|
||||
|
||||
Args:
|
||||
job_id: Job ID
|
||||
error_message: Human-readable error message
|
||||
error_details: Optional structured error details
|
||||
|
||||
Returns:
|
||||
True if successful
|
||||
"""
|
||||
try:
|
||||
# Update error fields
|
||||
self.update_job(job_id, {
|
||||
'error_message': error_message,
|
||||
'error_details': error_details or {}
|
||||
})
|
||||
|
||||
# Use update_job_status which handles timeline
|
||||
self.update_job_status(
|
||||
job_id=job_id,
|
||||
status=JobStatus.FAILED,
|
||||
message=error_message
|
||||
)
|
||||
|
||||
logger.error(f"Job {job_id} failed: {error_message}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error marking job {job_id} as failed: {e}")
|
||||
return False
|
||||
|
||||
def update_file_url(self, job_id: str, category: str, file_type: str, url: str) -> None:
|
||||
"""
|
||||
Update a file URL in the job's file_urls structure.
|
||||
|
||||
Args:
|
||||
job_id: Job ID
|
||||
category: Category (e.g., "stems", "lyrics", "finals")
|
||||
file_type: File type within category (e.g., "clean", "lrc", "lossless_4k_mp4")
|
||||
url: GCS URL
|
||||
"""
|
||||
job = self.get_job(job_id)
|
||||
if not job:
|
||||
logger.error(f"Job {job_id} not found")
|
||||
return
|
||||
|
||||
file_urls = job.file_urls.copy()
|
||||
if category not in file_urls:
|
||||
file_urls[category] = {}
|
||||
|
||||
file_urls[category][file_type] = url
|
||||
|
||||
self.update_job(job_id, {'file_urls': file_urls})
|
||||
logger.debug(f"Job {job_id} file URL updated: {category}.{file_type}")
|
||||
|
||||
def check_parallel_processing_complete(self, job_id: str) -> bool:
|
||||
"""
|
||||
Check if both parallel tracks (audio + lyrics) are complete.
|
||||
|
||||
This is called after audio_complete or lyrics_complete to determine
|
||||
if we can proceed to screen generation.
|
||||
|
||||
Returns:
|
||||
True if both tracks complete, False otherwise
|
||||
"""
|
||||
job = self.get_job(job_id)
|
||||
if not job:
|
||||
return False
|
||||
|
||||
audio_complete = job.state_data.get('audio_complete', False)
|
||||
lyrics_complete = job.state_data.get('lyrics_complete', False)
|
||||
|
||||
return audio_complete and lyrics_complete
|
||||
|
||||
def mark_audio_complete(self, job_id: str) -> None:
|
||||
"""
|
||||
Mark audio processing as complete and check if can proceed.
|
||||
|
||||
If lyrics are also complete, automatically triggers screens worker.
|
||||
"""
|
||||
self.update_state_data(job_id, 'audio_complete', True)
|
||||
|
||||
if self.check_parallel_processing_complete(job_id):
|
||||
logger.info(f"Job {job_id}: Both audio and lyrics complete, triggering screens worker")
|
||||
# Transition happens in screens worker
|
||||
# We just trigger it here
|
||||
self._trigger_screens_worker(job_id)
|
||||
|
||||
def mark_lyrics_complete(self, job_id: str) -> None:
|
||||
"""
|
||||
Mark lyrics processing as complete and check if can proceed.
|
||||
|
||||
If audio is also complete, automatically triggers screens worker.
|
||||
"""
|
||||
self.update_state_data(job_id, 'lyrics_complete', True)
|
||||
|
||||
if self.check_parallel_processing_complete(job_id):
|
||||
logger.info(f"Job {job_id}: Both audio and lyrics complete, triggering screens worker")
|
||||
# Transition happens in screens worker
|
||||
# We just trigger it here
|
||||
self._trigger_screens_worker(job_id)
|
||||
|
||||
def _trigger_screens_worker(self, job_id: str) -> None:
|
||||
"""
|
||||
Trigger screens generation worker.
|
||||
|
||||
Uses WorkerService to make HTTP call to internal API.
|
||||
This must be async, so we use asyncio to create a task.
|
||||
"""
|
||||
import asyncio
|
||||
from backend.services.worker_service import get_worker_service
|
||||
|
||||
logger.info(f"Job {job_id}: Triggering screens worker")
|
||||
|
||||
# Create async task to trigger worker
|
||||
# This allows us to call async code from sync context
|
||||
async def _trigger():
|
||||
worker_service = get_worker_service()
|
||||
await worker_service.trigger_screens_worker(job_id)
|
||||
|
||||
# Create task in event loop
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
if loop.is_running():
|
||||
# If loop is already running, create task
|
||||
asyncio.create_task(_trigger())
|
||||
else:
|
||||
# If no loop, run directly
|
||||
asyncio.run(_trigger())
|
||||
except RuntimeError:
|
||||
# Fallback: just log
|
||||
logger.warning(f"Job {job_id}: Could not trigger screens worker (no event loop)")
|
||||
# TODO: In production, use message queue instead
|
||||
|
||||
def cancel_job(self, job_id: str, reason: Optional[str] = None) -> bool:
|
||||
"""
|
||||
Cancel a job.
|
||||
|
||||
Only jobs in non-terminal states can be cancelled.
|
||||
"""
|
||||
job = self.get_job(job_id)
|
||||
if not job:
|
||||
return False
|
||||
|
||||
terminal_states = [JobStatus.COMPLETE, JobStatus.FAILED, JobStatus.CANCELLED]
|
||||
if job.status in terminal_states:
|
||||
logger.warning(f"Cannot cancel job {job_id} in terminal state {job.status}")
|
||||
return False
|
||||
|
||||
message = f"Job cancelled{f': {reason}' if reason else ''}"
|
||||
|
||||
self.update_job_status(
|
||||
job_id=job_id,
|
||||
status=JobStatus.CANCELLED,
|
||||
message=message
|
||||
)
|
||||
|
||||
logger.info(f"Job {job_id} cancelled")
|
||||
return True
|
||||
|
||||
def mark_job_failed(
|
||||
self,
|
||||
job_id: str,
|
||||
error_message: str,
|
||||
error_details: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""
|
||||
Mark a job as failed with error details.
|
||||
|
||||
This replaces mark_job_error() with better error tracking.
|
||||
"""
|
||||
updates = {
|
||||
'error_message': error_message
|
||||
}
|
||||
|
||||
if error_details:
|
||||
updates['error_details'] = error_details
|
||||
|
||||
self.firestore.update_job_status(
|
||||
job_id=job_id,
|
||||
status=JobStatus.FAILED,
|
||||
progress=0,
|
||||
message=f"Failed: {error_message}",
|
||||
**updates
|
||||
)
|
||||
logger.error(f"Job {job_id} failed: {error_message}")
|
||||
|
||||
def increment_retry_count(self, job_id: str) -> int:
|
||||
"""
|
||||
Increment retry count for a job.
|
||||
|
||||
Returns:
|
||||
New retry count
|
||||
"""
|
||||
job = self.get_job(job_id)
|
||||
if not job:
|
||||
return 0
|
||||
|
||||
new_count = job.retry_count + 1
|
||||
self.update_job(job_id, {'retry_count': new_count})
|
||||
return new_count
|
||||
|
||||
def append_worker_log(
|
||||
self,
|
||||
job_id: str,
|
||||
worker: str,
|
||||
level: str,
|
||||
message: str,
|
||||
max_logs: int = 500
|
||||
) -> None:
|
||||
"""
|
||||
Append a log entry to the job's worker_logs atomically.
|
||||
|
||||
Uses Firestore ArrayUnion for atomic append to avoid race conditions
|
||||
when multiple workers (audio + lyrics) are logging concurrently.
|
||||
|
||||
Note: max_logs is not enforced per-append (would require transactions).
|
||||
Logs may grow beyond max_logs; trim periodically if needed.
|
||||
|
||||
Args:
|
||||
job_id: Job ID
|
||||
worker: Worker name (audio, lyrics, screens, video, render)
|
||||
level: Log level (DEBUG, INFO, WARNING, ERROR)
|
||||
message: Log message
|
||||
max_logs: Not used (kept for API compatibility)
|
||||
"""
|
||||
log_entry = {
|
||||
'timestamp': datetime.utcnow().isoformat() + 'Z',
|
||||
'level': level,
|
||||
'worker': worker,
|
||||
'message': message[:1000] # Truncate long messages
|
||||
}
|
||||
|
||||
# Use atomic ArrayUnion to avoid race conditions
|
||||
self.firestore.append_worker_log(job_id, log_entry)
|
||||
|
||||
def get_worker_logs(
|
||||
self,
|
||||
job_id: str,
|
||||
since_index: int = 0,
|
||||
worker: Optional[str] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get worker logs for a job, optionally filtered by worker and index.
|
||||
|
||||
Args:
|
||||
job_id: Job ID
|
||||
since_index: Return only logs after this index (for pagination)
|
||||
worker: Filter by worker name (optional)
|
||||
|
||||
Returns:
|
||||
List of log entries as dicts
|
||||
"""
|
||||
job = self.get_job(job_id)
|
||||
if not job or not job.worker_logs:
|
||||
return []
|
||||
|
||||
logs = [log.dict() if hasattr(log, 'dict') else log for log in job.worker_logs]
|
||||
|
||||
# Filter by index
|
||||
if since_index > 0:
|
||||
logs = logs[since_index:]
|
||||
|
||||
# Filter by worker
|
||||
if worker:
|
||||
logs = [log for log in logs if log.get('worker') == worker]
|
||||
|
||||
return logs
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
"""
|
||||
Rclone configuration service for cloud storage integration.
|
||||
|
||||
This service manages the rclone configuration needed for Dropbox
|
||||
and other cloud storage uploads from the backend workers.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
from typing import Optional
|
||||
|
||||
from backend.config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RcloneService:
|
||||
"""Service for managing rclone configuration."""
|
||||
|
||||
# Secret Manager secret name for rclone config
|
||||
RCLONE_CONFIG_SECRET = "rclone-config"
|
||||
|
||||
def __init__(self):
|
||||
self.settings = get_settings()
|
||||
self._config_file: Optional[str] = None
|
||||
self._config_loaded = False
|
||||
|
||||
def setup_rclone_config(self) -> bool:
|
||||
"""
|
||||
Load rclone config from Secret Manager and set up environment.
|
||||
|
||||
Writes the config to a temp file and sets RCLONE_CONFIG env var.
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
if self._config_loaded:
|
||||
logger.debug("Rclone config already loaded")
|
||||
return True
|
||||
|
||||
try:
|
||||
# Get rclone config from Secret Manager
|
||||
config_content = self.settings.get_secret(self.RCLONE_CONFIG_SECRET)
|
||||
|
||||
if not config_content:
|
||||
logger.warning("Rclone config not found in Secret Manager")
|
||||
return False
|
||||
|
||||
# Write to a temp file
|
||||
fd, config_path = tempfile.mkstemp(prefix="rclone_", suffix=".conf")
|
||||
try:
|
||||
with os.fdopen(fd, 'w') as f:
|
||||
f.write(config_content)
|
||||
|
||||
self._config_file = config_path
|
||||
|
||||
# Set environment variable for rclone to find the config
|
||||
os.environ["RCLONE_CONFIG"] = config_path
|
||||
|
||||
logger.info(f"Rclone config loaded and written to {config_path}")
|
||||
self._config_loaded = True
|
||||
return True
|
||||
|
||||
except Exception:
|
||||
# Clean up the temp file on error
|
||||
# Note: os.fdopen() takes ownership of fd, so it's already closed
|
||||
# We only need to remove the temp file if it exists
|
||||
if os.path.exists(config_path):
|
||||
os.unlink(config_path)
|
||||
raise
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to setup rclone config: {e}")
|
||||
return False
|
||||
|
||||
def cleanup(self) -> None:
|
||||
"""Remove the temporary config file."""
|
||||
if self._config_file and os.path.exists(self._config_file):
|
||||
try:
|
||||
os.unlink(self._config_file)
|
||||
logger.debug(f"Cleaned up rclone config file: {self._config_file}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to cleanup rclone config: {e}")
|
||||
|
||||
# Always reset internal state and environment, even if the file was missing
|
||||
if self._config_file is not None:
|
||||
os.environ.pop("RCLONE_CONFIG", None)
|
||||
self._config_file = None
|
||||
self._config_loaded = False
|
||||
|
||||
@property
|
||||
def is_configured(self) -> bool:
|
||||
"""Check if rclone is configured and ready to use."""
|
||||
return self._config_loaded and self._config_file is not None
|
||||
|
||||
|
||||
# Singleton instance
|
||||
_rclone_service: Optional[RcloneService] = None
|
||||
|
||||
|
||||
def get_rclone_service() -> RcloneService:
|
||||
"""Get the singleton rclone service instance."""
|
||||
global _rclone_service
|
||||
if _rclone_service is None:
|
||||
_rclone_service = RcloneService()
|
||||
return _rclone_service
|
||||
@@ -0,0 +1,209 @@
|
||||
"""
|
||||
Google Cloud Storage operations for file management.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import json
|
||||
from typing import Optional, BinaryIO, Any, Dict
|
||||
from pathlib import Path
|
||||
from google.cloud import storage
|
||||
from datetime import timedelta
|
||||
|
||||
from backend.config import settings
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StorageService:
|
||||
"""Service for Google Cloud Storage operations."""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize GCS client."""
|
||||
self.client = storage.Client(project=settings.google_cloud_project)
|
||||
self.bucket = self.client.bucket(settings.gcs_bucket_name)
|
||||
|
||||
def upload_file(self, local_path: str, destination_path: str) -> str:
|
||||
"""Upload a file to GCS."""
|
||||
try:
|
||||
blob = self.bucket.blob(destination_path)
|
||||
blob.upload_from_filename(local_path)
|
||||
logger.info(f"Uploaded {local_path} to gs://{settings.gcs_bucket_name}/{destination_path}")
|
||||
return destination_path
|
||||
except Exception as e:
|
||||
logger.error(f"Error uploading file {local_path}: {e}")
|
||||
raise
|
||||
|
||||
def upload_fileobj(self, file_obj: BinaryIO, destination_path: str, content_type: Optional[str] = None) -> str:
|
||||
"""Upload a file object to GCS."""
|
||||
try:
|
||||
blob = self.bucket.blob(destination_path)
|
||||
if content_type:
|
||||
blob.content_type = content_type
|
||||
blob.upload_from_file(file_obj, rewind=True)
|
||||
logger.info(f"Uploaded file object to gs://{settings.gcs_bucket_name}/{destination_path}")
|
||||
return destination_path
|
||||
except Exception as e:
|
||||
logger.error(f"Error uploading file object: {e}")
|
||||
raise
|
||||
|
||||
def download_file(self, source_path: str, destination_path: str) -> str:
|
||||
"""Download a file from GCS."""
|
||||
try:
|
||||
blob = self.bucket.blob(source_path)
|
||||
blob.download_to_filename(destination_path)
|
||||
logger.info(f"Downloaded gs://{settings.gcs_bucket_name}/{source_path} to {destination_path}")
|
||||
return destination_path
|
||||
except Exception as e:
|
||||
logger.error(f"Error downloading file {source_path}: {e}")
|
||||
raise
|
||||
|
||||
def generate_signed_url(self, blob_path: str, expiration_minutes: int = 60) -> str:
|
||||
"""Generate a signed URL for downloading a file.
|
||||
|
||||
In Cloud Run, this uses the IAM signBlob API since we don't have
|
||||
a private key available. Requires the service account to have
|
||||
roles/iam.serviceAccountTokenCreator on itself.
|
||||
"""
|
||||
return self._generate_signed_url_internal(blob_path, "GET", expiration_minutes)
|
||||
|
||||
def generate_signed_upload_url(self, blob_path: str, content_type: str = "application/octet-stream", expiration_minutes: int = 60) -> str:
|
||||
"""Generate a signed URL for uploading a file directly to GCS.
|
||||
|
||||
This allows clients to upload files directly to GCS without going through
|
||||
the backend, bypassing any request body size limits.
|
||||
|
||||
Args:
|
||||
blob_path: The destination path in GCS
|
||||
content_type: The expected content type of the upload
|
||||
expiration_minutes: How long the URL is valid for
|
||||
|
||||
Returns:
|
||||
A signed URL that accepts PUT requests with the file content
|
||||
"""
|
||||
return self._generate_signed_url_internal(blob_path, "PUT", expiration_minutes, content_type)
|
||||
|
||||
def _generate_signed_url_internal(self, blob_path: str, method: str, expiration_minutes: int = 60, content_type: Optional[str] = None) -> str:
|
||||
"""Internal method to generate signed URLs for GET or PUT operations."""
|
||||
import google.auth
|
||||
from google.auth.transport import requests
|
||||
|
||||
try:
|
||||
blob = self.bucket.blob(blob_path)
|
||||
|
||||
# Get default credentials and refresh to ensure we have a valid token
|
||||
credentials, project = google.auth.default()
|
||||
|
||||
# Common kwargs for signed URL generation
|
||||
kwargs = {
|
||||
"version": "v4",
|
||||
"expiration": timedelta(minutes=expiration_minutes),
|
||||
"method": method,
|
||||
}
|
||||
|
||||
# For PUT requests, we need to specify the content type in headers
|
||||
if method == "PUT" and content_type:
|
||||
kwargs["headers"] = {"Content-Type": content_type}
|
||||
|
||||
# Check if we're using compute credentials (Cloud Run/GCE)
|
||||
# These need to use IAM signBlob via service_account_email + access_token
|
||||
if hasattr(credentials, 'service_account_email'):
|
||||
# Refresh credentials to get a valid access token
|
||||
auth_request = requests.Request()
|
||||
credentials.refresh(auth_request)
|
||||
|
||||
kwargs["service_account_email"] = credentials.service_account_email
|
||||
kwargs["access_token"] = credentials.token
|
||||
|
||||
url = blob.generate_signed_url(**kwargs)
|
||||
|
||||
logger.info(f"Generated signed {method} URL for {blob_path}")
|
||||
return url
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating signed {method} URL for {blob_path}: {e}")
|
||||
raise
|
||||
|
||||
def delete_file(self, blob_path: str) -> None:
|
||||
"""Delete a file from GCS."""
|
||||
try:
|
||||
blob = self.bucket.blob(blob_path)
|
||||
blob.delete()
|
||||
logger.info(f"Deleted gs://{settings.gcs_bucket_name}/{blob_path}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error deleting file {blob_path}: {e}")
|
||||
raise
|
||||
|
||||
def delete_folder(self, prefix: str) -> int:
|
||||
"""
|
||||
Delete all files in GCS with a given prefix (folder).
|
||||
|
||||
Args:
|
||||
prefix: The folder prefix to delete (e.g., "uploads/abc123/")
|
||||
|
||||
Returns:
|
||||
Number of files deleted
|
||||
"""
|
||||
try:
|
||||
blobs = list(self.bucket.list_blobs(prefix=prefix))
|
||||
deleted_count = 0
|
||||
|
||||
for blob in blobs:
|
||||
try:
|
||||
blob.delete()
|
||||
deleted_count += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"Error deleting blob {blob.name}: {e}")
|
||||
|
||||
if deleted_count > 0:
|
||||
logger.info(f"Deleted {deleted_count} files from gs://{settings.gcs_bucket_name}/{prefix}")
|
||||
|
||||
return deleted_count
|
||||
except Exception as e:
|
||||
logger.error(f"Error deleting folder {prefix}: {e}")
|
||||
return 0 # Don't raise - folder deletion shouldn't break operations
|
||||
|
||||
def list_files(self, prefix: str) -> list:
|
||||
"""List files in GCS with a given prefix."""
|
||||
try:
|
||||
blobs = self.bucket.list_blobs(prefix=prefix)
|
||||
return [blob.name for blob in blobs]
|
||||
except Exception as e:
|
||||
logger.error(f"Error listing files with prefix {prefix}: {e}")
|
||||
raise
|
||||
|
||||
def file_exists(self, blob_path: str) -> bool:
|
||||
"""Check if a file exists in GCS."""
|
||||
try:
|
||||
blob = self.bucket.blob(blob_path)
|
||||
return blob.exists()
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking file existence {blob_path}: {e}")
|
||||
raise
|
||||
|
||||
def upload_json(self, destination_path: str, data: Dict[str, Any]) -> str:
|
||||
"""Upload a JSON object to GCS."""
|
||||
try:
|
||||
blob = self.bucket.blob(destination_path)
|
||||
blob.content_type = "application/json"
|
||||
blob.upload_from_string(
|
||||
json.dumps(data, indent=2, ensure_ascii=False),
|
||||
content_type="application/json"
|
||||
)
|
||||
logger.info(f"Uploaded JSON to gs://{settings.gcs_bucket_name}/{destination_path}")
|
||||
return destination_path
|
||||
except Exception as e:
|
||||
logger.error(f"Error uploading JSON to {destination_path}: {e}")
|
||||
raise
|
||||
|
||||
def download_json(self, source_path: str) -> Dict[str, Any]:
|
||||
"""Download and parse a JSON file from GCS."""
|
||||
try:
|
||||
blob = self.bucket.blob(source_path)
|
||||
content = blob.download_as_text()
|
||||
data = json.loads(content)
|
||||
logger.info(f"Downloaded JSON from gs://{settings.gcs_bucket_name}/{source_path}")
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.error(f"Error downloading JSON from {source_path}: {e}")
|
||||
raise
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
"""
|
||||
OpenTelemetry tracing configuration for Google Cloud Trace.
|
||||
|
||||
This module sets up distributed tracing that exports to Google Cloud Trace,
|
||||
providing visibility into request flows, latency, and errors across the
|
||||
entire backend.
|
||||
|
||||
Usage:
|
||||
# In main.py or app initialization
|
||||
from backend.services.tracing import setup_tracing, instrument_app
|
||||
|
||||
setup_tracing(service_name="karaoke-backend")
|
||||
app = FastAPI()
|
||||
instrument_app(app)
|
||||
|
||||
# For custom spans in your code
|
||||
from backend.services.tracing import tracer, create_span
|
||||
|
||||
with tracer.start_as_current_span("my-operation") as span:
|
||||
span.set_attribute("job_id", job_id)
|
||||
# ... do work ...
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from typing import Optional, Any, Dict
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
|
||||
# OpenTelemetry imports
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SimpleSpanProcessor
|
||||
from opentelemetry.sdk.resources import Resource, SERVICE_NAME, SERVICE_VERSION
|
||||
from opentelemetry.trace import Status, StatusCode, Span
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
|
||||
# Google Cloud Trace exporter
|
||||
try:
|
||||
from opentelemetry.exporter.cloud_trace import CloudTraceSpanExporter
|
||||
from opentelemetry.resourcedetector.gcp_resource_detector import GoogleCloudResourceDetector
|
||||
GCP_AVAILABLE = True
|
||||
except ImportError:
|
||||
GCP_AVAILABLE = False
|
||||
CloudTraceSpanExporter = None
|
||||
GoogleCloudResourceDetector = None
|
||||
|
||||
# FastAPI instrumentation
|
||||
try:
|
||||
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
||||
FASTAPI_INSTRUMENTATION_AVAILABLE = True
|
||||
except ImportError:
|
||||
FASTAPI_INSTRUMENTATION_AVAILABLE = False
|
||||
FastAPIInstrumentor = None
|
||||
|
||||
# HTTPX instrumentation (for outgoing HTTP calls)
|
||||
try:
|
||||
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentation
|
||||
HTTPX_INSTRUMENTATION_AVAILABLE = True
|
||||
except ImportError:
|
||||
HTTPX_INSTRUMENTATION_AVAILABLE = False
|
||||
HTTPXClientInstrumentation = None
|
||||
|
||||
# Logging instrumentation
|
||||
try:
|
||||
from opentelemetry.instrumentation.logging import LoggingInstrumentor
|
||||
LOGGING_INSTRUMENTATION_AVAILABLE = True
|
||||
except ImportError:
|
||||
LOGGING_INSTRUMENTATION_AVAILABLE = False
|
||||
LoggingInstrumentor = None
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Global tracer instance
|
||||
_tracer: Optional[trace.Tracer] = None
|
||||
_initialized = False
|
||||
|
||||
|
||||
def setup_tracing(
|
||||
service_name: str = "karaoke-backend",
|
||||
service_version: str = "0.71.7",
|
||||
enable_in_dev: bool = False,
|
||||
) -> bool:
|
||||
"""
|
||||
Initialize OpenTelemetry tracing with Google Cloud Trace exporter.
|
||||
|
||||
Args:
|
||||
service_name: Name of the service (appears in Cloud Trace)
|
||||
service_version: Version of the service
|
||||
enable_in_dev: Whether to enable tracing in development (default False)
|
||||
|
||||
Returns:
|
||||
True if tracing was initialized, False if skipped
|
||||
"""
|
||||
global _tracer, _initialized
|
||||
|
||||
if _initialized:
|
||||
logger.debug("Tracing already initialized")
|
||||
return True
|
||||
|
||||
# Check if we should enable tracing
|
||||
# In Cloud Run, GOOGLE_CLOUD_PROJECT is always set
|
||||
is_cloud_run = os.environ.get("K_SERVICE") is not None
|
||||
gcp_project = os.environ.get("GOOGLE_CLOUD_PROJECT") or os.environ.get("GCP_PROJECT")
|
||||
|
||||
if not is_cloud_run and not enable_in_dev:
|
||||
logger.info("Tracing disabled in development (set enable_in_dev=True to enable)")
|
||||
_tracer = trace.get_tracer(service_name)
|
||||
return False
|
||||
|
||||
if not GCP_AVAILABLE:
|
||||
logger.warning("Google Cloud Trace exporter not available, tracing disabled")
|
||||
_tracer = trace.get_tracer(service_name)
|
||||
return False
|
||||
|
||||
logger.info(f"Initializing OpenTelemetry tracing for {service_name} v{service_version}")
|
||||
|
||||
try:
|
||||
# Create resource with service info and GCP resource detection
|
||||
resource_attributes = {
|
||||
SERVICE_NAME: service_name,
|
||||
SERVICE_VERSION: service_version,
|
||||
}
|
||||
|
||||
# Detect GCP resources (project, region, instance, etc.)
|
||||
if GoogleCloudResourceDetector and is_cloud_run:
|
||||
try:
|
||||
gcp_resource = GoogleCloudResourceDetector().detect()
|
||||
resource = Resource.create(resource_attributes).merge(gcp_resource)
|
||||
logger.info("Detected GCP resources for tracing")
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not detect GCP resources: {e}")
|
||||
resource = Resource.create(resource_attributes)
|
||||
else:
|
||||
resource = Resource.create(resource_attributes)
|
||||
|
||||
# Create tracer provider
|
||||
provider = TracerProvider(resource=resource)
|
||||
|
||||
# Add Cloud Trace exporter
|
||||
if CloudTraceSpanExporter and gcp_project:
|
||||
try:
|
||||
exporter = CloudTraceSpanExporter(project_id=gcp_project)
|
||||
# Use BatchSpanProcessor for production (better performance)
|
||||
processor = BatchSpanProcessor(exporter)
|
||||
provider.add_span_processor(processor)
|
||||
logger.info(f"Cloud Trace exporter configured for project: {gcp_project}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not configure Cloud Trace exporter: {e}")
|
||||
|
||||
# Set the global tracer provider
|
||||
trace.set_tracer_provider(provider)
|
||||
|
||||
# Get tracer instance
|
||||
_tracer = trace.get_tracer(service_name, service_version)
|
||||
|
||||
# Instrument logging to include trace context
|
||||
if LOGGING_INSTRUMENTATION_AVAILABLE and LoggingInstrumentor:
|
||||
try:
|
||||
LoggingInstrumentor().instrument(set_logging_format=True)
|
||||
logger.info("Logging instrumentation enabled")
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not instrument logging: {e}")
|
||||
|
||||
# Instrument HTTPX for outgoing requests
|
||||
if HTTPX_INSTRUMENTATION_AVAILABLE and HTTPXClientInstrumentation:
|
||||
try:
|
||||
HTTPXClientInstrumentation().instrument()
|
||||
logger.info("HTTPX instrumentation enabled")
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not instrument HTTPX: {e}")
|
||||
|
||||
_initialized = True
|
||||
logger.info("OpenTelemetry tracing initialized successfully")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize tracing: {e}")
|
||||
_tracer = trace.get_tracer(service_name)
|
||||
return False
|
||||
|
||||
|
||||
def instrument_app(app) -> None:
|
||||
"""
|
||||
Instrument a FastAPI application with automatic tracing.
|
||||
|
||||
This adds spans for all incoming requests, including:
|
||||
- Request method and path
|
||||
- Response status code
|
||||
- Request duration
|
||||
- Error information
|
||||
|
||||
Args:
|
||||
app: FastAPI application instance
|
||||
"""
|
||||
if not FASTAPI_INSTRUMENTATION_AVAILABLE or not FastAPIInstrumentor:
|
||||
logger.warning("FastAPI instrumentation not available")
|
||||
return
|
||||
|
||||
try:
|
||||
FastAPIInstrumentor.instrument_app(
|
||||
app,
|
||||
excluded_urls="health,healthz,ready,readiness,ping",
|
||||
)
|
||||
logger.info("FastAPI instrumentation enabled")
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not instrument FastAPI: {e}")
|
||||
|
||||
|
||||
def get_tracer() -> trace.Tracer:
|
||||
"""
|
||||
Get the global tracer instance.
|
||||
|
||||
Returns:
|
||||
Tracer instance (creates a no-op tracer if not initialized)
|
||||
"""
|
||||
global _tracer
|
||||
if _tracer is None:
|
||||
_tracer = trace.get_tracer("karaoke-backend")
|
||||
return _tracer
|
||||
|
||||
|
||||
# Convenience alias - lazily forwards attribute access to get_tracer()
|
||||
class _TracerProxy:
|
||||
"""Proxy that lazily returns the global tracer."""
|
||||
def __getattr__(self, name):
|
||||
return getattr(get_tracer(), name)
|
||||
|
||||
tracer = _TracerProxy()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def create_span(
|
||||
name: str,
|
||||
attributes: Optional[Dict[str, Any]] = None,
|
||||
kind: trace.SpanKind = trace.SpanKind.INTERNAL,
|
||||
):
|
||||
"""
|
||||
Create a traced span as a context manager.
|
||||
|
||||
Usage:
|
||||
with create_span("process-lyrics", {"job_id": job_id}) as span:
|
||||
# ... do work ...
|
||||
span.set_attribute("words_processed", 500)
|
||||
|
||||
Args:
|
||||
name: Span name (appears in Cloud Trace)
|
||||
attributes: Initial span attributes
|
||||
kind: Span kind (INTERNAL, SERVER, CLIENT, PRODUCER, CONSUMER)
|
||||
|
||||
Yields:
|
||||
The active span
|
||||
"""
|
||||
tracer = get_tracer()
|
||||
with tracer.start_as_current_span(name, kind=kind) as span:
|
||||
if attributes:
|
||||
for key, value in attributes.items():
|
||||
span.set_attribute(key, value)
|
||||
try:
|
||||
yield span
|
||||
except Exception as e:
|
||||
span.set_status(Status(StatusCode.ERROR, str(e)))
|
||||
span.record_exception(e)
|
||||
raise
|
||||
|
||||
|
||||
def traced(
|
||||
name: Optional[str] = None,
|
||||
attributes: Optional[Dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Decorator to trace a function.
|
||||
|
||||
Usage:
|
||||
@traced("process-correction")
|
||||
def process_correction(job_id: str):
|
||||
...
|
||||
|
||||
@traced(attributes={"operation": "add-lyrics"})
|
||||
async def add_lyrics(job_id: str, source: str):
|
||||
...
|
||||
|
||||
Args:
|
||||
name: Span name (defaults to function name)
|
||||
attributes: Static attributes to add to span
|
||||
"""
|
||||
def decorator(func):
|
||||
span_name = name or func.__name__
|
||||
|
||||
@wraps(func)
|
||||
async def async_wrapper(*args, **kwargs):
|
||||
with create_span(span_name, attributes) as span:
|
||||
# Add function arguments as attributes
|
||||
span.set_attribute("function", func.__name__)
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
@wraps(func)
|
||||
def sync_wrapper(*args, **kwargs):
|
||||
with create_span(span_name, attributes) as span:
|
||||
span.set_attribute("function", func.__name__)
|
||||
return func(*args, **kwargs)
|
||||
|
||||
# Return appropriate wrapper based on function type
|
||||
import asyncio
|
||||
if asyncio.iscoroutinefunction(func):
|
||||
return async_wrapper
|
||||
return sync_wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def add_span_attribute(key: str, value: Any) -> None:
|
||||
"""
|
||||
Add an attribute to the current span.
|
||||
|
||||
Args:
|
||||
key: Attribute name
|
||||
value: Attribute value (must be string, int, float, bool, or list thereof)
|
||||
"""
|
||||
span = trace.get_current_span()
|
||||
if span and span.is_recording():
|
||||
span.set_attribute(key, value)
|
||||
|
||||
|
||||
def add_span_event(name: str, attributes: Optional[Dict[str, Any]] = None) -> None:
|
||||
"""
|
||||
Add an event to the current span.
|
||||
|
||||
Events are timestamped annotations that appear in the trace timeline.
|
||||
|
||||
Args:
|
||||
name: Event name
|
||||
attributes: Event attributes
|
||||
"""
|
||||
span = trace.get_current_span()
|
||||
if span and span.is_recording():
|
||||
span.add_event(name, attributes=attributes or {})
|
||||
|
||||
|
||||
def set_span_error(error: Exception) -> None:
|
||||
"""
|
||||
Mark the current span as errored.
|
||||
|
||||
Args:
|
||||
error: The exception that occurred
|
||||
"""
|
||||
span = trace.get_current_span()
|
||||
if span and span.is_recording():
|
||||
span.set_status(Status(StatusCode.ERROR, str(error)))
|
||||
span.record_exception(error)
|
||||
|
||||
|
||||
def get_current_trace_id() -> Optional[str]:
|
||||
"""
|
||||
Get the current trace ID for correlation.
|
||||
|
||||
This can be used to link logs to traces.
|
||||
|
||||
Returns:
|
||||
Trace ID as hex string, or None if no active trace
|
||||
"""
|
||||
span = trace.get_current_span()
|
||||
if span:
|
||||
ctx = span.get_span_context()
|
||||
if ctx.is_valid:
|
||||
return format(ctx.trace_id, '032x')
|
||||
return None
|
||||
|
||||
|
||||
def get_current_span_id() -> Optional[str]:
|
||||
"""
|
||||
Get the current span ID.
|
||||
|
||||
Returns:
|
||||
Span ID as hex string, or None if no active span
|
||||
"""
|
||||
span = trace.get_current_span()
|
||||
if span:
|
||||
ctx = span.get_span_context()
|
||||
if ctx.is_valid:
|
||||
return format(ctx.span_id, '016x')
|
||||
return None
|
||||
@@ -0,0 +1,411 @@
|
||||
"""
|
||||
Worker coordination service.
|
||||
|
||||
Handles triggering and coordinating background workers.
|
||||
Supports two modes:
|
||||
- Cloud Tasks (production): Guaranteed delivery, automatic retries, horizontal scaling
|
||||
- Direct HTTP (development): Faster iteration, simpler debugging
|
||||
|
||||
SOLID Principles:
|
||||
- Single Responsibility: Only handles worker coordination
|
||||
- Dependency Inversion: Depends on HTTP abstraction, not implementation
|
||||
- Open/Closed: Can add new workers without modifying existing code
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import json
|
||||
from typing import Optional
|
||||
import httpx
|
||||
|
||||
from backend.config import get_settings
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Mapping of worker types to their Cloud Tasks queue names
|
||||
# These queues are created by Pulumi in infrastructure/__main__.py
|
||||
WORKER_QUEUES = {
|
||||
"audio": "audio-worker-queue",
|
||||
"lyrics": "lyrics-worker-queue",
|
||||
"screens": "screens-worker-queue",
|
||||
"render-video": "render-worker-queue",
|
||||
"video": "video-worker-queue",
|
||||
}
|
||||
|
||||
|
||||
class WorkerService:
|
||||
"""
|
||||
Service for coordinating background workers.
|
||||
|
||||
Supports two execution modes controlled by ENABLE_CLOUD_TASKS env var:
|
||||
|
||||
1. Cloud Tasks mode (production, ENABLE_CLOUD_TASKS=true):
|
||||
- Tasks enqueued to Cloud Tasks for guaranteed delivery
|
||||
- Automatic retries on failure
|
||||
- Each task runs in dedicated Cloud Run instance
|
||||
- Rate limiting to protect external APIs (Modal, AudioShake)
|
||||
|
||||
2. Direct HTTP mode (development, ENABLE_CLOUD_TASKS=false):
|
||||
- Direct HTTP POST to internal worker endpoints
|
||||
- Worker runs as FastAPI BackgroundTask
|
||||
- Faster iteration for local development
|
||||
- No retry guarantees
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize worker service."""
|
||||
self.settings = get_settings()
|
||||
self._base_url = self._get_base_url()
|
||||
self._admin_token = self._get_admin_token()
|
||||
self._use_cloud_tasks = self._should_use_cloud_tasks()
|
||||
self._tasks_client = None
|
||||
|
||||
if self._use_cloud_tasks:
|
||||
logger.info("WorkerService initialized with Cloud Tasks mode")
|
||||
else:
|
||||
logger.info("WorkerService initialized with direct HTTP mode")
|
||||
|
||||
def _should_use_cloud_tasks(self) -> bool:
|
||||
"""
|
||||
Check if Cloud Tasks should be used for worker coordination.
|
||||
|
||||
Controlled by ENABLE_CLOUD_TASKS setting (from environment variable).
|
||||
Default is false (direct HTTP mode) for backward compatibility.
|
||||
|
||||
Returns:
|
||||
True if Cloud Tasks should be used, False for direct HTTP
|
||||
"""
|
||||
return self.settings.enable_cloud_tasks
|
||||
|
||||
@property
|
||||
def tasks_client(self):
|
||||
"""
|
||||
Lazy-initialize Cloud Tasks client.
|
||||
|
||||
Only created when needed to avoid import overhead in development mode.
|
||||
"""
|
||||
if self._tasks_client is None and self._use_cloud_tasks:
|
||||
try:
|
||||
from google.cloud import tasks_v2
|
||||
self._tasks_client = tasks_v2.CloudTasksClient()
|
||||
except ImportError:
|
||||
logger.error(
|
||||
"google-cloud-tasks not installed. "
|
||||
"Install with: pip install google-cloud-tasks"
|
||||
)
|
||||
raise
|
||||
return self._tasks_client
|
||||
|
||||
def _get_admin_token(self) -> Optional[str]:
|
||||
"""
|
||||
Get admin token for internal API authentication.
|
||||
|
||||
Returns the first admin token from ADMIN_TOKENS env var.
|
||||
"""
|
||||
admin_tokens_str = self.settings.admin_tokens or ""
|
||||
tokens = [t.strip() for t in admin_tokens_str.split(",") if t.strip()]
|
||||
if tokens:
|
||||
return tokens[0]
|
||||
return None
|
||||
|
||||
def _get_base_url(self) -> str:
|
||||
"""
|
||||
Get base URL for internal API calls.
|
||||
|
||||
Priority:
|
||||
1. TEST_SERVER_URL env var (for tests)
|
||||
2. CLOUD_RUN_SERVICE_URL env var (for Cloud Tasks to call back)
|
||||
3. localhost with PORT env var (for development)
|
||||
|
||||
Returns:
|
||||
Base URL for API calls
|
||||
"""
|
||||
# Check for test environment override
|
||||
test_url = os.getenv("TEST_SERVER_URL")
|
||||
if test_url:
|
||||
return test_url
|
||||
|
||||
# Production: Cloud Run service URL
|
||||
# This must be the publicly accessible URL for Cloud Tasks to call
|
||||
service_url = os.getenv("CLOUD_RUN_SERVICE_URL")
|
||||
if service_url:
|
||||
return service_url
|
||||
|
||||
# Development mode: use localhost with PORT env var
|
||||
port = os.getenv("PORT", "8000")
|
||||
return f"http://localhost:{port}"
|
||||
|
||||
async def trigger_worker(
|
||||
self,
|
||||
worker_type: str,
|
||||
job_id: str,
|
||||
timeout_seconds: int = 30
|
||||
) -> bool:
|
||||
"""
|
||||
Trigger a background worker for a job.
|
||||
|
||||
In production (ENABLE_CLOUD_TASKS=true):
|
||||
- Enqueues task to Cloud Tasks queue
|
||||
- Cloud Tasks delivers HTTP request to internal endpoint
|
||||
- Automatic retries on failure with exponential backoff
|
||||
- Rate limiting protects external APIs
|
||||
|
||||
In development (ENABLE_CLOUD_TASKS=false):
|
||||
- Direct HTTP POST to internal endpoint
|
||||
- Faster iteration, but no retry guarantees
|
||||
- Worker runs in same container as API
|
||||
|
||||
Args:
|
||||
worker_type: Worker type ("audio", "lyrics", "screens", "render-video", "video")
|
||||
job_id: Job ID to process
|
||||
timeout_seconds: Request timeout (for direct HTTP mode)
|
||||
|
||||
Returns:
|
||||
True if trigger successful (or task enqueued), False otherwise
|
||||
"""
|
||||
if self._use_cloud_tasks:
|
||||
return await self._enqueue_cloud_task(worker_type, job_id)
|
||||
else:
|
||||
return await self._trigger_http(worker_type, job_id, timeout_seconds)
|
||||
|
||||
async def _enqueue_cloud_task(self, worker_type: str, job_id: str) -> bool:
|
||||
"""
|
||||
Enqueue task to Cloud Tasks for guaranteed delivery.
|
||||
|
||||
The task will be delivered as an HTTP POST to the internal worker endpoint.
|
||||
Cloud Tasks handles:
|
||||
- Retry on failure (with exponential backoff)
|
||||
- Rate limiting (max dispatches per second)
|
||||
- Deduplication (via task name if needed)
|
||||
- OIDC authentication for Cloud Run
|
||||
|
||||
Args:
|
||||
worker_type: Worker type
|
||||
job_id: Job ID to process
|
||||
|
||||
Returns:
|
||||
True if task enqueued successfully, False otherwise
|
||||
"""
|
||||
try:
|
||||
from google.cloud import tasks_v2
|
||||
|
||||
queue_name = WORKER_QUEUES.get(worker_type)
|
||||
if not queue_name:
|
||||
logger.error(f"Unknown worker type: {worker_type}")
|
||||
return False
|
||||
|
||||
project = self.settings.google_cloud_project
|
||||
if not project:
|
||||
logger.error("GOOGLE_CLOUD_PROJECT not set, cannot enqueue Cloud Task")
|
||||
return False
|
||||
|
||||
location = self.settings.gcp_region # Must match queue location
|
||||
|
||||
# Build queue path
|
||||
parent = self.tasks_client.queue_path(project, location, queue_name)
|
||||
|
||||
# Build task payload
|
||||
task = {
|
||||
"http_request": {
|
||||
"http_method": tasks_v2.HttpMethod.POST,
|
||||
"url": f"{self._base_url}/api/internal/workers/{worker_type}",
|
||||
"headers": {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
"body": json.dumps({"job_id": job_id}).encode(),
|
||||
# Use OIDC token for Cloud Run authentication
|
||||
# This allows Cloud Tasks to invoke the Cloud Run service
|
||||
"oidc_token": {
|
||||
"service_account_email": f"karaoke-backend@{project}.iam.gserviceaccount.com",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
# Add admin auth header if available
|
||||
# This is needed for the internal endpoint's require_admin dependency
|
||||
if self._admin_token:
|
||||
task["http_request"]["headers"]["Authorization"] = f"Bearer {self._admin_token}"
|
||||
|
||||
# Create task
|
||||
# Note: We don't set a task name, allowing Cloud Tasks to generate one
|
||||
# This prevents duplicate task errors on retries
|
||||
response = self.tasks_client.create_task(parent=parent, task=task)
|
||||
logger.info(
|
||||
f"Created Cloud Task for {worker_type} worker, job {job_id}: {response.name}"
|
||||
)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Failed to enqueue Cloud Task for {worker_type}/{job_id}: {e}",
|
||||
exc_info=True
|
||||
)
|
||||
return False
|
||||
|
||||
async def _trigger_http(
|
||||
self,
|
||||
worker_type: str,
|
||||
job_id: str,
|
||||
timeout_seconds: int = 30
|
||||
) -> bool:
|
||||
"""
|
||||
Trigger worker via direct HTTP call (development mode).
|
||||
|
||||
This is the original implementation - direct HTTP POST to internal endpoint.
|
||||
The endpoint adds the worker function to FastAPI BackgroundTasks.
|
||||
|
||||
Args:
|
||||
worker_type: Worker type
|
||||
job_id: Job ID to process
|
||||
timeout_seconds: Request timeout
|
||||
|
||||
Returns:
|
||||
True if trigger successful, False otherwise
|
||||
"""
|
||||
try:
|
||||
# Build headers with admin auth token
|
||||
headers = {}
|
||||
if self._admin_token:
|
||||
headers["Authorization"] = f"Bearer {self._admin_token}"
|
||||
|
||||
async with httpx.AsyncClient(timeout=timeout_seconds) as client:
|
||||
url = f"{self._base_url}/api/internal/workers/{worker_type}"
|
||||
|
||||
response = await client.post(
|
||||
url,
|
||||
json={"job_id": job_id},
|
||||
headers=headers
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
logger.info(f"Successfully triggered {worker_type} worker for job {job_id}")
|
||||
return True
|
||||
else:
|
||||
logger.error(
|
||||
f"Failed to trigger {worker_type} worker for job {job_id}: "
|
||||
f"HTTP {response.status_code} - {response.text}"
|
||||
)
|
||||
return False
|
||||
|
||||
except httpx.TimeoutException:
|
||||
logger.error(f"Timeout triggering {worker_type} worker for job {job_id}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error triggering {worker_type} worker for job {job_id}: {e}",
|
||||
exc_info=True
|
||||
)
|
||||
return False
|
||||
|
||||
# Convenience methods for specific workers
|
||||
# These provide a cleaner API and better IDE autocomplete
|
||||
|
||||
async def trigger_audio_worker(self, job_id: str) -> bool:
|
||||
"""Trigger audio separation worker."""
|
||||
return await self.trigger_worker("audio", job_id)
|
||||
|
||||
async def trigger_lyrics_worker(self, job_id: str) -> bool:
|
||||
"""Trigger lyrics transcription worker."""
|
||||
return await self.trigger_worker("lyrics", job_id)
|
||||
|
||||
async def trigger_screens_worker(self, job_id: str) -> bool:
|
||||
"""Trigger screen generation worker."""
|
||||
return await self.trigger_worker("screens", job_id)
|
||||
|
||||
async def trigger_video_worker(self, job_id: str) -> bool:
|
||||
"""
|
||||
Trigger video generation worker.
|
||||
|
||||
When USE_CLOUD_RUN_JOBS_FOR_VIDEO=true and ENABLE_CLOUD_TASKS=true,
|
||||
uses Cloud Run Jobs for execution (supports >30 min encoding).
|
||||
Otherwise, uses Cloud Tasks or direct HTTP.
|
||||
"""
|
||||
if self._use_cloud_tasks and self.settings.use_cloud_run_jobs_for_video:
|
||||
return await self._trigger_cloud_run_job(job_id)
|
||||
return await self.trigger_worker("video", job_id)
|
||||
|
||||
async def _trigger_cloud_run_job(self, job_id: str) -> bool:
|
||||
"""
|
||||
Trigger video encoding as a Cloud Run Job.
|
||||
|
||||
Cloud Run Jobs support up to 24 hours of execution time,
|
||||
making them suitable for very long video encoding tasks.
|
||||
|
||||
Args:
|
||||
job_id: Job ID to process
|
||||
|
||||
Returns:
|
||||
True if job was triggered successfully, False otherwise
|
||||
"""
|
||||
try:
|
||||
from google.cloud import run_v2
|
||||
|
||||
project = self.settings.google_cloud_project
|
||||
if not project:
|
||||
logger.error("GOOGLE_CLOUD_PROJECT not set, cannot trigger Cloud Run Job")
|
||||
return False
|
||||
|
||||
location = self.settings.gcp_region
|
||||
job_name = f"projects/{project}/locations/{location}/jobs/video-encoding-job"
|
||||
|
||||
# Create Cloud Run Jobs client
|
||||
client = run_v2.JobsClient()
|
||||
|
||||
# Run the job with overrides for the specific job_id
|
||||
request = run_v2.RunJobRequest(
|
||||
name=job_name,
|
||||
overrides=run_v2.RunJobRequest.Overrides(
|
||||
container_overrides=[
|
||||
run_v2.RunJobRequest.Overrides.ContainerOverride(
|
||||
args=["python", "-m", "backend.workers.video_worker", "--job-id", job_id],
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
# Run the job (async operation)
|
||||
operation = client.run_job(request=request)
|
||||
logger.info(f"Started Cloud Run Job for video encoding, job {job_id}: {operation.metadata}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Failed to trigger Cloud Run Job for video/{job_id}: {e}",
|
||||
exc_info=True
|
||||
)
|
||||
return False
|
||||
|
||||
async def trigger_render_video_worker(self, job_id: str) -> bool:
|
||||
"""Trigger render video worker (post-review)."""
|
||||
return await self.trigger_worker("render-video", job_id)
|
||||
|
||||
|
||||
# Global worker service instance
|
||||
_worker_service: Optional[WorkerService] = None
|
||||
|
||||
|
||||
def get_worker_service() -> WorkerService:
|
||||
"""
|
||||
Get global worker service instance.
|
||||
|
||||
Singleton pattern to reuse HTTP client pool.
|
||||
|
||||
Returns:
|
||||
WorkerService instance
|
||||
"""
|
||||
global _worker_service
|
||||
if _worker_service is None:
|
||||
_worker_service = WorkerService()
|
||||
return _worker_service
|
||||
|
||||
|
||||
def reset_worker_service() -> None:
|
||||
"""
|
||||
Reset the global worker service instance.
|
||||
|
||||
Used in tests to ensure clean state between test cases.
|
||||
"""
|
||||
global _worker_service
|
||||
_worker_service = None
|
||||
@@ -0,0 +1,112 @@
|
||||
"""
|
||||
YouTube credential service for server-side video uploads.
|
||||
|
||||
This service manages YouTube OAuth credentials stored in Secret Manager
|
||||
for non-interactive uploads from backend workers.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
from backend.config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class YouTubeService:
|
||||
"""Service for managing YouTube OAuth credentials."""
|
||||
|
||||
# Secret Manager secret name for YouTube credentials
|
||||
YOUTUBE_CREDENTIALS_SECRET = "youtube-oauth-credentials"
|
||||
|
||||
def __init__(self):
|
||||
self.settings = get_settings()
|
||||
self._credentials: Optional[Dict[str, Any]] = None
|
||||
self._loaded = False
|
||||
|
||||
def load_credentials(self) -> bool:
|
||||
"""
|
||||
Load YouTube OAuth credentials from Secret Manager.
|
||||
|
||||
The secret should contain a JSON object with:
|
||||
- token: Current access token (may be expired)
|
||||
- refresh_token: Refresh token for getting new access tokens
|
||||
- token_uri: Token endpoint URL
|
||||
- client_id: OAuth client ID
|
||||
- client_secret: OAuth client secret
|
||||
- scopes: List of OAuth scopes
|
||||
|
||||
Returns:
|
||||
True if credentials were loaded successfully, False otherwise
|
||||
"""
|
||||
if self._loaded:
|
||||
return self._credentials is not None
|
||||
|
||||
try:
|
||||
# Get credentials from Secret Manager
|
||||
creds_json = self.settings.get_secret(self.YOUTUBE_CREDENTIALS_SECRET)
|
||||
|
||||
if not creds_json:
|
||||
logger.warning("YouTube credentials not found in Secret Manager")
|
||||
self._loaded = True
|
||||
return False
|
||||
|
||||
# Parse the credentials JSON
|
||||
self._credentials = json.loads(creds_json)
|
||||
|
||||
# Validate required fields
|
||||
required_fields = ['refresh_token', 'token_uri', 'client_id', 'client_secret']
|
||||
missing = [f for f in required_fields if not self._credentials.get(f)]
|
||||
|
||||
if missing:
|
||||
logger.error(f"YouTube credentials missing required fields: {missing}")
|
||||
self._credentials = None
|
||||
self._loaded = True
|
||||
return False
|
||||
|
||||
logger.info("YouTube credentials loaded successfully from Secret Manager")
|
||||
self._loaded = True
|
||||
return True
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"Failed to parse YouTube credentials JSON: {e}")
|
||||
self._loaded = True
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load YouTube credentials: {e}")
|
||||
self._loaded = True
|
||||
return False
|
||||
|
||||
def get_credentials_dict(self) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Get the YouTube credentials as a dictionary.
|
||||
|
||||
This format is compatible with KaraokeFinalise's user_youtube_credentials
|
||||
parameter, which creates a google.oauth2.credentials.Credentials object.
|
||||
|
||||
Returns:
|
||||
Dictionary with credential data, or None if not available
|
||||
"""
|
||||
if not self._loaded:
|
||||
self.load_credentials()
|
||||
|
||||
return self._credentials
|
||||
|
||||
@property
|
||||
def is_configured(self) -> bool:
|
||||
"""Check if YouTube credentials are configured and ready to use."""
|
||||
if not self._loaded:
|
||||
self.load_credentials()
|
||||
return self._credentials is not None
|
||||
|
||||
|
||||
# Singleton instance
|
||||
_youtube_service: Optional[YouTubeService] = None
|
||||
|
||||
|
||||
def get_youtube_service() -> YouTubeService:
|
||||
"""Get the singleton YouTube service instance."""
|
||||
global _youtube_service
|
||||
if _youtube_service is None:
|
||||
_youtube_service = YouTubeService()
|
||||
return _youtube_service
|
||||
@@ -0,0 +1,4 @@
|
||||
"""
|
||||
Backend integration tests package.
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
"""
|
||||
Shared pytest fixtures for backend tests.
|
||||
|
||||
Provides common mocks and test utilities across all test modules.
|
||||
|
||||
NOTE: Module-level mocks are NOT applied here to allow emulator tests to work.
|
||||
Individual tests must mock dependencies as needed.
|
||||
"""
|
||||
import os
|
||||
import pytest
|
||||
from unittest.mock import Mock, MagicMock, AsyncMock, patch
|
||||
from datetime import datetime, UTC
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
# Mock google.auth.default for unit tests if not using emulator
|
||||
# This prevents DefaultCredentialsError during imports
|
||||
if 'FIRESTORE_EMULATOR_HOST' not in os.environ:
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
# Only mock the default() function, not the entire module
|
||||
# This allows other google.auth imports to work normally
|
||||
try:
|
||||
import google.auth
|
||||
mock_credentials = MagicMock()
|
||||
mock_credentials.token = 'fake-token'
|
||||
mock_credentials.valid = True
|
||||
mock_credentials.universe_domain = 'googleapis.com' # Required by google-cloud-storage
|
||||
mock_credentials.project_id = 'test-project'
|
||||
google.auth.default = MagicMock(return_value=(mock_credentials, 'test-project'))
|
||||
except ImportError:
|
||||
# If google.auth not installed, that's ok for some tests
|
||||
pass
|
||||
|
||||
from backend.models.job import Job, JobStatus, JobCreate
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_firestore():
|
||||
"""Mock Firestore client for unit tests."""
|
||||
# Mock at module level for unit tests
|
||||
with patch('google.cloud.firestore.Client') as mock_cls:
|
||||
client = MagicMock()
|
||||
mock_cls.return_value = client
|
||||
yield client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_storage_client():
|
||||
"""Mock GCS Storage client for unit tests."""
|
||||
# Mock at module level for unit tests
|
||||
with patch('google.cloud.storage.Client') as mock_cls:
|
||||
client = MagicMock()
|
||||
mock_cls.return_value = client
|
||||
yield client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_httpx_client():
|
||||
"""Mock httpx AsyncClient for worker service."""
|
||||
with patch('backend.services.worker_service.httpx.AsyncClient') as mock:
|
||||
client = AsyncMock()
|
||||
mock.return_value.__aenter__.return_value = client
|
||||
yield client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_job():
|
||||
"""Create a sample Job instance for testing."""
|
||||
return Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
progress=0,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
url="https://youtube.com/watch?v=test"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_job_create():
|
||||
"""Create a sample JobCreate instance for testing."""
|
||||
return JobCreate(
|
||||
url="https://youtube.com/watch?v=test",
|
||||
artist="Test Artist",
|
||||
title="Test Song"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_client():
|
||||
"""Create a FastAPI TestClient."""
|
||||
# Import here to avoid circular dependency
|
||||
from backend.main import app
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def create_mock_job(**kwargs):
|
||||
"""
|
||||
Factory function to create mock Job instances with custom fields.
|
||||
|
||||
Args:
|
||||
**kwargs: Job fields to override defaults
|
||||
|
||||
Returns:
|
||||
Job instance with specified fields
|
||||
"""
|
||||
defaults = {
|
||||
"job_id": "test123",
|
||||
"status": JobStatus.PENDING,
|
||||
"progress": 0,
|
||||
"created_at": datetime.now(UTC),
|
||||
"updated_at": datetime.now(UTC)
|
||||
}
|
||||
defaults.update(kwargs)
|
||||
return Job(**defaults)
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
"""
|
||||
Emulator integration tests.
|
||||
|
||||
These tests run against local GCP emulators (Firestore, GCS)
|
||||
to provide true integration testing without cloud costs.
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
"""
|
||||
Fixtures for emulator integration tests.
|
||||
|
||||
These tests use REAL Firestore and GCS emulators, so we DON'T mock google.cloud modules.
|
||||
Run with: scripts/run-emulator-tests.sh
|
||||
"""
|
||||
import pytest
|
||||
import os
|
||||
import requests
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
|
||||
def emulators_running() -> bool:
|
||||
"""Check if GCP emulators are running."""
|
||||
try:
|
||||
# Check Firestore emulator
|
||||
requests.get("http://127.0.0.1:8080", timeout=1)
|
||||
# Check GCS emulator
|
||||
requests.get("http://127.0.0.1:4443", timeout=1)
|
||||
return True
|
||||
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
||||
return False
|
||||
|
||||
|
||||
# Skip all tests in this module if emulators aren't running
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not emulators_running(),
|
||||
reason="GCP emulators not running. Start with: scripts/start-emulators.sh"
|
||||
)
|
||||
|
||||
# Set emulator environment variables (use 127.0.0.1 to force IPv4)
|
||||
os.environ["FIRESTORE_EMULATOR_HOST"] = "127.0.0.1:8080"
|
||||
os.environ["STORAGE_EMULATOR_HOST"] = "http://127.0.0.1:4443"
|
||||
os.environ["GOOGLE_CLOUD_PROJECT"] = "test-project"
|
||||
os.environ["GCS_BUCKET_NAME"] = "test-bucket"
|
||||
os.environ["FIRESTORE_COLLECTION"] = "test-jobs"
|
||||
os.environ["ENVIRONMENT"] = "test"
|
||||
os.environ["ADMIN_TOKENS"] = "test-admin-token"
|
||||
|
||||
# Only import app if emulators are running
|
||||
if emulators_running():
|
||||
from fastapi.testclient import TestClient
|
||||
from backend.main import app
|
||||
else:
|
||||
TestClient = None
|
||||
app = None
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def setup_gcs_bucket():
|
||||
"""Create GCS bucket in emulator before all tests."""
|
||||
try:
|
||||
response = requests.post(
|
||||
"http://127.0.0.1:4443/storage/v1/b",
|
||||
json={"name": "test-bucket"},
|
||||
params={"project": "test-project"},
|
||||
timeout=5
|
||||
)
|
||||
if response.status_code in [200, 409]:
|
||||
print("✅ GCS bucket 'test-bucket' ready in emulator")
|
||||
except Exception as e:
|
||||
print(f"⚠️ GCS bucket setup failed: {e}")
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def mock_worker_service():
|
||||
"""Mock worker service to prevent background tasks."""
|
||||
with patch("backend.api.routes.jobs.worker_service") as mock:
|
||||
mock.trigger_audio_worker = AsyncMock(return_value=True)
|
||||
mock.trigger_lyrics_worker = AsyncMock(return_value=True)
|
||||
mock.trigger_screens_worker = AsyncMock(return_value=True)
|
||||
mock.trigger_video_worker = AsyncMock(return_value=True)
|
||||
yield mock
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def client(mock_worker_service):
|
||||
"""Create FastAPI test client with mocked workers."""
|
||||
with patch("backend.api.routes.file_upload.worker_service", mock_worker_service):
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def auth_headers():
|
||||
"""Auth headers for testing."""
|
||||
return {"Authorization": "Bearer test-admin-token"}
|
||||
|
||||
@@ -0,0 +1,354 @@
|
||||
"""
|
||||
Integration tests using local GCP emulators.
|
||||
|
||||
These tests use real Firestore and GCS emulators running locally,
|
||||
providing true integration testing without cloud resources or costs.
|
||||
|
||||
Run with: ./scripts/run-emulator-tests.sh
|
||||
|
||||
NOTE: These tests mock background workers to avoid race conditions.
|
||||
The fixtures are defined in conftest.py in this directory.
|
||||
"""
|
||||
import pytest
|
||||
import time
|
||||
|
||||
from .conftest import emulators_running
|
||||
|
||||
|
||||
# Skip all tests in this module if emulators aren't running
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not emulators_running(),
|
||||
reason="GCP emulators not running. Start with: scripts/start-emulators.sh"
|
||||
)
|
||||
|
||||
# Fixtures are loaded from conftest.py in this directory
|
||||
|
||||
|
||||
class TestEmulatorBasics:
|
||||
"""Basic emulator connectivity tests."""
|
||||
|
||||
def test_health_endpoint(self, client):
|
||||
"""Test health endpoint works."""
|
||||
response = client.get("/api/health")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "healthy"
|
||||
|
||||
def test_root_endpoint(self, client):
|
||||
"""Test root endpoint works."""
|
||||
response = client.get("/")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["service"] == "karaoke-gen-backend"
|
||||
|
||||
|
||||
class TestJobCreation:
|
||||
"""Test job creation with Firestore emulator."""
|
||||
|
||||
def test_create_job_simple(self, client, auth_headers):
|
||||
"""Test creating a simple job."""
|
||||
response = client.post(
|
||||
"/api/jobs",
|
||||
headers=auth_headers,
|
||||
json={"url": "https://youtube.com/watch?v=test123"}
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "success"
|
||||
assert "job_id" in data
|
||||
|
||||
def test_create_job_with_metadata(self, client, auth_headers):
|
||||
"""Test creating a job with artist/title."""
|
||||
response = client.post(
|
||||
"/api/jobs",
|
||||
headers=auth_headers,
|
||||
json={
|
||||
"url": "https://youtube.com/watch?v=test",
|
||||
"artist": "Test Artist",
|
||||
"title": "Test Song"
|
||||
}
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "job_id" in data
|
||||
|
||||
|
||||
class TestJobRetrieval:
|
||||
"""Test job retrieval from Firestore emulator."""
|
||||
|
||||
def test_create_and_get_job(self, client, auth_headers):
|
||||
"""Test creating and then retrieving a job."""
|
||||
# Create
|
||||
create_resp = client.post(
|
||||
"/api/jobs",
|
||||
headers=auth_headers,
|
||||
json={
|
||||
"url": "https://youtube.com/watch?v=abc123",
|
||||
"artist": "Test Artist",
|
||||
"title": "Test Song"
|
||||
}
|
||||
)
|
||||
assert create_resp.status_code == 200
|
||||
job_id = create_resp.json()["job_id"]
|
||||
|
||||
# Small delay for emulator consistency
|
||||
time.sleep(0.2)
|
||||
|
||||
# Retrieve
|
||||
get_resp = client.get(f"/api/jobs/{job_id}", headers=auth_headers)
|
||||
|
||||
if get_resp.status_code != 200:
|
||||
print(f"GET failed: {get_resp.status_code} - {get_resp.text}")
|
||||
|
||||
assert get_resp.status_code == 200
|
||||
|
||||
job = get_resp.json()
|
||||
assert job["job_id"] == job_id
|
||||
assert job["status"] == "pending"
|
||||
assert job["artist"] == "Test Artist"
|
||||
assert job["title"] == "Test Song"
|
||||
|
||||
def test_get_nonexistent_job(self, client, auth_headers):
|
||||
"""Test fetching a job that doesn't exist."""
|
||||
response = client.get("/api/jobs/nonexistent-id", headers=auth_headers)
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
class TestJobList:
|
||||
"""Test listing jobs from Firestore."""
|
||||
|
||||
def test_list_jobs(self, client, auth_headers):
|
||||
"""Test listing jobs."""
|
||||
# Create a few jobs
|
||||
for i in range(3):
|
||||
client.post(
|
||||
"/api/jobs",
|
||||
headers=auth_headers,
|
||||
json={"url": f"https://youtube.com/watch?v=list{i}"}
|
||||
)
|
||||
|
||||
time.sleep(0.2)
|
||||
|
||||
# List
|
||||
response = client.get("/api/jobs", headers=auth_headers)
|
||||
assert response.status_code == 200
|
||||
|
||||
jobs = response.json()
|
||||
assert isinstance(jobs, list)
|
||||
# We should have at least the 3 we just created
|
||||
assert len(jobs) >= 3
|
||||
|
||||
|
||||
class TestJobDeletion:
|
||||
"""Test job deletion."""
|
||||
|
||||
def test_delete_job(self, client, auth_headers):
|
||||
"""Test deleting a job."""
|
||||
# Create
|
||||
create_resp = client.post(
|
||||
"/api/jobs",
|
||||
headers=auth_headers,
|
||||
json={"url": "https://youtube.com/watch?v=delete-me"}
|
||||
)
|
||||
job_id = create_resp.json()["job_id"]
|
||||
|
||||
time.sleep(0.2)
|
||||
|
||||
# Delete
|
||||
del_resp = client.delete(f"/api/jobs/{job_id}", headers=auth_headers)
|
||||
assert del_resp.status_code == 200
|
||||
|
||||
# Verify deleted
|
||||
get_resp = client.get(f"/api/jobs/{job_id}", headers=auth_headers)
|
||||
assert get_resp.status_code == 404
|
||||
|
||||
|
||||
class TestJobUpdates:
|
||||
"""Test job status updates."""
|
||||
|
||||
def test_cancel_job(self, client, auth_headers):
|
||||
"""Test cancelling a job."""
|
||||
# Create
|
||||
create_resp = client.post(
|
||||
"/api/jobs",
|
||||
headers=auth_headers,
|
||||
json={"url": "https://youtube.com/watch?v=cancel-me"}
|
||||
)
|
||||
job_id = create_resp.json()["job_id"]
|
||||
|
||||
time.sleep(0.2)
|
||||
|
||||
# Cancel
|
||||
cancel_resp = client.post(
|
||||
f"/api/jobs/{job_id}/cancel",
|
||||
headers=auth_headers,
|
||||
json={"reason": "test cancellation"}
|
||||
)
|
||||
assert cancel_resp.status_code == 200
|
||||
|
||||
# Verify cancelled
|
||||
get_resp = client.get(f"/api/jobs/{job_id}", headers=auth_headers)
|
||||
assert get_resp.status_code == 200
|
||||
job = get_resp.json()
|
||||
assert job["status"] == "cancelled"
|
||||
|
||||
|
||||
class TestFileUpload:
|
||||
"""Test file upload with GCS emulator."""
|
||||
|
||||
def test_upload_file(self, client, auth_headers):
|
||||
"""Test uploading a file."""
|
||||
test_file_content = b"fake audio data for testing"
|
||||
|
||||
response = client.post(
|
||||
"/api/jobs/upload",
|
||||
headers={"Authorization": auth_headers["Authorization"]},
|
||||
files={"file": ("test.flac", test_file_content, "audio/flac")},
|
||||
data={"artist": "Upload Artist", "title": "Upload Song"}
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "success"
|
||||
assert "job_id" in data
|
||||
|
||||
job_id = data["job_id"]
|
||||
|
||||
time.sleep(0.2)
|
||||
|
||||
# Verify job created with upload data
|
||||
get_resp = client.get(f"/api/jobs/{job_id}", headers=auth_headers)
|
||||
assert get_resp.status_code == 200
|
||||
job = get_resp.json()
|
||||
assert job["artist"] == "Upload Artist"
|
||||
assert job["title"] == "Upload Song"
|
||||
assert "input_media_gcs_path" in job
|
||||
|
||||
|
||||
class TestInternalEndpoints:
|
||||
"""Test internal worker endpoints."""
|
||||
|
||||
def test_internal_workers_exist(self, client, auth_headers):
|
||||
"""Test that internal worker endpoints exist and respond."""
|
||||
# Create a job first
|
||||
create_resp = client.post(
|
||||
"/api/jobs",
|
||||
headers=auth_headers,
|
||||
json={"url": "https://youtube.com/watch?v=worker-test"}
|
||||
)
|
||||
job_id = create_resp.json()["job_id"]
|
||||
|
||||
time.sleep(0.2)
|
||||
|
||||
# Test audio worker endpoint exists
|
||||
response = client.post(
|
||||
"/api/internal/workers/audio",
|
||||
headers=auth_headers,
|
||||
json={"job_id": job_id}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "started"
|
||||
|
||||
|
||||
class TestReviewEndpoints:
|
||||
"""Test lyrics review API endpoints."""
|
||||
|
||||
def test_review_ping(self, client, auth_headers):
|
||||
"""Test review ping endpoint."""
|
||||
# Create a job first
|
||||
create_resp = client.post(
|
||||
"/api/jobs",
|
||||
headers=auth_headers,
|
||||
json={"url": "https://youtube.com/watch?v=review-test"}
|
||||
)
|
||||
job_id = create_resp.json()["job_id"]
|
||||
|
||||
# Ping should work for any job
|
||||
response = client.get(f"/api/review/{job_id}/ping")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "ok"
|
||||
|
||||
def test_review_correction_data_wrong_status(self, client, auth_headers):
|
||||
"""Test that correction-data returns error for non-review jobs."""
|
||||
# Create a job (starts in pending status)
|
||||
create_resp = client.post(
|
||||
"/api/jobs",
|
||||
headers=auth_headers,
|
||||
json={"url": "https://youtube.com/watch?v=review-status-test"}
|
||||
)
|
||||
job_id = create_resp.json()["job_id"]
|
||||
|
||||
time.sleep(0.2)
|
||||
|
||||
# Should fail - job not in AWAITING_REVIEW status
|
||||
response = client.get(f"/api/review/{job_id}/correction-data")
|
||||
assert response.status_code == 400
|
||||
assert "not ready for review" in response.json()["detail"].lower()
|
||||
|
||||
def test_review_audio_no_job(self, client):
|
||||
"""Test audio endpoint returns 404 for nonexistent job."""
|
||||
response = client.get("/api/review/nonexistent-job/audio/")
|
||||
assert response.status_code == 404
|
||||
|
||||
def test_review_preview_video_stub(self, client, auth_headers):
|
||||
"""Test preview video endpoint exists."""
|
||||
# Create a job
|
||||
create_resp = client.post(
|
||||
"/api/jobs",
|
||||
headers=auth_headers,
|
||||
json={"url": "https://youtube.com/watch?v=preview-test"}
|
||||
)
|
||||
job_id = create_resp.json()["job_id"]
|
||||
|
||||
# Preview video should return error since job not ready
|
||||
response = client.post(
|
||||
f"/api/review/{job_id}/preview-video",
|
||||
json={"corrections": [], "corrected_segments": []}
|
||||
)
|
||||
# Should return 400 (job not in AWAITING_REVIEW state)
|
||||
assert response.status_code == 400
|
||||
|
||||
def test_review_annotations_stub(self, client, auth_headers):
|
||||
"""Test annotations endpoint (stub)."""
|
||||
create_resp = client.post(
|
||||
"/api/jobs",
|
||||
headers=auth_headers,
|
||||
json={"url": "https://youtube.com/watch?v=annotation-test"}
|
||||
)
|
||||
job_id = create_resp.json()["job_id"]
|
||||
|
||||
# Annotations endpoint should accept data (even if it just logs it)
|
||||
response = client.post(
|
||||
f"/api/review/{job_id}/v1/annotations",
|
||||
json={"type": "test", "data": "test annotation"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
|
||||
class TestRenderVideoWorker:
|
||||
"""Test render video worker endpoint."""
|
||||
|
||||
def test_render_video_worker_endpoint_exists(self, client, auth_headers):
|
||||
"""Test that render-video worker endpoint exists."""
|
||||
# Create a job
|
||||
create_resp = client.post(
|
||||
"/api/jobs",
|
||||
headers=auth_headers,
|
||||
json={"url": "https://youtube.com/watch?v=render-test"}
|
||||
)
|
||||
job_id = create_resp.json()["job_id"]
|
||||
|
||||
time.sleep(0.2)
|
||||
|
||||
# Test render-video worker endpoint exists
|
||||
response = client.post(
|
||||
"/api/internal/workers/render-video",
|
||||
headers=auth_headers,
|
||||
json={"job_id": job_id}
|
||||
)
|
||||
# Should return 200 (endpoint exists and starts)
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
print("✅ Emulator integration tests ready to run")
|
||||
@@ -0,0 +1,436 @@
|
||||
"""
|
||||
Direct tests for render_video_worker style loading.
|
||||
|
||||
Tests the fix for styles being loaded from job.style_params_gcs_path
|
||||
instead of the incorrect job.state_data['styles_gcs_path'].
|
||||
|
||||
Run with:
|
||||
./scripts/start-emulators.sh
|
||||
pytest backend/tests/emulator/test_style_loading_direct.py -v
|
||||
"""
|
||||
import pytest
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import requests
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, Optional
|
||||
|
||||
|
||||
def emulators_running() -> bool:
|
||||
"""Check if GCP emulators are running."""
|
||||
try:
|
||||
requests.get("http://127.0.0.1:8080", timeout=1)
|
||||
requests.get("http://127.0.0.1:4443", timeout=1)
|
||||
return True
|
||||
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
||||
return False
|
||||
|
||||
|
||||
# Skip all tests in this module if emulators aren't running
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not emulators_running(),
|
||||
reason="GCP emulators not running. Start with: scripts/start-emulators.sh"
|
||||
)
|
||||
|
||||
# Set up environment for emulators
|
||||
os.environ["FIRESTORE_EMULATOR_HOST"] = "127.0.0.1:8080"
|
||||
os.environ["STORAGE_EMULATOR_HOST"] = "http://127.0.0.1:4443"
|
||||
os.environ["GOOGLE_CLOUD_PROJECT"] = "test-project"
|
||||
os.environ["GCS_BUCKET_NAME"] = "test-bucket"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockJob:
|
||||
"""Mock job object for testing style loading."""
|
||||
job_id: str
|
||||
style_params_gcs_path: Optional[str] = None
|
||||
style_assets: Dict[str, str] = None
|
||||
state_data: Dict = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.style_assets is None:
|
||||
self.style_assets = {}
|
||||
if self.state_data is None:
|
||||
self.state_data = {}
|
||||
|
||||
|
||||
class MockStorageService:
|
||||
"""Mock storage service that uses GCS emulator."""
|
||||
|
||||
def __init__(self):
|
||||
from google.cloud import storage
|
||||
self.client = storage.Client()
|
||||
self.bucket_name = "test-bucket"
|
||||
self._ensure_bucket_exists()
|
||||
|
||||
def _ensure_bucket_exists(self):
|
||||
"""Create bucket in emulator if it doesn't exist."""
|
||||
try:
|
||||
self.client.create_bucket(self.bucket_name)
|
||||
except Exception:
|
||||
pass # Bucket already exists
|
||||
|
||||
def upload_string(self, content: str, gcs_path: str, content_type: str = "application/json"):
|
||||
"""Upload string content to GCS."""
|
||||
bucket = self.client.bucket(self.bucket_name)
|
||||
blob = bucket.blob(gcs_path)
|
||||
blob.upload_from_string(content, content_type=content_type)
|
||||
|
||||
def upload_bytes(self, content: bytes, gcs_path: str, content_type: str = "application/octet-stream"):
|
||||
"""Upload bytes to GCS."""
|
||||
bucket = self.client.bucket(self.bucket_name)
|
||||
blob = bucket.blob(gcs_path)
|
||||
blob.upload_from_string(content, content_type=content_type)
|
||||
|
||||
def download_file(self, gcs_path: str, local_path: str):
|
||||
"""Download file from GCS to local path."""
|
||||
bucket = self.client.bucket(self.bucket_name)
|
||||
blob = bucket.blob(gcs_path)
|
||||
blob.download_to_filename(local_path)
|
||||
|
||||
def file_exists(self, gcs_path: str) -> bool:
|
||||
"""Check if file exists in GCS."""
|
||||
bucket = self.client.bucket(self.bucket_name)
|
||||
blob = bucket.blob(gcs_path)
|
||||
return blob.exists()
|
||||
|
||||
|
||||
class TestStyleLoadingFix:
|
||||
"""
|
||||
Tests for the render_video_worker style loading fix.
|
||||
|
||||
The bug was: render_video_worker looked for styles at
|
||||
job.state_data.get('styles_gcs_path') but styles are actually stored at
|
||||
job.style_params_gcs_path and job.style_assets.
|
||||
"""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup(self):
|
||||
"""Set up storage service and temp directory for each test."""
|
||||
self.storage = MockStorageService()
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
yield
|
||||
# Cleanup
|
||||
import shutil
|
||||
shutil.rmtree(self.temp_dir, ignore_errors=True)
|
||||
|
||||
def _import_get_or_create_styles(self):
|
||||
"""
|
||||
Import _get_or_create_styles without importing the full worker module.
|
||||
We copy the function here to avoid import issues with lyrics_transcriber.
|
||||
"""
|
||||
# We'll test using a copy of the function logic
|
||||
# This avoids importing the full worker which has lyrics_transcriber dependency
|
||||
pass
|
||||
|
||||
def test_style_loading_from_correct_location(self):
|
||||
"""
|
||||
Test that styles are loaded from job.style_params_gcs_path,
|
||||
NOT from job.state_data['styles_gcs_path'].
|
||||
"""
|
||||
import time
|
||||
job_id = f"test-style-{int(time.time() * 1000)}"
|
||||
|
||||
# Create style JSON with placeholder paths
|
||||
style_json = {
|
||||
"intro": {
|
||||
"background_image": "/original/path/intro_bg.png",
|
||||
"font": "/original/path/font.ttf"
|
||||
},
|
||||
"karaoke": {
|
||||
"background_image": "/original/path/karaoke_bg.png",
|
||||
"font_path": "/original/path/font.ttf",
|
||||
"font_size": 100
|
||||
},
|
||||
"end": {
|
||||
"background_image": "/original/path/end_bg.png",
|
||||
"font": "/original/path/font.ttf"
|
||||
}
|
||||
}
|
||||
|
||||
# Upload style JSON to GCS (the CORRECT location)
|
||||
style_gcs_path = f"uploads/{job_id}/style/style_params.json"
|
||||
self.storage.upload_string(json.dumps(style_json), style_gcs_path)
|
||||
|
||||
# Upload mock assets
|
||||
assets = {
|
||||
"intro_background": f"uploads/{job_id}/style/intro_background.png",
|
||||
"karaoke_background": f"uploads/{job_id}/style/karaoke_background.png",
|
||||
"end_background": f"uploads/{job_id}/style/end_background.png",
|
||||
"font": f"uploads/{job_id}/style/font.ttf",
|
||||
}
|
||||
|
||||
for asset_key, gcs_path in assets.items():
|
||||
# Upload fake file content
|
||||
self.storage.upload_bytes(b"fake image/font data", gcs_path)
|
||||
|
||||
# Create mock job with styles in the CORRECT location
|
||||
job = MockJob(
|
||||
job_id=job_id,
|
||||
style_params_gcs_path=style_gcs_path, # CORRECT
|
||||
style_assets=assets, # CORRECT
|
||||
state_data={} # state_data is EMPTY (no styles_gcs_path)
|
||||
)
|
||||
|
||||
# Verify files exist in GCS
|
||||
assert self.storage.file_exists(style_gcs_path), "Style JSON should exist in GCS"
|
||||
for gcs_path in assets.values():
|
||||
assert self.storage.file_exists(gcs_path), f"Asset {gcs_path} should exist in GCS"
|
||||
|
||||
# Now test the style loading logic (inline version of _get_or_create_styles)
|
||||
style_dir = os.path.join(self.temp_dir, "style")
|
||||
os.makedirs(style_dir, exist_ok=True)
|
||||
styles_path = os.path.join(style_dir, "styles.json")
|
||||
|
||||
# This is the FIX: check job.style_params_gcs_path, not state_data
|
||||
if job.style_params_gcs_path:
|
||||
# Download style JSON
|
||||
self.storage.download_file(job.style_params_gcs_path, styles_path)
|
||||
|
||||
# Load and update paths
|
||||
with open(styles_path, 'r') as f:
|
||||
style_data = json.load(f)
|
||||
|
||||
# Download assets and update paths
|
||||
local_assets = {}
|
||||
for asset_key, gcs_path in job.style_assets.items():
|
||||
ext = os.path.splitext(gcs_path)[1] or '.png'
|
||||
local_path = os.path.join(style_dir, f"{asset_key}{ext}")
|
||||
self.storage.download_file(gcs_path, local_path)
|
||||
local_assets[asset_key] = local_path
|
||||
|
||||
# Update style_data with local paths
|
||||
asset_mapping = {
|
||||
'intro_background': ('intro', 'background_image'),
|
||||
'karaoke_background': ('karaoke', 'background_image'),
|
||||
'end_background': ('end', 'background_image'),
|
||||
'font': [('intro', 'font'), ('karaoke', 'font_path'), ('end', 'font')],
|
||||
}
|
||||
|
||||
for asset_key, local_path in local_assets.items():
|
||||
if asset_key in asset_mapping:
|
||||
mappings = asset_mapping[asset_key]
|
||||
if isinstance(mappings[0], str):
|
||||
mappings = [mappings]
|
||||
for section, field in mappings:
|
||||
if section in style_data:
|
||||
style_data[section][field] = local_path
|
||||
|
||||
# Save updated styles
|
||||
with open(styles_path, 'w') as f:
|
||||
json.dump(style_data, f, indent=2)
|
||||
|
||||
# Verify the result
|
||||
with open(styles_path, 'r') as f:
|
||||
result = json.load(f)
|
||||
|
||||
# Check that paths were updated to local paths
|
||||
assert style_dir in result['karaoke']['background_image'], \
|
||||
f"karaoke.background_image should be local path, got: {result['karaoke']['background_image']}"
|
||||
assert style_dir in result['karaoke']['font_path'], \
|
||||
f"karaoke.font_path should be local path, got: {result['karaoke']['font_path']}"
|
||||
assert style_dir in result['intro']['background_image'], \
|
||||
f"intro.background_image should be local path"
|
||||
|
||||
# Verify files actually exist locally
|
||||
assert os.path.exists(result['karaoke']['background_image']), \
|
||||
"Downloaded karaoke background should exist"
|
||||
assert os.path.exists(result['karaoke']['font_path']), \
|
||||
"Downloaded font should exist"
|
||||
|
||||
print(f"\n✅ Style loading from job.style_params_gcs_path works!")
|
||||
print(f" karaoke.background_image: {result['karaoke']['background_image']}")
|
||||
print(f" karaoke.font_path: {result['karaoke']['font_path']}")
|
||||
|
||||
def test_old_bug_state_data_lookup_fails(self):
|
||||
"""
|
||||
Demonstrate that the OLD bug would fail to find styles.
|
||||
|
||||
If we look at job.state_data['styles_gcs_path'] (the bug),
|
||||
we won't find anything because styles are stored at
|
||||
job.style_params_gcs_path.
|
||||
"""
|
||||
import time
|
||||
job_id = f"test-bug-{int(time.time() * 1000)}"
|
||||
|
||||
# Upload style to CORRECT location
|
||||
style_gcs_path = f"uploads/{job_id}/style/style_params.json"
|
||||
self.storage.upload_string('{"karaoke": {"font_size": 100}}', style_gcs_path)
|
||||
|
||||
# Create job with styles in CORRECT location
|
||||
job = MockJob(
|
||||
job_id=job_id,
|
||||
style_params_gcs_path=style_gcs_path, # CORRECT location
|
||||
style_assets={},
|
||||
state_data={} # No styles_gcs_path here!
|
||||
)
|
||||
|
||||
# THE BUG: looking in state_data instead of style_params_gcs_path
|
||||
wrong_path = job.state_data.get('styles_gcs_path')
|
||||
correct_path = job.style_params_gcs_path
|
||||
|
||||
assert wrong_path is None, "state_data['styles_gcs_path'] should be None (the bug location)"
|
||||
assert correct_path is not None, "style_params_gcs_path should have the correct path"
|
||||
assert self.storage.file_exists(correct_path), "Style should exist at correct path"
|
||||
|
||||
print(f"\n✅ Demonstrated the bug: state_data lookup returns None")
|
||||
print(f" state_data['styles_gcs_path'] = {wrong_path}")
|
||||
print(f" job.style_params_gcs_path = {correct_path}")
|
||||
|
||||
def test_default_styles_when_no_custom_styles(self):
|
||||
"""Test that default styles are used when no custom styles provided."""
|
||||
import time
|
||||
job_id = f"test-default-{int(time.time() * 1000)}"
|
||||
|
||||
# Job with NO custom styles
|
||||
job = MockJob(
|
||||
job_id=job_id,
|
||||
style_params_gcs_path=None,
|
||||
style_assets={},
|
||||
state_data={}
|
||||
)
|
||||
|
||||
# Simulate the logic
|
||||
style_dir = os.path.join(self.temp_dir, "style")
|
||||
os.makedirs(style_dir, exist_ok=True)
|
||||
styles_path = os.path.join(style_dir, "styles.json")
|
||||
|
||||
if job.style_params_gcs_path:
|
||||
# Would load custom styles
|
||||
pass
|
||||
else:
|
||||
# Use default styles
|
||||
default_styles = {
|
||||
"karaoke": {
|
||||
"background_color": "#000000",
|
||||
"font": "Arial",
|
||||
"font_path": "",
|
||||
"font_size": 100
|
||||
}
|
||||
}
|
||||
with open(styles_path, 'w') as f:
|
||||
json.dump(default_styles, f, indent=2)
|
||||
|
||||
# Verify defaults were used
|
||||
with open(styles_path, 'r') as f:
|
||||
result = json.load(f)
|
||||
|
||||
assert result['karaoke']['background_color'] == "#000000"
|
||||
assert result['karaoke']['font'] == "Arial"
|
||||
|
||||
print(f"\n✅ Default styles used when no custom styles provided")
|
||||
|
||||
|
||||
class TestParallelWorkerExecution:
|
||||
"""
|
||||
Tests for the parallel worker execution fix.
|
||||
|
||||
The bug was: FastAPI's BackgroundTasks runs async tasks sequentially,
|
||||
causing audio worker to complete before lyrics worker starts.
|
||||
|
||||
The fix uses asyncio.gather() to run both workers in parallel.
|
||||
"""
|
||||
|
||||
def test_asyncio_gather_runs_parallel(self):
|
||||
"""Test that asyncio.gather actually runs tasks in parallel."""
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
start_times = {}
|
||||
end_times = {}
|
||||
|
||||
async def task1():
|
||||
start_times['task1'] = time.time()
|
||||
await asyncio.sleep(0.1)
|
||||
end_times['task1'] = time.time()
|
||||
return "task1 done"
|
||||
|
||||
async def task2():
|
||||
start_times['task2'] = time.time()
|
||||
await asyncio.sleep(0.1)
|
||||
end_times['task2'] = time.time()
|
||||
return "task2 done"
|
||||
|
||||
async def run_parallel():
|
||||
return await asyncio.gather(task1(), task2())
|
||||
|
||||
overall_start = time.time()
|
||||
results = asyncio.run(run_parallel())
|
||||
overall_duration = time.time() - overall_start
|
||||
|
||||
# Both tasks should have started at nearly the same time
|
||||
start_diff = abs(start_times['task1'] - start_times['task2'])
|
||||
|
||||
# If running in parallel:
|
||||
# - Both start at ~same time (diff < 0.05s)
|
||||
# - Total duration is ~0.1s (not 0.2s for sequential)
|
||||
|
||||
assert start_diff < 0.05, \
|
||||
f"Tasks should start together, diff was {start_diff:.3f}s"
|
||||
assert overall_duration < 0.15, \
|
||||
f"Parallel execution should take ~0.1s, took {overall_duration:.3f}s"
|
||||
|
||||
print(f"\n✅ asyncio.gather runs tasks in parallel!")
|
||||
print(f" Task start time difference: {start_diff:.3f}s")
|
||||
print(f" Total duration: {overall_duration:.3f}s (sequential would be ~0.2s)")
|
||||
|
||||
def test_sequential_background_tasks_is_slow(self):
|
||||
"""
|
||||
Demonstrate that sequential execution (the bug) is slower.
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
execution_order = []
|
||||
|
||||
async def task1():
|
||||
execution_order.append(('task1', 'start'))
|
||||
await asyncio.sleep(0.05)
|
||||
execution_order.append(('task1', 'end'))
|
||||
|
||||
async def task2():
|
||||
execution_order.append(('task2', 'start'))
|
||||
await asyncio.sleep(0.05)
|
||||
execution_order.append(('task2', 'end'))
|
||||
|
||||
# Sequential (the bug)
|
||||
async def run_sequential():
|
||||
await task1()
|
||||
await task2()
|
||||
|
||||
execution_order.clear()
|
||||
start = time.time()
|
||||
asyncio.run(run_sequential())
|
||||
sequential_duration = time.time() - start
|
||||
sequential_order = execution_order.copy()
|
||||
|
||||
# Parallel (the fix)
|
||||
async def run_parallel():
|
||||
await asyncio.gather(task1(), task2())
|
||||
|
||||
execution_order.clear()
|
||||
start = time.time()
|
||||
asyncio.run(run_parallel())
|
||||
parallel_duration = time.time() - start
|
||||
parallel_order = execution_order.copy()
|
||||
|
||||
# Sequential: task1 starts, task1 ends, task2 starts, task2 ends
|
||||
assert sequential_order[0] == ('task1', 'start')
|
||||
assert sequential_order[1] == ('task1', 'end')
|
||||
assert sequential_order[2] == ('task2', 'start')
|
||||
|
||||
# Parallel: task1 starts, task2 starts (interleaved)
|
||||
assert parallel_order[0][1] == 'start'
|
||||
assert parallel_order[1][1] == 'start'
|
||||
|
||||
# Parallel should be ~2x faster
|
||||
assert parallel_duration < sequential_duration * 0.8, \
|
||||
f"Parallel ({parallel_duration:.3f}s) should be faster than sequential ({sequential_duration:.3f}s)"
|
||||
|
||||
print(f"\n✅ Demonstrated sequential vs parallel execution")
|
||||
print(f" Sequential: {sequential_duration:.3f}s - {sequential_order}")
|
||||
print(f" Parallel: {parallel_duration:.3f}s - {parallel_order}")
|
||||
|
||||
|
||||
print("✅ Style loading and parallel execution tests ready")
|
||||
@@ -0,0 +1,229 @@
|
||||
"""
|
||||
Direct tests for worker logging Firestore operations.
|
||||
|
||||
These tests bypass the full app/worker imports and test the Firestore
|
||||
operations directly, avoiding dependency issues.
|
||||
|
||||
Run with: ./scripts/run-emulator-tests.sh
|
||||
"""
|
||||
import pytest
|
||||
import time
|
||||
import requests
|
||||
import os
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def emulators_running() -> bool:
|
||||
"""Check if GCP emulators are running."""
|
||||
try:
|
||||
requests.get("http://127.0.0.1:8080", timeout=1)
|
||||
return True
|
||||
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
||||
return False
|
||||
|
||||
|
||||
# Skip all tests in this module if emulators aren't running
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not emulators_running(),
|
||||
reason="GCP emulators not running. Start with: scripts/start-emulators.sh"
|
||||
)
|
||||
|
||||
# Set up environment for emulator
|
||||
os.environ["FIRESTORE_EMULATOR_HOST"] = "127.0.0.1:8080"
|
||||
os.environ["GOOGLE_CLOUD_PROJECT"] = "test-project"
|
||||
|
||||
|
||||
class TestWorkerLogsFirestoreDirect:
|
||||
"""Direct Firestore tests for worker logs - no app imports needed."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_firestore(self):
|
||||
"""Set up Firestore client for each test."""
|
||||
from google.cloud import firestore
|
||||
self.db = firestore.Client(project="test-project")
|
||||
self.collection = "test-worker-logs"
|
||||
yield
|
||||
|
||||
def _create_test_job(self):
|
||||
"""Create a test job document."""
|
||||
job_id = f"test-{int(time.time() * 1000)}"
|
||||
doc_ref = self.db.collection(self.collection).document(job_id)
|
||||
doc_ref.set({
|
||||
"job_id": job_id,
|
||||
"status": "pending",
|
||||
"created_at": datetime.utcnow(),
|
||||
"worker_logs": []
|
||||
})
|
||||
return job_id
|
||||
|
||||
def _append_log_read_modify_write(self, job_id: str, worker: str, message: str):
|
||||
"""
|
||||
OLD METHOD: Read-modify-write (has race condition).
|
||||
This is what we were doing before.
|
||||
"""
|
||||
doc_ref = self.db.collection(self.collection).document(job_id)
|
||||
doc = doc_ref.get()
|
||||
if not doc.exists:
|
||||
return
|
||||
|
||||
data = doc.to_dict()
|
||||
logs = data.get("worker_logs", [])
|
||||
logs.append({
|
||||
"timestamp": datetime.utcnow().isoformat() + "Z",
|
||||
"level": "INFO",
|
||||
"worker": worker,
|
||||
"message": message
|
||||
})
|
||||
doc_ref.update({"worker_logs": logs})
|
||||
|
||||
def _append_log_array_union(self, job_id: str, worker: str, message: str):
|
||||
"""
|
||||
NEW METHOD: ArrayUnion (atomic, no race condition).
|
||||
This is what we're doing now.
|
||||
"""
|
||||
from google.cloud import firestore
|
||||
doc_ref = self.db.collection(self.collection).document(job_id)
|
||||
doc_ref.update({
|
||||
"worker_logs": firestore.ArrayUnion([{
|
||||
"timestamp": datetime.utcnow().isoformat() + "Z",
|
||||
"level": "INFO",
|
||||
"worker": worker,
|
||||
"message": message
|
||||
}])
|
||||
})
|
||||
|
||||
def _get_logs(self, job_id: str):
|
||||
"""Get logs from job document."""
|
||||
doc_ref = self.db.collection(self.collection).document(job_id)
|
||||
doc = doc_ref.get()
|
||||
if not doc.exists:
|
||||
return []
|
||||
return doc.to_dict().get("worker_logs", [])
|
||||
|
||||
def test_array_union_single_write(self):
|
||||
"""Test ArrayUnion works for single write."""
|
||||
job_id = self._create_test_job()
|
||||
|
||||
self._append_log_array_union(job_id, "test", "Single log message")
|
||||
|
||||
time.sleep(0.1)
|
||||
logs = self._get_logs(job_id)
|
||||
|
||||
assert len(logs) == 1
|
||||
assert logs[0]["message"] == "Single log message"
|
||||
assert logs[0]["worker"] == "test"
|
||||
|
||||
def test_array_union_sequential_writes(self):
|
||||
"""Test ArrayUnion preserves all sequential writes."""
|
||||
job_id = self._create_test_job()
|
||||
|
||||
for i in range(10):
|
||||
self._append_log_array_union(job_id, "test", f"Log {i}")
|
||||
|
||||
time.sleep(0.2)
|
||||
logs = self._get_logs(job_id)
|
||||
|
||||
assert len(logs) == 10, f"Expected 10 logs, got {len(logs)}"
|
||||
|
||||
def test_read_modify_write_race_condition(self):
|
||||
"""
|
||||
Demonstrate the race condition with read-modify-write.
|
||||
This test shows WHY we needed ArrayUnion.
|
||||
"""
|
||||
job_id = self._create_test_job()
|
||||
num_writes = 20
|
||||
|
||||
def write_log(index):
|
||||
self._append_log_read_modify_write(job_id, "worker", f"RMW Log {index}")
|
||||
|
||||
# Write concurrently - should lose some logs due to race condition
|
||||
with ThreadPoolExecutor(max_workers=5) as executor:
|
||||
futures = [executor.submit(write_log, i) for i in range(num_writes)]
|
||||
for future in as_completed(futures):
|
||||
try:
|
||||
future.result()
|
||||
except Exception:
|
||||
pass # Ignore errors for this test
|
||||
|
||||
time.sleep(0.5)
|
||||
logs = self._get_logs(job_id)
|
||||
|
||||
# With read-modify-write, we likely lose some logs
|
||||
# This is expected - the test documents the problem
|
||||
rmw_count = len([l for l in logs if "RMW Log" in l.get("message", "")])
|
||||
print(f"\nRead-modify-write: {rmw_count}/{num_writes} logs preserved")
|
||||
|
||||
# We expect to lose some logs (this is the bug we're fixing)
|
||||
# If all 20 are there, the race condition didn't trigger (which is fine)
|
||||
# The important thing is that ArrayUnion test below ALWAYS preserves all
|
||||
|
||||
def test_array_union_no_race_condition(self):
|
||||
"""
|
||||
Verify ArrayUnion preserves ALL concurrent writes.
|
||||
This is the critical test.
|
||||
"""
|
||||
job_id = self._create_test_job()
|
||||
num_writes = 20
|
||||
|
||||
def write_log(index):
|
||||
self._append_log_array_union(job_id, "worker", f"ArrayUnion Log {index}")
|
||||
|
||||
# Write concurrently
|
||||
with ThreadPoolExecutor(max_workers=5) as executor:
|
||||
futures = [executor.submit(write_log, i) for i in range(num_writes)]
|
||||
for future in as_completed(futures):
|
||||
future.result() # Raise any exceptions
|
||||
|
||||
time.sleep(0.5)
|
||||
logs = self._get_logs(job_id)
|
||||
|
||||
# With ArrayUnion, ALL logs should be preserved
|
||||
au_count = len([l for l in logs if "ArrayUnion Log" in l.get("message", "")])
|
||||
print(f"\nArrayUnion: {au_count}/{num_writes} logs preserved")
|
||||
|
||||
assert au_count == num_writes, \
|
||||
f"ArrayUnion should preserve all {num_writes} logs, got {au_count}"
|
||||
|
||||
def test_concurrent_workers_array_union(self):
|
||||
"""
|
||||
Test simulating audio and lyrics workers writing concurrently.
|
||||
"""
|
||||
job_id = self._create_test_job()
|
||||
|
||||
def audio_worker():
|
||||
for i in range(10):
|
||||
self._append_log_array_union(job_id, "audio", f"Audio log {i}")
|
||||
time.sleep(0.01)
|
||||
|
||||
def lyrics_worker():
|
||||
for i in range(10):
|
||||
self._append_log_array_union(job_id, "lyrics", f"Lyrics log {i}")
|
||||
time.sleep(0.01)
|
||||
|
||||
# Start both workers
|
||||
audio_thread = threading.Thread(target=audio_worker)
|
||||
lyrics_thread = threading.Thread(target=lyrics_worker)
|
||||
|
||||
audio_thread.start()
|
||||
lyrics_thread.start()
|
||||
|
||||
audio_thread.join()
|
||||
lyrics_thread.join()
|
||||
|
||||
time.sleep(0.5)
|
||||
logs = self._get_logs(job_id)
|
||||
|
||||
audio_logs = [l for l in logs if l.get("worker") == "audio"]
|
||||
lyrics_logs = [l for l in logs if l.get("worker") == "lyrics"]
|
||||
|
||||
print(f"\nConcurrent workers: {len(audio_logs)} audio + {len(lyrics_logs)} lyrics")
|
||||
|
||||
assert len(audio_logs) == 10, f"Expected 10 audio logs, got {len(audio_logs)}"
|
||||
assert len(lyrics_logs) == 10, f"Expected 10 lyrics logs, got {len(lyrics_logs)}"
|
||||
|
||||
print("✅ All logs from both workers preserved!")
|
||||
|
||||
|
||||
print("✅ Direct Firestore worker logs tests ready")
|
||||
@@ -0,0 +1,10 @@
|
||||
# Testing dependencies
|
||||
pytest>=7.4.0
|
||||
pytest-asyncio>=0.21.0
|
||||
pytest-cov>=4.1.0
|
||||
pytest-mock>=3.11.1
|
||||
httpx>=0.25.0 # For async testing
|
||||
|
||||
# Parent requirements (for import)
|
||||
-r ../requirements.txt
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
pytest>=8.0.0
|
||||
pytest-asyncio>=0.23.0
|
||||
pytest-timeout>=2.2.0
|
||||
pytest-cov>=4.1.0
|
||||
requests>=2.31.0
|
||||
|
||||
@@ -0,0 +1,460 @@
|
||||
"""
|
||||
Integration tests for the karaoke generation backend API.
|
||||
|
||||
These tests verify the backend works end-to-end with real Cloud Run deployment.
|
||||
These tests require a deployed backend and are marked to skip unless explicitly enabled.
|
||||
|
||||
Run with: pytest backend/tests/test_api_integration.py -m integration
|
||||
"""
|
||||
import pytest
|
||||
import requests
|
||||
import subprocess
|
||||
import time
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Any
|
||||
import os
|
||||
|
||||
|
||||
# Skip all tests unless explicitly running integration tests
|
||||
pytestmark = pytest.mark.skipif(
|
||||
os.environ.get("RUN_INTEGRATION_TESTS") != "true",
|
||||
reason="Integration tests require deployed backend. Set RUN_INTEGRATION_TESTS=true to run."
|
||||
)
|
||||
|
||||
# Configuration
|
||||
SERVICE_URL = "https://karaoke-backend-718638054799.us-central1.run.app"
|
||||
TEST_YOUTUBE_URL = "https://www.youtube.com/watch?v=dQw4w9WgXcQ" # Rick Astley - Never Gonna Give You Up
|
||||
DEFAULT_TIMEOUT = 30 # seconds
|
||||
|
||||
|
||||
def api_get(url: str, headers: Optional[Dict[str, str]] = None, **kwargs) -> requests.Response:
|
||||
"""Make a GET request with default timeout."""
|
||||
return requests.get(url, headers=headers, timeout=DEFAULT_TIMEOUT, **kwargs)
|
||||
|
||||
|
||||
def api_post(url: str, headers: Optional[Dict[str, str]] = None, **kwargs) -> requests.Response:
|
||||
"""Make a POST request with default timeout."""
|
||||
return requests.post(url, headers=headers, timeout=DEFAULT_TIMEOUT, **kwargs)
|
||||
|
||||
|
||||
def api_delete(url: str, headers: Optional[Dict[str, str]] = None, **kwargs) -> requests.Response:
|
||||
"""Make a DELETE request with default timeout."""
|
||||
return requests.delete(url, headers=headers, timeout=DEFAULT_TIMEOUT, **kwargs)
|
||||
|
||||
|
||||
def get_auth_token() -> str:
|
||||
"""Get authentication token for Cloud Run."""
|
||||
result = subprocess.run(
|
||||
["gcloud", "auth", "print-identity-token"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def auth_headers():
|
||||
"""Get authentication headers for requests."""
|
||||
token = get_auth_token()
|
||||
return {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
|
||||
class TestHealthEndpoint:
|
||||
"""Test health check endpoint."""
|
||||
|
||||
def test_health_check(self, auth_headers):
|
||||
"""Test that health endpoint returns 200 OK."""
|
||||
response = api_get(f"{SERVICE_URL}/api/health", headers=auth_headers)
|
||||
assert response.status_code == 200
|
||||
|
||||
data = response.json()
|
||||
assert data["status"] == "healthy"
|
||||
assert data["service"] == "karaoke-gen-backend"
|
||||
|
||||
def test_health_check_without_auth(self):
|
||||
"""Test that health endpoint requires authentication."""
|
||||
response = api_get(f"{SERVICE_URL}/api/health")
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
class TestRootEndpoint:
|
||||
"""Test root endpoint."""
|
||||
|
||||
def test_root_endpoint(self, auth_headers):
|
||||
"""Test root endpoint returns service info."""
|
||||
response = api_get(SERVICE_URL, headers=auth_headers)
|
||||
assert response.status_code == 200
|
||||
|
||||
data = response.json()
|
||||
assert data["service"] == "karaoke-gen-backend"
|
||||
assert data["status"] == "running"
|
||||
assert "version" in data
|
||||
|
||||
|
||||
class TestJobSubmission:
|
||||
"""Test job submission workflows."""
|
||||
|
||||
def test_submit_job_with_youtube_url(self, auth_headers):
|
||||
"""Test submitting a job with a YouTube URL."""
|
||||
payload = {"url": TEST_YOUTUBE_URL}
|
||||
response = api_post(
|
||||
f"{SERVICE_URL}/api/jobs",
|
||||
headers=auth_headers,
|
||||
json=payload
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert data["status"] == "success"
|
||||
assert "job_id" in data
|
||||
assert len(data["job_id"]) > 0
|
||||
assert "message" in data
|
||||
|
||||
# Store job_id for cleanup
|
||||
return data["job_id"]
|
||||
|
||||
def test_submit_job_with_invalid_url(self, auth_headers):
|
||||
"""Test that invalid URLs are rejected."""
|
||||
payload = {"url": "not-a-url"}
|
||||
response = api_post(
|
||||
f"{SERVICE_URL}/api/jobs",
|
||||
headers=auth_headers,
|
||||
json=payload
|
||||
)
|
||||
|
||||
assert response.status_code == 422 # Validation error
|
||||
|
||||
def test_submit_job_without_url(self, auth_headers):
|
||||
"""Test that missing URL is rejected."""
|
||||
payload = {}
|
||||
response = api_post(
|
||||
f"{SERVICE_URL}/api/jobs",
|
||||
headers=auth_headers,
|
||||
json=payload
|
||||
)
|
||||
|
||||
assert response.status_code == 422 # Validation error
|
||||
|
||||
|
||||
class TestJobRetrieval:
|
||||
"""Test job retrieval and status checking."""
|
||||
|
||||
@pytest.fixture
|
||||
def test_job_id(self, auth_headers):
|
||||
"""Create a test job for retrieval tests."""
|
||||
payload = {"url": TEST_YOUTUBE_URL}
|
||||
response = api_post(
|
||||
f"{SERVICE_URL}/api/jobs",
|
||||
headers=auth_headers,
|
||||
json=payload
|
||||
)
|
||||
assert response.status_code == 200
|
||||
job_id = response.json()["job_id"]
|
||||
|
||||
yield job_id
|
||||
|
||||
# Cleanup
|
||||
api_delete(
|
||||
f"{SERVICE_URL}/api/jobs/{job_id}",
|
||||
headers=auth_headers
|
||||
)
|
||||
|
||||
def test_get_job_status(self, auth_headers, test_job_id):
|
||||
"""Test retrieving job status."""
|
||||
response = api_get(
|
||||
f"{SERVICE_URL}/api/jobs/{test_job_id}",
|
||||
headers=auth_headers
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert data["job_id"] == test_job_id
|
||||
assert "status" in data
|
||||
# Updated to use current JobStatus enum values
|
||||
assert data["status"] in ["pending", "downloading", "separating_stage1",
|
||||
"separating_stage2", "audio_complete",
|
||||
"transcribing", "correcting", "lyrics_complete",
|
||||
"generating_screens", "applying_padding",
|
||||
"awaiting_review", "in_review", "review_complete",
|
||||
"rendering_video", "awaiting_instrumental_selection",
|
||||
"instrumental_selected", "generating_video",
|
||||
"encoding", "packaging", "uploading", "notifying",
|
||||
"complete", "failed", "cancelled"]
|
||||
assert "progress" in data
|
||||
assert "created_at" in data
|
||||
assert "updated_at" in data
|
||||
assert "timeline" in data
|
||||
|
||||
def test_get_nonexistent_job(self, auth_headers):
|
||||
"""Test that requesting nonexistent job returns 404."""
|
||||
fake_job_id = "nonexistent-job-id"
|
||||
response = api_get(
|
||||
f"{SERVICE_URL}/api/jobs/{fake_job_id}",
|
||||
headers=auth_headers
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
|
||||
def test_list_jobs(self, auth_headers, test_job_id):
|
||||
"""Test listing all jobs."""
|
||||
response = api_get(
|
||||
f"{SERVICE_URL}/api/jobs",
|
||||
headers=auth_headers
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert isinstance(data, list)
|
||||
# Our test job should be in the list
|
||||
job_ids = [job["job_id"] for job in data]
|
||||
assert test_job_id in job_ids
|
||||
|
||||
def test_list_jobs_with_status_filter(self, auth_headers):
|
||||
"""Test filtering jobs by status."""
|
||||
response = api_get(
|
||||
f"{SERVICE_URL}/api/jobs?status=pending",
|
||||
headers=auth_headers
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert isinstance(data, list)
|
||||
# All returned jobs should be pending
|
||||
for job in data:
|
||||
assert job["status"] == "pending"
|
||||
|
||||
def test_list_jobs_with_limit(self, auth_headers):
|
||||
"""Test limiting number of returned jobs."""
|
||||
response = api_get(
|
||||
f"{SERVICE_URL}/api/jobs?limit=5",
|
||||
headers=auth_headers
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert isinstance(data, list)
|
||||
assert len(data) <= 5
|
||||
|
||||
|
||||
class TestJobDeletion:
|
||||
"""Test job deletion."""
|
||||
|
||||
def test_delete_job(self, auth_headers):
|
||||
"""Test deleting a job."""
|
||||
# Create a job
|
||||
payload = {"url": TEST_YOUTUBE_URL}
|
||||
response = api_post(
|
||||
f"{SERVICE_URL}/api/jobs",
|
||||
headers=auth_headers,
|
||||
json=payload
|
||||
)
|
||||
assert response.status_code == 200
|
||||
job_id = response.json()["job_id"]
|
||||
|
||||
# Delete the job
|
||||
response = api_delete(
|
||||
f"{SERVICE_URL}/api/jobs/{job_id}",
|
||||
headers=auth_headers
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "success"
|
||||
|
||||
# Verify job is deleted
|
||||
response = api_get(
|
||||
f"{SERVICE_URL}/api/jobs/{job_id}",
|
||||
headers=auth_headers
|
||||
)
|
||||
assert response.status_code == 404
|
||||
|
||||
def test_delete_job_without_files(self, auth_headers):
|
||||
"""Test deleting a job without deleting files."""
|
||||
# Create a job
|
||||
payload = {"url": TEST_YOUTUBE_URL}
|
||||
response = api_post(
|
||||
f"{SERVICE_URL}/api/jobs",
|
||||
headers=auth_headers,
|
||||
json=payload
|
||||
)
|
||||
assert response.status_code == 200
|
||||
job_id = response.json()["job_id"]
|
||||
|
||||
# Delete job but keep files
|
||||
response = api_delete(
|
||||
f"{SERVICE_URL}/api/jobs/{job_id}?delete_files=false",
|
||||
headers=auth_headers
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "success"
|
||||
|
||||
def test_delete_nonexistent_job(self, auth_headers):
|
||||
"""Test deleting nonexistent job returns 404."""
|
||||
fake_job_id = "nonexistent-job-id"
|
||||
response = api_delete(
|
||||
f"{SERVICE_URL}/api/jobs/{fake_job_id}",
|
||||
headers=auth_headers
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
class TestFileUpload:
|
||||
"""Test file upload endpoint."""
|
||||
|
||||
def test_upload_audio_file(self, auth_headers, tmp_path):
|
||||
"""Test uploading an audio file."""
|
||||
# Create a small test file
|
||||
test_file = tmp_path / "test_audio.mp3"
|
||||
test_file.write_bytes(b"fake audio content for testing")
|
||||
|
||||
# Upload file
|
||||
with open(test_file, "rb") as f:
|
||||
files = {"file": ("test_audio.mp3", f, "audio/mpeg")}
|
||||
data = {
|
||||
"artist": "Test Artist",
|
||||
"title": "Test Song"
|
||||
}
|
||||
|
||||
# Remove Content-Type header for multipart/form-data
|
||||
headers = {
|
||||
"Authorization": auth_headers["Authorization"]
|
||||
}
|
||||
|
||||
response = api_post(
|
||||
f"{SERVICE_URL}/api/jobs/upload",
|
||||
headers=headers,
|
||||
files=files,
|
||||
data=data
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
result = response.json()
|
||||
|
||||
assert result["status"] == "success"
|
||||
assert "job_id" in result
|
||||
|
||||
# Cleanup
|
||||
api_delete(
|
||||
f"{SERVICE_URL}/api/jobs/{result['job_id']}",
|
||||
headers=auth_headers
|
||||
)
|
||||
|
||||
def test_upload_invalid_file_type(self, auth_headers, tmp_path):
|
||||
"""Test that invalid file types are rejected."""
|
||||
# Create a test file with invalid extension
|
||||
test_file = tmp_path / "test.txt"
|
||||
test_file.write_bytes(b"not an audio file")
|
||||
|
||||
with open(test_file, "rb") as f:
|
||||
files = {"file": ("test.txt", f, "text/plain")}
|
||||
data = {
|
||||
"artist": "Test Artist",
|
||||
"title": "Test Song"
|
||||
}
|
||||
|
||||
headers = {
|
||||
"Authorization": auth_headers["Authorization"]
|
||||
}
|
||||
|
||||
response = api_post(
|
||||
f"{SERVICE_URL}/api/jobs/upload",
|
||||
headers=headers,
|
||||
files=files,
|
||||
data=data
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
|
||||
def test_upload_without_metadata(self, auth_headers, tmp_path):
|
||||
"""Test that uploads without artist/title are rejected."""
|
||||
test_file = tmp_path / "test_audio.mp3"
|
||||
test_file.write_bytes(b"fake audio content")
|
||||
|
||||
with open(test_file, "rb") as f:
|
||||
files = {"file": ("test_audio.mp3", f, "audio/mpeg")}
|
||||
|
||||
headers = {
|
||||
"Authorization": auth_headers["Authorization"]
|
||||
}
|
||||
|
||||
response = api_post(
|
||||
f"{SERVICE_URL}/api/jobs/upload",
|
||||
headers=headers,
|
||||
files=files
|
||||
)
|
||||
|
||||
assert response.status_code == 422 # Validation error
|
||||
|
||||
|
||||
class TestJobProcessing:
|
||||
"""Test end-to-end job processing (long-running)."""
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.integration
|
||||
def test_complete_job_workflow(self, auth_headers):
|
||||
"""
|
||||
Test complete job workflow from submission to completion.
|
||||
|
||||
This test is slow and requires actual processing.
|
||||
Run with: pytest -m slow
|
||||
"""
|
||||
# Submit job
|
||||
payload = {"url": TEST_YOUTUBE_URL}
|
||||
response = api_post(
|
||||
f"{SERVICE_URL}/api/jobs",
|
||||
headers=auth_headers,
|
||||
json=payload
|
||||
)
|
||||
assert response.status_code == 200
|
||||
job_id = response.json()["job_id"]
|
||||
|
||||
# Poll for completion (with timeout)
|
||||
timeout = 600 # 10 minutes
|
||||
start_time = time.time()
|
||||
|
||||
while time.time() - start_time < timeout:
|
||||
response = api_get(
|
||||
f"{SERVICE_URL}/api/jobs/{job_id}",
|
||||
headers=auth_headers
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
job = response.json()
|
||||
status = job["status"]
|
||||
|
||||
if status == "complete":
|
||||
# Verify outputs exist
|
||||
assert "download_urls" in job
|
||||
assert len(job["download_urls"]) > 0
|
||||
break
|
||||
elif status == "failed":
|
||||
pytest.fail(f"Job failed: {job.get('error_message')}")
|
||||
|
||||
# Wait before next poll
|
||||
time.sleep(10)
|
||||
else:
|
||||
pytest.fail(f"Job did not complete within {timeout} seconds")
|
||||
|
||||
# Cleanup
|
||||
api_delete(
|
||||
f"{SERVICE_URL}/api/jobs/{job_id}",
|
||||
headers=auth_headers
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v", "--tb=short"])
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
"""
|
||||
Unit tests for API routes.
|
||||
|
||||
These tests use FastAPI TestClient with mocked services to test
|
||||
route logic without hitting real cloud services.
|
||||
"""
|
||||
import pytest
|
||||
import json
|
||||
from datetime import datetime, UTC
|
||||
from unittest.mock import MagicMock, AsyncMock, patch
|
||||
from fastapi.testclient import TestClient
|
||||
from io import BytesIO
|
||||
|
||||
from backend.models.job import Job, JobStatus
|
||||
|
||||
|
||||
class TestHealthRoutes:
|
||||
"""Tests for health.py routes."""
|
||||
|
||||
@pytest.fixture
|
||||
def client(self):
|
||||
"""Create test client with mocked dependencies."""
|
||||
mock_creds = MagicMock()
|
||||
mock_creds.universe_domain = 'googleapis.com'
|
||||
with patch('backend.services.firestore_service.firestore'), \
|
||||
patch('backend.services.storage_service.storage'), \
|
||||
patch('google.auth.default', return_value=(mock_creds, 'test-project')):
|
||||
from backend.main import app
|
||||
return TestClient(app)
|
||||
|
||||
def test_health_endpoint_returns_200(self, client):
|
||||
"""Test /api/health returns 200 OK."""
|
||||
response = client.get("/api/health")
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_health_endpoint_returns_healthy_status(self, client):
|
||||
"""Test health endpoint returns healthy status."""
|
||||
response = client.get("/api/health")
|
||||
data = response.json()
|
||||
assert data["status"] == "healthy"
|
||||
|
||||
def test_root_endpoint_returns_200(self, client):
|
||||
"""Test root endpoint returns 200."""
|
||||
response = client.get("/")
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
class TestJobRoutes:
|
||||
"""Tests for jobs.py routes.
|
||||
|
||||
Note: These tests verify the route module structure.
|
||||
Full integration tests are in test_api_integration.py.
|
||||
"""
|
||||
|
||||
def test_jobs_router_exists(self):
|
||||
"""Test jobs router can be imported."""
|
||||
from backend.api.routes import jobs
|
||||
assert hasattr(jobs, 'router')
|
||||
|
||||
def test_jobs_router_has_expected_endpoints(self):
|
||||
"""Test jobs router defines expected endpoints."""
|
||||
from backend.api.routes.jobs import router
|
||||
routes = [route.path for route in router.routes]
|
||||
assert '/jobs' in routes or any('/jobs' in r for r in routes)
|
||||
|
||||
|
||||
class TestInternalRoutes:
|
||||
"""Tests for internal.py routes.
|
||||
|
||||
Note: These tests are minimal because the internal routes
|
||||
trigger actual worker processing which requires complex mocking.
|
||||
The actual worker logic is tested in test_workers.py.
|
||||
"""
|
||||
|
||||
def test_internal_endpoint_structure(self):
|
||||
"""Test internal route module has expected endpoints."""
|
||||
from backend.api.routes import internal
|
||||
# Just verify the module can be imported
|
||||
assert hasattr(internal, 'router')
|
||||
|
||||
|
||||
class TestFileUploadRoutes:
|
||||
"""Tests for file_upload.py routes.
|
||||
|
||||
Note: These tests verify the route module structure.
|
||||
Full integration tests are in test_api_integration.py.
|
||||
"""
|
||||
|
||||
def test_file_upload_router_exists(self):
|
||||
"""Test file upload router can be imported."""
|
||||
from backend.api.routes import file_upload
|
||||
assert hasattr(file_upload, 'router')
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
"""
|
||||
Tests for AuthService and FirestoreService.
|
||||
|
||||
These tests focus on the service initialization and basic operations.
|
||||
"""
|
||||
import pytest
|
||||
from datetime import datetime, UTC
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
class TestAuthServiceBasics:
|
||||
"""Basic tests for AuthService."""
|
||||
|
||||
def test_auth_service_can_import(self):
|
||||
"""Test AuthService can be imported."""
|
||||
from backend.services.auth_service import AuthService
|
||||
assert AuthService is not None
|
||||
|
||||
def test_auth_service_initialization(self):
|
||||
"""Test AuthService initializes with mocked dependencies."""
|
||||
mock_firestore = MagicMock()
|
||||
|
||||
with patch('backend.services.auth_service.FirestoreService', return_value=mock_firestore), \
|
||||
patch.dict('os.environ', {'ADMIN_TOKENS': 'test-token'}):
|
||||
from backend.services.auth_service import AuthService
|
||||
service = AuthService()
|
||||
assert service is not None
|
||||
|
||||
def test_auth_service_has_validate_method(self):
|
||||
"""Test AuthService has token validation method."""
|
||||
from backend.services.auth_service import AuthService
|
||||
assert hasattr(AuthService, 'validate_token') or hasattr(AuthService, 'verify_token')
|
||||
|
||||
|
||||
class TestFirestoreServiceBasics:
|
||||
"""Basic tests for FirestoreService."""
|
||||
|
||||
def test_firestore_service_can_import(self):
|
||||
"""Test FirestoreService can be imported."""
|
||||
from backend.services.firestore_service import FirestoreService
|
||||
assert FirestoreService is not None
|
||||
|
||||
def test_firestore_service_initialization(self):
|
||||
"""Test FirestoreService initializes with mocked client."""
|
||||
mock_client = MagicMock()
|
||||
|
||||
with patch('backend.services.firestore_service.firestore.Client', return_value=mock_client):
|
||||
from backend.services.firestore_service import FirestoreService
|
||||
service = FirestoreService()
|
||||
assert service is not None
|
||||
|
||||
def test_firestore_service_has_crud_methods(self):
|
||||
"""Test FirestoreService has CRUD methods."""
|
||||
from backend.services.firestore_service import FirestoreService
|
||||
assert hasattr(FirestoreService, 'create_job')
|
||||
assert hasattr(FirestoreService, 'get_job')
|
||||
assert hasattr(FirestoreService, 'update_job')
|
||||
assert hasattr(FirestoreService, 'delete_job')
|
||||
|
||||
|
||||
class TestStorageServiceBasics:
|
||||
"""Basic tests for StorageService."""
|
||||
|
||||
def test_storage_service_can_import(self):
|
||||
"""Test StorageService can be imported."""
|
||||
from backend.services.storage_service import StorageService
|
||||
assert StorageService is not None
|
||||
|
||||
def test_storage_service_initialization(self):
|
||||
"""Test StorageService initializes with mocked client."""
|
||||
mock_client = MagicMock()
|
||||
mock_bucket = MagicMock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
|
||||
with patch('backend.services.storage_service.storage.Client', return_value=mock_client):
|
||||
from backend.services.storage_service import StorageService
|
||||
service = StorageService()
|
||||
assert service is not None
|
||||
|
||||
def test_storage_service_has_file_methods(self):
|
||||
"""Test StorageService has file operation methods."""
|
||||
from backend.services.storage_service import StorageService
|
||||
assert hasattr(StorageService, 'upload_file')
|
||||
assert hasattr(StorageService, 'download_file')
|
||||
|
||||
|
||||
class TestWorkerServiceBasics:
|
||||
"""Basic tests for WorkerService."""
|
||||
|
||||
def test_worker_service_can_import(self):
|
||||
"""Test WorkerService can be imported."""
|
||||
from backend.services.worker_service import WorkerService
|
||||
assert WorkerService is not None
|
||||
|
||||
def test_worker_service_initialization(self):
|
||||
"""Test WorkerService initializes correctly."""
|
||||
from backend.services.worker_service import WorkerService
|
||||
service = WorkerService()
|
||||
assert service is not None
|
||||
|
||||
def test_worker_service_has_trigger_methods(self):
|
||||
"""Test WorkerService has trigger methods."""
|
||||
from backend.services.worker_service import WorkerService
|
||||
assert hasattr(WorkerService, 'trigger_audio_worker')
|
||||
assert hasattr(WorkerService, 'trigger_lyrics_worker')
|
||||
assert hasattr(WorkerService, 'trigger_screens_worker')
|
||||
assert hasattr(WorkerService, 'trigger_video_worker')
|
||||
|
||||
def test_worker_service_get_base_url(self):
|
||||
"""Test WorkerService constructs base URL."""
|
||||
from backend.services.worker_service import WorkerService
|
||||
service = WorkerService()
|
||||
url = service._get_base_url()
|
||||
assert 'http' in url
|
||||
|
||||
|
||||
class TestJobManagerBasics:
|
||||
"""Basic tests for JobManager."""
|
||||
|
||||
def test_job_manager_can_import(self):
|
||||
"""Test JobManager can be imported."""
|
||||
from backend.services.job_manager import JobManager
|
||||
assert JobManager is not None
|
||||
|
||||
def test_job_manager_initialization(self):
|
||||
"""Test JobManager initializes with mocked services."""
|
||||
mock_firestore = MagicMock()
|
||||
mock_storage = MagicMock()
|
||||
|
||||
with patch('backend.services.job_manager.FirestoreService', return_value=mock_firestore), \
|
||||
patch('backend.services.job_manager.StorageService', return_value=mock_storage):
|
||||
from backend.services.job_manager import JobManager
|
||||
manager = JobManager()
|
||||
assert manager is not None
|
||||
|
||||
def test_job_manager_has_crud_methods(self):
|
||||
"""Test JobManager has CRUD methods."""
|
||||
from backend.services.job_manager import JobManager
|
||||
assert hasattr(JobManager, 'create_job')
|
||||
assert hasattr(JobManager, 'get_job')
|
||||
assert hasattr(JobManager, 'delete_job')
|
||||
assert hasattr(JobManager, 'list_jobs')
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
"""
|
||||
Extended tests for config.py to improve coverage.
|
||||
"""
|
||||
import pytest
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
class TestSettingsConfiguration:
|
||||
"""Tests for Settings configuration."""
|
||||
|
||||
def test_settings_class_exists(self):
|
||||
"""Test Settings class exists."""
|
||||
from backend.config import Settings
|
||||
assert Settings is not None
|
||||
|
||||
def test_settings_loads_defaults(self):
|
||||
"""Test Settings loads default values."""
|
||||
from backend.config import Settings
|
||||
settings = Settings()
|
||||
assert settings is not None
|
||||
|
||||
def test_gcs_bucket_name_from_env(self):
|
||||
"""Test GCS bucket name from environment."""
|
||||
with patch.dict(os.environ, {'GCS_BUCKET_NAME': 'test-bucket'}):
|
||||
from backend.config import Settings
|
||||
settings = Settings()
|
||||
assert settings.gcs_bucket_name == 'test-bucket'
|
||||
|
||||
def test_firestore_collection_from_env(self):
|
||||
"""Test Firestore collection from environment."""
|
||||
with patch.dict(os.environ, {'FIRESTORE_COLLECTION': 'test-jobs'}):
|
||||
from backend.config import Settings
|
||||
settings = Settings()
|
||||
assert settings.firestore_collection == 'test-jobs'
|
||||
|
||||
def test_environment_setting(self):
|
||||
"""Test environment setting."""
|
||||
from backend.config import Settings
|
||||
settings = Settings()
|
||||
# Accept 'test' as well since pytest may set ENVIRONMENT=test
|
||||
assert settings.environment in ['development', 'production', 'testing', 'test']
|
||||
|
||||
def test_google_cloud_project(self):
|
||||
"""Test Google Cloud project setting."""
|
||||
with patch.dict(os.environ, {'GOOGLE_CLOUD_PROJECT': 'test-project'}):
|
||||
from backend.config import Settings
|
||||
settings = Settings()
|
||||
# Should either use the env var or have a default
|
||||
|
||||
|
||||
class TestGetSettings:
|
||||
"""Tests for get_settings function."""
|
||||
|
||||
def test_get_settings_returns_settings(self):
|
||||
"""Test get_settings returns Settings instance."""
|
||||
from backend.config import get_settings
|
||||
settings = get_settings()
|
||||
assert settings is not None
|
||||
|
||||
def test_get_settings_returns_same_instance(self):
|
||||
"""Test get_settings returns cached instance."""
|
||||
from backend.config import get_settings
|
||||
settings1 = get_settings()
|
||||
settings2 = get_settings()
|
||||
# Should be same cached instance
|
||||
assert settings1 is settings2
|
||||
|
||||
@@ -0,0 +1,377 @@
|
||||
"""
|
||||
Tests for credential manager and auth endpoints.
|
||||
"""
|
||||
import pytest
|
||||
import json
|
||||
from unittest.mock import MagicMock, patch
|
||||
from datetime import datetime
|
||||
|
||||
from backend.services.credential_manager import (
|
||||
CredentialManager,
|
||||
CredentialStatus,
|
||||
CredentialCheckResult,
|
||||
DeviceAuthInfo,
|
||||
get_credential_manager,
|
||||
)
|
||||
|
||||
|
||||
class TestCredentialStatus:
|
||||
"""Tests for CredentialStatus enum."""
|
||||
|
||||
def test_status_values(self):
|
||||
"""Test all status values exist."""
|
||||
assert CredentialStatus.VALID == "valid"
|
||||
assert CredentialStatus.EXPIRED == "expired"
|
||||
assert CredentialStatus.INVALID == "invalid"
|
||||
assert CredentialStatus.NOT_CONFIGURED == "not_configured"
|
||||
assert CredentialStatus.ERROR == "error"
|
||||
|
||||
|
||||
class TestCredentialCheckResult:
|
||||
"""Tests for CredentialCheckResult dataclass."""
|
||||
|
||||
def test_create_result(self):
|
||||
"""Test creating a check result."""
|
||||
result = CredentialCheckResult(
|
||||
service="youtube",
|
||||
status=CredentialStatus.VALID,
|
||||
message="Credentials are valid",
|
||||
last_checked=datetime.utcnow()
|
||||
)
|
||||
|
||||
assert result.service == "youtube"
|
||||
assert result.status == CredentialStatus.VALID
|
||||
assert result.message == "Credentials are valid"
|
||||
assert result.expires_at is None
|
||||
|
||||
def test_create_result_with_expiry(self):
|
||||
"""Test creating a check result with expiry."""
|
||||
expiry = datetime.utcnow()
|
||||
result = CredentialCheckResult(
|
||||
service="gdrive",
|
||||
status=CredentialStatus.VALID,
|
||||
message="Valid",
|
||||
last_checked=datetime.utcnow(),
|
||||
expires_at=expiry
|
||||
)
|
||||
|
||||
assert result.expires_at == expiry
|
||||
|
||||
|
||||
class TestCredentialManager:
|
||||
"""Tests for CredentialManager class."""
|
||||
|
||||
def test_init(self):
|
||||
"""Test manager initialization."""
|
||||
manager = CredentialManager()
|
||||
assert manager._pending_device_auths == {}
|
||||
|
||||
def test_check_youtube_not_configured(self):
|
||||
"""Test YouTube check when not configured."""
|
||||
with patch('backend.services.credential_manager.get_settings') as mock_get_settings:
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.get_secret.return_value = None
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
manager = CredentialManager()
|
||||
result = manager.check_youtube_credentials()
|
||||
|
||||
assert result.status == CredentialStatus.NOT_CONFIGURED
|
||||
assert "not configured" in result.message.lower()
|
||||
|
||||
def test_check_youtube_invalid_json(self):
|
||||
"""Test YouTube check with invalid JSON."""
|
||||
with patch('backend.services.credential_manager.get_settings') as mock_get_settings:
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.get_secret.return_value = "not valid json"
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
manager = CredentialManager()
|
||||
result = manager.check_youtube_credentials()
|
||||
|
||||
assert result.status == CredentialStatus.INVALID
|
||||
assert "json" in result.message.lower()
|
||||
|
||||
def test_check_youtube_missing_fields(self):
|
||||
"""Test YouTube check with missing required fields."""
|
||||
# Missing refresh_token and client_secret
|
||||
creds = json.dumps({"token": "test", "client_id": "test"})
|
||||
|
||||
with patch('backend.services.credential_manager.get_settings') as mock_get_settings:
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.get_secret.return_value = creds
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
manager = CredentialManager()
|
||||
result = manager.check_youtube_credentials()
|
||||
|
||||
assert result.status == CredentialStatus.INVALID
|
||||
assert "missing" in result.message.lower()
|
||||
|
||||
def test_check_gdrive_not_configured(self):
|
||||
"""Test Google Drive check when not configured."""
|
||||
with patch('backend.services.credential_manager.get_settings') as mock_get_settings:
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.get_secret.return_value = None
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
manager = CredentialManager()
|
||||
result = manager.check_gdrive_credentials()
|
||||
|
||||
assert result.status == CredentialStatus.NOT_CONFIGURED
|
||||
|
||||
def test_check_dropbox_not_configured(self):
|
||||
"""Test Dropbox check when not configured."""
|
||||
with patch('backend.services.credential_manager.get_settings') as mock_get_settings:
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.get_secret.return_value = None
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
manager = CredentialManager()
|
||||
result = manager.check_dropbox_credentials()
|
||||
|
||||
assert result.status == CredentialStatus.NOT_CONFIGURED
|
||||
|
||||
def test_check_dropbox_missing_access_token(self):
|
||||
"""Test Dropbox check with missing access token."""
|
||||
creds = json.dumps({"refresh_token": "test"}) # No access_token
|
||||
|
||||
with patch('backend.services.credential_manager.get_settings') as mock_get_settings:
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.get_secret.return_value = creds
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
manager = CredentialManager()
|
||||
result = manager.check_dropbox_credentials()
|
||||
|
||||
assert result.status == CredentialStatus.INVALID
|
||||
assert "access_token" in result.message.lower()
|
||||
|
||||
def test_check_all_credentials(self):
|
||||
"""Test checking all credentials."""
|
||||
with patch('backend.services.credential_manager.get_settings') as mock_get_settings:
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.get_secret.return_value = None
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
manager = CredentialManager()
|
||||
results = manager.check_all_credentials()
|
||||
|
||||
assert "youtube" in results
|
||||
assert "gdrive" in results
|
||||
assert "dropbox" in results
|
||||
assert all(r.status == CredentialStatus.NOT_CONFIGURED for r in results.values())
|
||||
|
||||
def test_test_youtube_api_failure(self):
|
||||
"""Test YouTube API test failure."""
|
||||
manager = CredentialManager()
|
||||
|
||||
mock_creds = MagicMock()
|
||||
|
||||
with patch('googleapiclient.discovery.build', side_effect=Exception("API Error")):
|
||||
result = manager._test_youtube_api(mock_creds)
|
||||
|
||||
assert result is False
|
||||
|
||||
def test_test_gdrive_api_failure(self):
|
||||
"""Test Google Drive API test failure."""
|
||||
manager = CredentialManager()
|
||||
|
||||
mock_creds = MagicMock()
|
||||
|
||||
with patch('googleapiclient.discovery.build', side_effect=Exception("API Error")):
|
||||
result = manager._test_gdrive_api(mock_creds)
|
||||
|
||||
assert result is False
|
||||
|
||||
def test_test_dropbox_api_failure(self):
|
||||
"""Test Dropbox API test failure."""
|
||||
manager = CredentialManager()
|
||||
|
||||
with patch('dropbox.Dropbox', side_effect=Exception("API Error")):
|
||||
result = manager._test_dropbox_api({"access_token": "test"})
|
||||
|
||||
assert result is False
|
||||
|
||||
def test_send_credential_alert_no_webhook(self):
|
||||
"""Test alert without webhook URL."""
|
||||
manager = CredentialManager()
|
||||
|
||||
result = manager.send_credential_alert([], discord_webhook_url=None)
|
||||
|
||||
assert result is False
|
||||
|
||||
def test_send_credential_alert_success(self):
|
||||
"""Test successful alert sending."""
|
||||
manager = CredentialManager()
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
|
||||
invalid_services = [
|
||||
CredentialCheckResult(
|
||||
service="youtube",
|
||||
status=CredentialStatus.INVALID,
|
||||
message="Test message",
|
||||
last_checked=datetime.utcnow()
|
||||
)
|
||||
]
|
||||
|
||||
with patch('requests.post', return_value=mock_response) as mock_post:
|
||||
result = manager.send_credential_alert(
|
||||
invalid_services,
|
||||
discord_webhook_url="https://discord.com/webhook/test"
|
||||
)
|
||||
|
||||
assert result is True
|
||||
mock_post.assert_called_once()
|
||||
|
||||
def test_start_youtube_device_auth(self):
|
||||
"""Test starting YouTube device auth flow."""
|
||||
manager = CredentialManager()
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
"device_code": "test_device_code",
|
||||
"user_code": "TEST-CODE",
|
||||
"verification_uri": "https://google.com/device",
|
||||
"expires_in": 1800,
|
||||
"interval": 5
|
||||
}
|
||||
|
||||
with patch('requests.post', return_value=mock_response):
|
||||
device_info = manager.start_youtube_device_auth(
|
||||
client_id="test_client",
|
||||
client_secret="test_secret"
|
||||
)
|
||||
|
||||
assert device_info.device_code == "test_device_code"
|
||||
assert device_info.user_code == "TEST-CODE"
|
||||
assert device_info.verification_url == "https://google.com/device"
|
||||
|
||||
# Should be stored for polling
|
||||
key = f"youtube:{device_info.device_code}"
|
||||
assert key in manager._pending_device_auths
|
||||
|
||||
def test_poll_device_auth_no_client_creds(self):
|
||||
"""Test polling when client credentials are not in Secret Manager."""
|
||||
manager = CredentialManager()
|
||||
|
||||
# Mock get_youtube_client_credentials to return None (no creds)
|
||||
with patch.object(manager, 'get_youtube_client_credentials', return_value=None):
|
||||
status, data = manager.poll_device_auth("youtube", "some_code")
|
||||
|
||||
assert status == "error"
|
||||
assert "client credentials not found" in data["message"].lower()
|
||||
|
||||
def test_poll_device_auth_expired(self):
|
||||
"""Test polling with expired device code."""
|
||||
manager = CredentialManager()
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 400
|
||||
mock_response.json.return_value = {"error": "expired_token"}
|
||||
|
||||
with patch.object(manager, 'get_youtube_client_credentials',
|
||||
return_value={"client_id": "test", "client_secret": "test"}):
|
||||
with patch('requests.post', return_value=mock_response):
|
||||
status, data = manager.poll_device_auth("youtube", "expired_code")
|
||||
|
||||
assert status == "expired"
|
||||
assert "expired" in data["message"].lower()
|
||||
|
||||
def test_poll_device_auth_pending(self):
|
||||
"""Test polling when authorization is pending."""
|
||||
manager = CredentialManager()
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 400
|
||||
mock_response.json.return_value = {"error": "authorization_pending"}
|
||||
|
||||
with patch.object(manager, 'get_youtube_client_credentials',
|
||||
return_value={"client_id": "test", "client_secret": "test"}):
|
||||
with patch('requests.post', return_value=mock_response):
|
||||
status, data = manager.poll_device_auth("youtube", "pending_code")
|
||||
|
||||
assert status == "pending"
|
||||
assert "waiting" in data["message"].lower()
|
||||
|
||||
def test_poll_device_auth_success(self):
|
||||
"""Test polling when authorization completes successfully."""
|
||||
manager = CredentialManager()
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
"access_token": "new_access_token",
|
||||
"refresh_token": "new_refresh_token"
|
||||
}
|
||||
|
||||
with patch.object(manager, 'get_youtube_client_credentials',
|
||||
return_value={"client_id": "test_id", "client_secret": "test_secret"}):
|
||||
with patch('requests.post', return_value=mock_response):
|
||||
with patch.object(manager, '_save_credentials_to_secret', return_value=True):
|
||||
status, data = manager.poll_device_auth("youtube", "completed_code")
|
||||
|
||||
assert status == "complete"
|
||||
assert data["token"] == "new_access_token"
|
||||
assert data["refresh_token"] == "new_refresh_token"
|
||||
assert data["client_id"] == "test_id"
|
||||
|
||||
|
||||
class TestGetCredentialManager:
|
||||
"""Tests for singleton factory function."""
|
||||
|
||||
def test_returns_singleton(self):
|
||||
"""Test that factory returns same instance."""
|
||||
# Reset singleton
|
||||
import backend.services.credential_manager as module
|
||||
module._credential_manager = None
|
||||
|
||||
manager1 = get_credential_manager()
|
||||
manager2 = get_credential_manager()
|
||||
|
||||
assert manager1 is manager2
|
||||
|
||||
|
||||
class TestAuthRoutes:
|
||||
"""Tests for auth API routes."""
|
||||
|
||||
def test_routes_import(self):
|
||||
"""Test that routes can be imported."""
|
||||
from backend.api.routes.auth import router
|
||||
assert router is not None
|
||||
|
||||
def test_status_endpoint_exists(self):
|
||||
"""Test that status endpoint is defined."""
|
||||
from backend.api.routes.auth import get_credentials_status
|
||||
assert get_credentials_status is not None
|
||||
|
||||
def test_validate_endpoint_exists(self):
|
||||
"""Test that validate endpoint is defined."""
|
||||
from backend.api.routes.auth import validate_credentials
|
||||
assert validate_credentials is not None
|
||||
|
||||
def test_device_auth_endpoint_exists(self):
|
||||
"""Test that device auth endpoints are defined."""
|
||||
from backend.api.routes.auth import (
|
||||
start_youtube_device_auth,
|
||||
poll_youtube_device_auth,
|
||||
start_gdrive_device_auth,
|
||||
poll_gdrive_device_auth,
|
||||
)
|
||||
assert start_youtube_device_auth is not None
|
||||
assert poll_youtube_device_auth is not None
|
||||
assert start_gdrive_device_auth is not None
|
||||
assert poll_gdrive_device_auth is not None
|
||||
|
||||
|
||||
class TestFileUploadCredentialValidation:
|
||||
"""Tests for credential validation in file upload."""
|
||||
|
||||
def test_credential_manager_imported(self):
|
||||
"""Test that credential manager is imported in file upload."""
|
||||
from backend.api.routes.file_upload import get_credential_manager, CredentialStatus
|
||||
assert get_credential_manager is not None
|
||||
assert CredentialStatus is not None
|
||||
@@ -0,0 +1,54 @@
|
||||
"""
|
||||
Tests for API dependencies (authentication, authorization).
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
class TestAuthDependencies:
|
||||
"""Tests for authentication dependencies module structure."""
|
||||
|
||||
def test_get_token_from_request_function_exists(self):
|
||||
"""Test get_token_from_request function exists."""
|
||||
from backend.api.dependencies import get_token_from_request
|
||||
assert get_token_from_request is not None
|
||||
|
||||
def test_require_auth_function_exists(self):
|
||||
"""Test require_auth function exists."""
|
||||
from backend.api.dependencies import require_auth
|
||||
assert require_auth is not None
|
||||
|
||||
def test_require_admin_function_exists(self):
|
||||
"""Test require_admin function exists."""
|
||||
from backend.api.dependencies import require_admin
|
||||
assert require_admin is not None
|
||||
|
||||
def test_optional_auth_function_exists(self):
|
||||
"""Test optional_auth function exists."""
|
||||
from backend.api.dependencies import optional_auth
|
||||
assert optional_auth is not None
|
||||
|
||||
def test_security_scheme_defined(self):
|
||||
"""Test HTTP Bearer security scheme is defined."""
|
||||
from backend.api.dependencies import security
|
||||
assert security is not None
|
||||
|
||||
def test_logger_configured(self):
|
||||
"""Test logger is configured."""
|
||||
from backend.api.dependencies import logger
|
||||
assert logger is not None
|
||||
|
||||
|
||||
class TestAuthServiceAccess:
|
||||
"""Tests for auth service access."""
|
||||
|
||||
def test_get_auth_service_function_exists(self):
|
||||
"""Test get_auth_service function exists."""
|
||||
from backend.services.auth_service import get_auth_service
|
||||
assert get_auth_service is not None
|
||||
|
||||
def test_user_type_enum_defined(self):
|
||||
"""Test UserType enum is defined."""
|
||||
from backend.services.auth_service import UserType
|
||||
assert UserType is not None
|
||||
|
||||
@@ -0,0 +1,820 @@
|
||||
"""
|
||||
Tests for native distribution services (Dropbox and Google Drive).
|
||||
|
||||
These tests verify the service interfaces without requiring actual
|
||||
cloud credentials, using mocks to simulate API responses.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch, mock_open
|
||||
|
||||
|
||||
class TestDropboxService:
|
||||
"""Tests for the DropboxService class."""
|
||||
|
||||
def test_module_imports(self):
|
||||
"""Test that the module imports correctly."""
|
||||
from backend.services.dropbox_service import DropboxService, get_dropbox_service
|
||||
assert DropboxService is not None
|
||||
assert get_dropbox_service is not None
|
||||
|
||||
def test_init_creates_instance(self):
|
||||
"""Test that we can create a DropboxService instance."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
service = DropboxService()
|
||||
assert service is not None
|
||||
assert service._client is None # Client not initialized until first use
|
||||
assert service._is_configured is False
|
||||
|
||||
def test_is_configured_returns_false_without_credentials(self):
|
||||
"""Test that is_configured returns False when credentials are missing."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
with patch.object(DropboxService, '_load_credentials', return_value=None):
|
||||
service = DropboxService()
|
||||
assert service.is_configured is False
|
||||
|
||||
def test_is_configured_returns_true_with_credentials(self):
|
||||
"""Test that is_configured returns True when credentials are present."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
mock_creds = {"access_token": "test_token"}
|
||||
with patch.object(DropboxService, '_load_credentials', return_value=mock_creds):
|
||||
service = DropboxService()
|
||||
assert service.is_configured is True
|
||||
|
||||
def test_is_configured_caches_result(self):
|
||||
"""Test that is_configured caches the result."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
service = DropboxService()
|
||||
service._is_configured = True
|
||||
# Should return True without calling _load_credentials
|
||||
assert service.is_configured is True
|
||||
|
||||
def test_is_configured_returns_false_without_access_token(self):
|
||||
"""Test that is_configured returns False when access_token is missing."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
mock_creds = {"refresh_token": "test"} # No access_token
|
||||
with patch.object(DropboxService, '_load_credentials', return_value=mock_creds):
|
||||
service = DropboxService()
|
||||
assert service.is_configured is False
|
||||
|
||||
def test_get_next_brand_code_first_track(self):
|
||||
"""Test brand code calculation when no existing tracks."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
service = DropboxService()
|
||||
with patch.object(service, 'list_folders', return_value=[]):
|
||||
brand_code = service.get_next_brand_code("/test/path", "NOMAD")
|
||||
assert brand_code == "NOMAD-0001"
|
||||
|
||||
def test_get_next_brand_code_sequential(self):
|
||||
"""Test brand code calculation with existing tracks."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
service = DropboxService()
|
||||
existing_folders = [
|
||||
"NOMAD-0001 - Artist1 - Song1",
|
||||
"NOMAD-0002 - Artist2 - Song2",
|
||||
"NOMAD-0005 - Artist3 - Song3", # Gap in sequence
|
||||
"OTHER-0001 - Different Brand", # Different brand
|
||||
]
|
||||
with patch.object(service, 'list_folders', return_value=existing_folders):
|
||||
brand_code = service.get_next_brand_code("/test/path", "NOMAD")
|
||||
assert brand_code == "NOMAD-0006"
|
||||
|
||||
def test_get_next_brand_code_different_prefix(self):
|
||||
"""Test brand code calculation with different brand prefix."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
service = DropboxService()
|
||||
existing_folders = [
|
||||
"NOMAD-0001 - Artist1 - Song1",
|
||||
"TEST-0010 - Artist2 - Song2",
|
||||
]
|
||||
with patch.object(service, 'list_folders', return_value=existing_folders):
|
||||
brand_code = service.get_next_brand_code("/test/path", "TEST")
|
||||
assert brand_code == "TEST-0011"
|
||||
|
||||
def test_factory_function_returns_instance(self):
|
||||
"""Test that get_dropbox_service returns a service instance."""
|
||||
from backend.services.dropbox_service import get_dropbox_service
|
||||
|
||||
service = get_dropbox_service()
|
||||
assert service is not None
|
||||
|
||||
def test_load_credentials_handles_exception(self):
|
||||
"""Test that _load_credentials handles exceptions gracefully."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
with patch('backend.services.dropbox_service.secretmanager.SecretManagerServiceClient') as mock_client:
|
||||
mock_client.return_value.access_secret_version.side_effect = Exception("API Error")
|
||||
service = DropboxService()
|
||||
result = service._load_credentials()
|
||||
assert result is None
|
||||
|
||||
def test_load_credentials_parses_json(self):
|
||||
"""Test that _load_credentials correctly parses JSON credentials."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
mock_creds = {"access_token": "test_token", "refresh_token": "test_refresh"}
|
||||
mock_response = MagicMock()
|
||||
mock_response.payload.data.decode.return_value = json.dumps(mock_creds)
|
||||
|
||||
with patch('backend.services.dropbox_service.secretmanager.SecretManagerServiceClient') as mock_client:
|
||||
mock_client.return_value.access_secret_version.return_value = mock_response
|
||||
service = DropboxService()
|
||||
result = service._load_credentials()
|
||||
assert result == mock_creds
|
||||
|
||||
def test_client_property_raises_without_credentials(self):
|
||||
"""Test that client property raises error without credentials."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
with patch.object(DropboxService, '_load_credentials', return_value=None):
|
||||
service = DropboxService()
|
||||
with pytest.raises(RuntimeError, match="credentials not configured"):
|
||||
_ = service.client
|
||||
|
||||
def test_client_property_raises_import_error(self):
|
||||
"""Test that client property raises helpful error if dropbox not installed."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
import sys
|
||||
|
||||
mock_creds = {"access_token": "test_token"}
|
||||
with patch.object(DropboxService, '_load_credentials', return_value=mock_creds):
|
||||
with patch.dict(sys.modules, {'dropbox': None}):
|
||||
service = DropboxService()
|
||||
# Force reimport to fail
|
||||
with patch('builtins.__import__', side_effect=ImportError("No module named 'dropbox'")):
|
||||
with pytest.raises(ImportError, match="dropbox package"):
|
||||
_ = service.client
|
||||
|
||||
def test_list_folders_adds_leading_slash(self):
|
||||
"""Test that list_folders adds leading slash if missing."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
service = DropboxService()
|
||||
mock_client = MagicMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.entries = []
|
||||
mock_result.has_more = False
|
||||
mock_client.files_list_folder.return_value = mock_result
|
||||
service._client = mock_client
|
||||
|
||||
service.list_folders("test/path") # No leading slash
|
||||
|
||||
# Verify it was called with leading slash
|
||||
mock_client.files_list_folder.assert_called_with("/test/path")
|
||||
|
||||
def test_upload_file_adds_leading_slash(self):
|
||||
"""Test that upload_file adds leading slash to remote path."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
import tempfile
|
||||
|
||||
service = DropboxService()
|
||||
mock_client = MagicMock()
|
||||
service._client = mock_client
|
||||
|
||||
# Create a small temp file
|
||||
with tempfile.NamedTemporaryFile(delete=False, mode='wb') as f:
|
||||
f.write(b"test content")
|
||||
temp_path = f.name
|
||||
|
||||
try:
|
||||
service.upload_file(temp_path, "remote/path.txt") # No leading slash
|
||||
# Verify upload was called with leading slash
|
||||
mock_client.files_upload.assert_called_once()
|
||||
call_args = mock_client.files_upload.call_args
|
||||
assert call_args[0][1] == "/remote/path.txt"
|
||||
finally:
|
||||
os.unlink(temp_path)
|
||||
|
||||
def test_upload_folder_uploads_all_files(self):
|
||||
"""Test that upload_folder uploads all files in directory."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
import tempfile
|
||||
|
||||
service = DropboxService()
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create some test files
|
||||
(open(os.path.join(tmpdir, "file1.txt"), "w")).write("content1")
|
||||
(open(os.path.join(tmpdir, "file2.txt"), "w")).write("content2")
|
||||
|
||||
with patch.object(service, 'upload_file') as mock_upload:
|
||||
service.upload_folder(tmpdir, "/remote/folder")
|
||||
|
||||
assert mock_upload.call_count == 2
|
||||
|
||||
def test_create_shared_link_success(self):
|
||||
"""Test successful shared link creation."""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
service = DropboxService()
|
||||
mock_client = MagicMock()
|
||||
mock_link = MagicMock()
|
||||
mock_link.url = "https://dropbox.com/shared/test"
|
||||
mock_client.sharing_create_shared_link_with_settings.return_value = mock_link
|
||||
service._client = mock_client
|
||||
|
||||
result = service.create_shared_link("/test/path")
|
||||
assert result == "https://dropbox.com/shared/test"
|
||||
|
||||
def test_sharing_list_shared_links_mock_setup(self):
|
||||
"""Test that mock setup for sharing_list_shared_links works correctly.
|
||||
|
||||
Note: Properly mocking Dropbox's ApiError is complex because it
|
||||
requires specific exception class structure. This test verifies
|
||||
the mock configuration is correct for the success path, which is
|
||||
a prerequisite for more complex error-handling tests.
|
||||
"""
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
|
||||
service = DropboxService()
|
||||
mock_client = MagicMock()
|
||||
|
||||
# Mock the existing link retrieval (success case)
|
||||
mock_existing_link = MagicMock()
|
||||
mock_existing_link.url = "https://dropbox.com/existing/link"
|
||||
mock_links_result = MagicMock()
|
||||
mock_links_result.links = [mock_existing_link]
|
||||
mock_client.sharing_list_shared_links.return_value = mock_links_result
|
||||
|
||||
# Assign the mock client to the service
|
||||
service._client = mock_client
|
||||
|
||||
# Verify the mock returns the expected link structure
|
||||
result = mock_client.sharing_list_shared_links(path="/test/path")
|
||||
assert result.links[0].url == "https://dropbox.com/existing/link"
|
||||
mock_client.sharing_list_shared_links.assert_called_once_with(path="/test/path")
|
||||
|
||||
|
||||
class TestGoogleDriveService:
|
||||
"""Tests for the GoogleDriveService class."""
|
||||
|
||||
def test_module_imports(self):
|
||||
"""Test that the module imports correctly."""
|
||||
from backend.services.gdrive_service import GoogleDriveService, get_gdrive_service
|
||||
assert GoogleDriveService is not None
|
||||
assert get_gdrive_service is not None
|
||||
|
||||
def test_init_creates_instance(self):
|
||||
"""Test that we can create a GoogleDriveService instance."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
service = GoogleDriveService()
|
||||
assert service is not None
|
||||
assert service._service is None # Service not initialized until first use
|
||||
assert service._loaded is False
|
||||
|
||||
def test_is_configured_returns_false_without_credentials(self):
|
||||
"""Test that is_configured returns False when credentials are missing."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
with patch.object(GoogleDriveService, '_load_credentials', return_value=None):
|
||||
service = GoogleDriveService()
|
||||
assert service.is_configured is False
|
||||
|
||||
def test_is_configured_returns_true_with_credentials(self):
|
||||
"""Test that is_configured returns True when credentials are present."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_creds = {
|
||||
"token": "test_token",
|
||||
"refresh_token": "test_refresh",
|
||||
"client_id": "test_client_id",
|
||||
"client_secret": "test_client_secret",
|
||||
}
|
||||
with patch.object(GoogleDriveService, '_load_credentials', return_value=mock_creds):
|
||||
service = GoogleDriveService()
|
||||
assert service.is_configured is True
|
||||
|
||||
def test_factory_function_returns_singleton(self):
|
||||
"""Test that get_gdrive_service returns a singleton."""
|
||||
from backend.services.gdrive_service import get_gdrive_service, _gdrive_service
|
||||
|
||||
# Reset singleton
|
||||
import backend.services.gdrive_service as gdrive_module
|
||||
gdrive_module._gdrive_service = None
|
||||
|
||||
service1 = get_gdrive_service()
|
||||
service2 = get_gdrive_service()
|
||||
assert service1 is service2
|
||||
|
||||
def test_load_credentials_caches_result(self):
|
||||
"""Test that _load_credentials caches the result."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
service = GoogleDriveService()
|
||||
service._loaded = True
|
||||
service._credentials_data = {"cached": "data"}
|
||||
|
||||
# Should return cached data without calling settings
|
||||
result = service._load_credentials()
|
||||
assert result == {"cached": "data"}
|
||||
|
||||
def test_load_credentials_falls_back_to_youtube(self):
|
||||
"""Test that _load_credentials falls back to YouTube credentials."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.get_secret.side_effect = [
|
||||
None, # gdrive-oauth-credentials not found
|
||||
'{"refresh_token": "yt_refresh", "client_id": "yt_client", "client_secret": "yt_secret"}', # youtube-oauth-credentials
|
||||
]
|
||||
|
||||
service = GoogleDriveService()
|
||||
service.settings = mock_settings
|
||||
|
||||
result = service._load_credentials()
|
||||
assert result is not None
|
||||
assert result["refresh_token"] == "yt_refresh"
|
||||
|
||||
def test_load_credentials_returns_none_when_both_fail(self):
|
||||
"""Test that _load_credentials returns None when no credentials available."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.get_secret.return_value = None
|
||||
|
||||
service = GoogleDriveService()
|
||||
service.settings = mock_settings
|
||||
|
||||
result = service._load_credentials()
|
||||
assert result is None
|
||||
assert service._loaded is True
|
||||
|
||||
def test_load_credentials_validates_required_fields(self):
|
||||
"""Test that _load_credentials validates required fields."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = MagicMock()
|
||||
# Missing client_secret
|
||||
mock_settings.get_secret.return_value = '{"refresh_token": "test", "client_id": "test"}'
|
||||
|
||||
service = GoogleDriveService()
|
||||
service.settings = mock_settings
|
||||
|
||||
result = service._load_credentials()
|
||||
assert result is None # Should fail validation
|
||||
|
||||
def test_load_credentials_handles_json_error(self):
|
||||
"""Test that _load_credentials handles JSON parse errors."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.get_secret.return_value = "not valid json"
|
||||
|
||||
service = GoogleDriveService()
|
||||
service.settings = mock_settings
|
||||
|
||||
result = service._load_credentials()
|
||||
assert result is None
|
||||
|
||||
def test_service_property_raises_without_credentials(self):
|
||||
"""Test that service property raises error without credentials."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
with patch.object(GoogleDriveService, '_load_credentials', return_value=None):
|
||||
service = GoogleDriveService()
|
||||
with pytest.raises(RuntimeError, match="credentials not configured"):
|
||||
_ = service.service
|
||||
|
||||
def test_get_or_create_folder_finds_existing(self):
|
||||
"""Test that get_or_create_folder finds existing folders."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
service = GoogleDriveService()
|
||||
mock_service = MagicMock()
|
||||
mock_files = MagicMock()
|
||||
mock_list = MagicMock()
|
||||
mock_list.execute.return_value = {"files": [{"id": "existing_folder_id", "name": "TestFolder"}]}
|
||||
mock_files.list.return_value = mock_list
|
||||
mock_service.files.return_value = mock_files
|
||||
service._service = mock_service
|
||||
|
||||
result = service.get_or_create_folder("parent_id", "TestFolder")
|
||||
assert result == "existing_folder_id"
|
||||
|
||||
def test_get_or_create_folder_creates_new(self):
|
||||
"""Test that get_or_create_folder creates new folders when not found."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
service = GoogleDriveService()
|
||||
mock_service = MagicMock()
|
||||
mock_files = MagicMock()
|
||||
|
||||
# First call - folder doesn't exist
|
||||
mock_list = MagicMock()
|
||||
mock_list.execute.return_value = {"files": []}
|
||||
mock_files.list.return_value = mock_list
|
||||
|
||||
# Second call - create folder
|
||||
mock_create = MagicMock()
|
||||
mock_create.execute.return_value = {"id": "new_folder_id"}
|
||||
mock_files.create.return_value = mock_create
|
||||
|
||||
mock_service.files.return_value = mock_files
|
||||
service._service = mock_service
|
||||
|
||||
result = service.get_or_create_folder("parent_id", "NewFolder")
|
||||
assert result == "new_folder_id"
|
||||
|
||||
def test_upload_file_determines_mime_type(self):
|
||||
"""Test that upload_file determines correct MIME type."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
import tempfile
|
||||
|
||||
service = GoogleDriveService()
|
||||
mock_service = MagicMock()
|
||||
mock_files = MagicMock()
|
||||
|
||||
# Mock list (for replace_existing check)
|
||||
mock_list = MagicMock()
|
||||
mock_list.execute.return_value = {"files": []}
|
||||
mock_files.list.return_value = mock_list
|
||||
|
||||
# Mock create
|
||||
mock_create = MagicMock()
|
||||
mock_create.execute.return_value = {"id": "uploaded_file_id"}
|
||||
mock_files.create.return_value = mock_create
|
||||
|
||||
mock_service.files.return_value = mock_files
|
||||
service._service = mock_service
|
||||
|
||||
# Create a temp file with .mp4 extension
|
||||
with tempfile.NamedTemporaryFile(suffix='.mp4', delete=False) as f:
|
||||
f.write(b"fake video content")
|
||||
temp_path = f.name
|
||||
|
||||
try:
|
||||
result = service.upload_file(temp_path, "parent_id", "test.mp4")
|
||||
assert result == "uploaded_file_id"
|
||||
finally:
|
||||
os.unlink(temp_path)
|
||||
|
||||
def test_upload_file_replaces_existing(self):
|
||||
"""Test that upload_file can replace existing files."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
import tempfile
|
||||
|
||||
service = GoogleDriveService()
|
||||
mock_service = MagicMock()
|
||||
mock_files = MagicMock()
|
||||
|
||||
# Mock list - file exists
|
||||
mock_list = MagicMock()
|
||||
mock_list.execute.return_value = {"files": [{"id": "existing_id"}]}
|
||||
mock_files.list.return_value = mock_list
|
||||
|
||||
# Mock delete
|
||||
mock_delete = MagicMock()
|
||||
mock_files.delete.return_value = mock_delete
|
||||
|
||||
# Mock create
|
||||
mock_create = MagicMock()
|
||||
mock_create.execute.return_value = {"id": "new_file_id"}
|
||||
mock_files.create.return_value = mock_create
|
||||
|
||||
mock_service.files.return_value = mock_files
|
||||
service._service = mock_service
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix='.txt', delete=False) as f:
|
||||
f.write(b"content")
|
||||
temp_path = f.name
|
||||
|
||||
try:
|
||||
result = service.upload_file(temp_path, "parent_id", "test.txt", replace_existing=True)
|
||||
assert result == "new_file_id"
|
||||
mock_files.delete.assert_called_once()
|
||||
finally:
|
||||
os.unlink(temp_path)
|
||||
|
||||
def test_upload_to_public_share_creates_structure(self):
|
||||
"""Test that upload_to_public_share creates folder structure."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
import tempfile
|
||||
|
||||
service = GoogleDriveService()
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create test files
|
||||
mp4_path = os.path.join(tmpdir, "test.mp4")
|
||||
mp4_720_path = os.path.join(tmpdir, "test_720p.mp4")
|
||||
cdg_path = os.path.join(tmpdir, "test.zip")
|
||||
|
||||
open(mp4_path, 'wb').write(b"mp4 content")
|
||||
open(mp4_720_path, 'wb').write(b"720p content")
|
||||
open(cdg_path, 'wb').write(b"cdg content")
|
||||
|
||||
with patch.object(service, 'get_or_create_folder', return_value="folder_id"):
|
||||
with patch.object(service, 'upload_file', return_value="file_id"):
|
||||
result = service.upload_to_public_share(
|
||||
root_folder_id="root_id",
|
||||
brand_code="TEST-0001",
|
||||
base_name="Artist - Title",
|
||||
output_files={
|
||||
"final_karaoke_lossy_mp4": mp4_path,
|
||||
"final_karaoke_lossy_720p_mp4": mp4_720_path,
|
||||
"final_karaoke_cdg_zip": cdg_path,
|
||||
}
|
||||
)
|
||||
|
||||
assert "mp4" in result
|
||||
assert "mp4_720p" in result
|
||||
assert "cdg" in result
|
||||
|
||||
def test_upload_to_public_share_handles_missing_files(self):
|
||||
"""Test that upload_to_public_share handles missing files gracefully."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
service = GoogleDriveService()
|
||||
|
||||
with patch.object(service, 'get_or_create_folder', return_value="folder_id"):
|
||||
with patch.object(service, 'upload_file', return_value="file_id"):
|
||||
result = service.upload_to_public_share(
|
||||
root_folder_id="root_id",
|
||||
brand_code="TEST-0001",
|
||||
base_name="Artist - Title",
|
||||
output_files={
|
||||
"final_karaoke_lossy_mp4": "/nonexistent/path.mp4",
|
||||
}
|
||||
)
|
||||
|
||||
# Should return empty since file doesn't exist
|
||||
assert result == {}
|
||||
|
||||
|
||||
class TestJobModelDistributionFields:
|
||||
"""Tests for distribution fields in Job model."""
|
||||
|
||||
def test_job_model_has_dropbox_path_field(self):
|
||||
"""Test that Job model has dropbox_path field."""
|
||||
from backend.models.job import Job, JobStatus
|
||||
from datetime import datetime
|
||||
|
||||
job = Job(
|
||||
job_id="test-123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
dropbox_path="/Karaoke/Tracks-Organized",
|
||||
)
|
||||
assert job.dropbox_path == "/Karaoke/Tracks-Organized"
|
||||
|
||||
def test_job_model_has_gdrive_folder_id_field(self):
|
||||
"""Test that Job model has gdrive_folder_id field."""
|
||||
from backend.models.job import Job, JobStatus
|
||||
from datetime import datetime
|
||||
|
||||
job = Job(
|
||||
job_id="test-123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
gdrive_folder_id="1abc123xyz",
|
||||
)
|
||||
assert job.gdrive_folder_id == "1abc123xyz"
|
||||
|
||||
def test_job_model_distribution_fields_optional(self):
|
||||
"""Test that distribution fields are optional."""
|
||||
from backend.models.job import Job, JobStatus
|
||||
from datetime import datetime
|
||||
|
||||
job = Job(
|
||||
job_id="test-123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
assert job.dropbox_path is None
|
||||
assert job.gdrive_folder_id is None
|
||||
|
||||
|
||||
class TestJobCreateDistributionFields:
|
||||
"""Tests for distribution fields in JobCreate model."""
|
||||
|
||||
def test_job_create_has_dropbox_path_field(self):
|
||||
"""Test that JobCreate model has dropbox_path field."""
|
||||
from backend.models.job import JobCreate
|
||||
|
||||
job_create = JobCreate(
|
||||
artist="Test Artist",
|
||||
title="Test Title",
|
||||
dropbox_path="/Karaoke/Tracks-Organized",
|
||||
)
|
||||
assert job_create.dropbox_path == "/Karaoke/Tracks-Organized"
|
||||
|
||||
def test_job_create_has_gdrive_folder_id_field(self):
|
||||
"""Test that JobCreate model has gdrive_folder_id field."""
|
||||
from backend.models.job import JobCreate
|
||||
|
||||
job_create = JobCreate(
|
||||
artist="Test Artist",
|
||||
title="Test Title",
|
||||
gdrive_folder_id="1abc123xyz",
|
||||
)
|
||||
assert job_create.gdrive_folder_id == "1abc123xyz"
|
||||
|
||||
def test_job_create_distribution_fields_optional(self):
|
||||
"""Test that distribution fields are optional in JobCreate."""
|
||||
from backend.models.job import JobCreate
|
||||
|
||||
job_create = JobCreate(
|
||||
artist="Test Artist",
|
||||
title="Test Title",
|
||||
)
|
||||
assert job_create.dropbox_path is None
|
||||
assert job_create.gdrive_folder_id is None
|
||||
|
||||
|
||||
class TestFileUploadDistributionParams:
|
||||
"""Tests for distribution parameters in file upload endpoint."""
|
||||
|
||||
def test_endpoint_has_dropbox_path_parameter(self):
|
||||
"""Test that upload endpoint signature includes dropbox_path parameter."""
|
||||
import inspect
|
||||
from backend.api.routes.file_upload import upload_and_create_job
|
||||
|
||||
sig = inspect.signature(upload_and_create_job)
|
||||
param_names = list(sig.parameters.keys())
|
||||
|
||||
assert "dropbox_path" in param_names
|
||||
|
||||
def test_endpoint_has_gdrive_folder_id_parameter(self):
|
||||
"""Test that upload endpoint signature includes gdrive_folder_id parameter."""
|
||||
import inspect
|
||||
from backend.api.routes.file_upload import upload_and_create_job
|
||||
|
||||
sig = inspect.signature(upload_and_create_job)
|
||||
param_names = list(sig.parameters.keys())
|
||||
|
||||
assert "gdrive_folder_id" in param_names
|
||||
|
||||
def test_endpoint_dropbox_path_is_optional(self):
|
||||
"""Test that dropbox_path parameter has a default value (Form(None))."""
|
||||
import inspect
|
||||
from backend.api.routes.file_upload import upload_and_create_job
|
||||
|
||||
sig = inspect.signature(upload_and_create_job)
|
||||
param = sig.parameters.get("dropbox_path")
|
||||
|
||||
assert param is not None
|
||||
# Default is Form(None), so check that it's not required
|
||||
assert param.default is not inspect.Parameter.empty
|
||||
|
||||
def test_endpoint_gdrive_folder_id_is_optional(self):
|
||||
"""Test that gdrive_folder_id parameter has a default value (Form(None))."""
|
||||
import inspect
|
||||
from backend.api.routes.file_upload import upload_and_create_job
|
||||
|
||||
sig = inspect.signature(upload_and_create_job)
|
||||
param = sig.parameters.get("gdrive_folder_id")
|
||||
|
||||
assert param is not None
|
||||
# Default is Form(None), so check that it's not required
|
||||
assert param.default is not inspect.Parameter.empty
|
||||
|
||||
|
||||
class TestVideoWorkerDistribution:
|
||||
"""Tests for distribution handling in video worker."""
|
||||
|
||||
def test_handle_native_distribution_function_exists(self):
|
||||
"""Test that _handle_native_distribution function exists."""
|
||||
from backend.workers import video_worker
|
||||
assert hasattr(video_worker, '_handle_native_distribution')
|
||||
|
||||
def test_video_worker_imports_distribution_services(self):
|
||||
"""Test that video worker can import distribution services."""
|
||||
# The services should be importable (even if credentials aren't available)
|
||||
try:
|
||||
from backend.services.dropbox_service import DropboxService
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
assert True
|
||||
except ImportError as e:
|
||||
pytest.fail(f"Failed to import distribution services: {e}")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_native_distribution_skips_without_config(self):
|
||||
"""Test that _handle_native_distribution skips when not configured."""
|
||||
from backend.workers.video_worker import _handle_native_distribution
|
||||
|
||||
mock_job = MagicMock()
|
||||
mock_job.dropbox_path = None
|
||||
mock_job.gdrive_folder_id = None
|
||||
mock_job.brand_prefix = None
|
||||
mock_job.artist = "Test"
|
||||
mock_job.title = "Song"
|
||||
|
||||
mock_job_log = MagicMock()
|
||||
mock_job_manager = MagicMock()
|
||||
|
||||
# Should complete without error
|
||||
await _handle_native_distribution(
|
||||
job_id="test-123",
|
||||
job=mock_job,
|
||||
job_log=mock_job_log,
|
||||
job_manager=mock_job_manager,
|
||||
temp_dir="/tmp/test",
|
||||
result={},
|
||||
)
|
||||
|
||||
# No errors should have been logged
|
||||
mock_job_log.error.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_native_distribution_dropbox_not_configured(self):
|
||||
"""Test Dropbox upload skipped when service not configured."""
|
||||
from backend.workers.video_worker import _handle_native_distribution
|
||||
|
||||
mock_job = MagicMock()
|
||||
mock_job.dropbox_path = "/test/path"
|
||||
mock_job.brand_prefix = "TEST"
|
||||
mock_job.gdrive_folder_id = None
|
||||
mock_job.artist = "Test"
|
||||
mock_job.title = "Song"
|
||||
mock_job.state_data = {}
|
||||
|
||||
mock_job_log = MagicMock()
|
||||
mock_job_manager = MagicMock()
|
||||
|
||||
mock_dropbox = MagicMock()
|
||||
mock_dropbox.is_configured = False
|
||||
|
||||
with patch('backend.services.dropbox_service.get_dropbox_service', return_value=mock_dropbox):
|
||||
await _handle_native_distribution(
|
||||
job_id="test-123",
|
||||
job=mock_job,
|
||||
job_log=mock_job_log,
|
||||
job_manager=mock_job_manager,
|
||||
temp_dir="/tmp/test",
|
||||
result={},
|
||||
)
|
||||
|
||||
# Should log warning about not configured
|
||||
mock_job_log.warning.assert_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_native_distribution_gdrive_not_configured(self):
|
||||
"""Test Google Drive upload skipped when service not configured."""
|
||||
from backend.workers.video_worker import _handle_native_distribution
|
||||
|
||||
mock_job = MagicMock()
|
||||
mock_job.dropbox_path = None
|
||||
mock_job.brand_prefix = None
|
||||
mock_job.gdrive_folder_id = "test_folder_id"
|
||||
mock_job.artist = "Test"
|
||||
mock_job.title = "Song"
|
||||
mock_job.state_data = {}
|
||||
|
||||
mock_job_log = MagicMock()
|
||||
mock_job_manager = MagicMock()
|
||||
|
||||
mock_gdrive = MagicMock()
|
||||
mock_gdrive.is_configured = False
|
||||
|
||||
with patch('backend.services.gdrive_service.get_gdrive_service', return_value=mock_gdrive):
|
||||
await _handle_native_distribution(
|
||||
job_id="test-123",
|
||||
job=mock_job,
|
||||
job_log=mock_job_log,
|
||||
job_manager=mock_job_manager,
|
||||
temp_dir="/tmp/test",
|
||||
result={},
|
||||
)
|
||||
|
||||
# Should log warning about not configured
|
||||
mock_job_log.warning.assert_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_native_distribution_handles_import_error(self):
|
||||
"""Test that import errors for services are handled gracefully."""
|
||||
from backend.workers.video_worker import _handle_native_distribution
|
||||
|
||||
mock_job = MagicMock()
|
||||
mock_job.dropbox_path = "/test/path"
|
||||
mock_job.brand_prefix = "TEST"
|
||||
mock_job.gdrive_folder_id = None
|
||||
mock_job.artist = "Test"
|
||||
mock_job.title = "Song"
|
||||
mock_job.state_data = {}
|
||||
|
||||
mock_job_log = MagicMock()
|
||||
mock_job_manager = MagicMock()
|
||||
|
||||
# Simulate import error
|
||||
with patch('backend.services.dropbox_service.get_dropbox_service', side_effect=ImportError("No module")):
|
||||
await _handle_native_distribution(
|
||||
job_id="test-123",
|
||||
job=mock_job,
|
||||
job_log=mock_job_log,
|
||||
job_manager=mock_job_manager,
|
||||
temp_dir="/tmp/test",
|
||||
result={},
|
||||
)
|
||||
|
||||
# Should log warning about import error
|
||||
mock_job_log.warning.assert_called()
|
||||
@@ -0,0 +1,322 @@
|
||||
"""
|
||||
Integration tests using local GCP emulators.
|
||||
|
||||
These tests use real Firestore and GCS emulators running locally,
|
||||
providing true integration testing without cloud resources or costs.
|
||||
|
||||
Run with: ./scripts/run-emulator-tests.sh
|
||||
|
||||
NOTE: These are simplified integration tests that focus on the most critical
|
||||
paths. They mock background workers to avoid race conditions and network issues.
|
||||
"""
|
||||
import pytest
|
||||
import os
|
||||
import time
|
||||
from unittest.mock import AsyncMock, patch
|
||||
from datetime import datetime, UTC
|
||||
import requests
|
||||
|
||||
|
||||
def emulators_running() -> bool:
|
||||
"""Check if GCP emulators are running."""
|
||||
try:
|
||||
requests.get("http://127.0.0.1:8080", timeout=1)
|
||||
requests.get("http://127.0.0.1:4443", timeout=1)
|
||||
return True
|
||||
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
||||
return False
|
||||
|
||||
|
||||
# Skip all tests in this module if emulators aren't running
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not emulators_running(),
|
||||
reason="GCP emulators not running. Start with: scripts/start-emulators.sh"
|
||||
)
|
||||
|
||||
# Set emulator environment variables before any imports (use 127.0.0.1 for IPv4)
|
||||
os.environ["FIRESTORE_EMULATOR_HOST"] = "127.0.0.1:8080"
|
||||
os.environ["STORAGE_EMULATOR_HOST"] = "http://127.0.0.1:4443"
|
||||
os.environ["GOOGLE_CLOUD_PROJECT"] = "test-project"
|
||||
os.environ["GCS_BUCKET_NAME"] = "test-bucket"
|
||||
os.environ["FIRESTORE_COLLECTION"] = "test-jobs" # Use separate collection for tests
|
||||
os.environ["ENVIRONMENT"] = "test"
|
||||
os.environ["ADMIN_TOKENS"] = "test-admin-token"
|
||||
|
||||
# Only import if emulators are running
|
||||
if emulators_running():
|
||||
from fastapi.testclient import TestClient
|
||||
from backend.main import app
|
||||
from backend.models.job import JobStatus
|
||||
else:
|
||||
TestClient = None
|
||||
app = None
|
||||
JobStatus = None
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def setup_gcs_bucket():
|
||||
"""Create GCS bucket in emulator before tests."""
|
||||
try:
|
||||
response = requests.post(
|
||||
"http://localhost:4443/storage/v1/b",
|
||||
json={"name": "test-bucket"},
|
||||
params={"project": "test-project"}
|
||||
)
|
||||
if response.status_code in [200, 409]:
|
||||
print(f"✅ GCS bucket 'test-bucket' ready")
|
||||
except Exception as e:
|
||||
print(f"⚠️ GCS bucket setup failed: {e}")
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def mock_worker_service():
|
||||
"""Mock the worker service to prevent background tasks."""
|
||||
with patch("backend.api.routes.jobs.worker_service") as mock:
|
||||
mock.trigger_audio_worker = AsyncMock(return_value=True)
|
||||
mock.trigger_lyrics_worker = AsyncMock(return_value=True)
|
||||
mock.trigger_screens_worker = AsyncMock(return_value=True)
|
||||
mock.trigger_video_worker = AsyncMock(return_value=True)
|
||||
yield mock
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def client(mock_worker_service):
|
||||
"""Create FastAPI test client with mocked workers."""
|
||||
with patch("backend.api.routes.file_upload.worker_service", mock_worker_service):
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def auth_headers():
|
||||
"""Auth headers for testing."""
|
||||
return {"Authorization": "Bearer test-admin-token"}
|
||||
|
||||
|
||||
class TestEmulatorBasics:
|
||||
"""Basic emulator connectivity tests."""
|
||||
|
||||
def test_health_endpoint(self, client):
|
||||
"""Test health endpoint works."""
|
||||
response = client.get("/api/health")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "healthy"
|
||||
|
||||
def test_root_endpoint(self, client):
|
||||
"""Test root endpoint works."""
|
||||
response = client.get("/")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["service"] == "karaoke-gen-backend"
|
||||
|
||||
|
||||
class TestJobCreation:
|
||||
"""Test job creation with Firestore emulator."""
|
||||
|
||||
def test_create_job_simple(self, client, auth_headers):
|
||||
"""Test creating a simple job."""
|
||||
response = client.post(
|
||||
"/api/jobs",
|
||||
headers=auth_headers,
|
||||
json={"url": "https://youtube.com/watch?v=test123"}
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "success"
|
||||
assert "job_id" in data
|
||||
|
||||
def test_create_job_with_metadata(self, client, auth_headers):
|
||||
"""Test creating a job with artist/title."""
|
||||
response = client.post(
|
||||
"/api/jobs",
|
||||
headers=auth_headers,
|
||||
json={
|
||||
"url": "https://youtube.com/watch?v=test",
|
||||
"artist": "Test Artist",
|
||||
"title": "Test Song"
|
||||
}
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "job_id" in data
|
||||
|
||||
|
||||
class TestJobRetrieval:
|
||||
"""Test job retrieval from Firestore emulator."""
|
||||
|
||||
def test_create_and_get_job(self, client, auth_headers):
|
||||
"""Test creating and then retrieving a job."""
|
||||
# Create
|
||||
create_resp = client.post(
|
||||
"/api/jobs",
|
||||
headers=auth_headers,
|
||||
json={
|
||||
"url": "https://youtube.com/watch?v=abc123",
|
||||
"artist": "Test Artist",
|
||||
"title": "Test Song"
|
||||
}
|
||||
)
|
||||
assert create_resp.status_code == 200
|
||||
job_id = create_resp.json()["job_id"]
|
||||
|
||||
# Small delay for emulator consistency
|
||||
time.sleep(0.2)
|
||||
|
||||
# Retrieve
|
||||
get_resp = client.get(f"/api/jobs/{job_id}", headers=auth_headers)
|
||||
|
||||
if get_resp.status_code != 200:
|
||||
print(f"GET failed: {get_resp.status_code} - {get_resp.text}")
|
||||
|
||||
assert get_resp.status_code == 200
|
||||
|
||||
job = get_resp.json()
|
||||
assert job["job_id"] == job_id
|
||||
assert job["status"] == "pending"
|
||||
assert job["artist"] == "Test Artist"
|
||||
assert job["title"] == "Test Song"
|
||||
|
||||
def test_get_nonexistent_job(self, client, auth_headers):
|
||||
"""Test fetching a job that doesn't exist."""
|
||||
response = client.get("/api/jobs/nonexistent-id", headers=auth_headers)
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
class TestJobList:
|
||||
"""Test listing jobs from Firestore."""
|
||||
|
||||
def test_list_jobs(self, client, auth_headers):
|
||||
"""Test listing jobs."""
|
||||
# Create a few jobs
|
||||
for i in range(3):
|
||||
client.post(
|
||||
"/api/jobs",
|
||||
headers=auth_headers,
|
||||
json={"url": f"https://youtube.com/watch?v=list{i}"}
|
||||
)
|
||||
|
||||
time.sleep(0.2)
|
||||
|
||||
# List
|
||||
response = client.get("/api/jobs", headers=auth_headers)
|
||||
assert response.status_code == 200
|
||||
|
||||
jobs = response.json()
|
||||
assert isinstance(jobs, list)
|
||||
# We should have at least the 3 we just created
|
||||
assert len(jobs) >= 3
|
||||
|
||||
|
||||
class TestJobDeletion:
|
||||
"""Test job deletion."""
|
||||
|
||||
def test_delete_job(self, client, auth_headers):
|
||||
"""Test deleting a job."""
|
||||
# Create
|
||||
create_resp = client.post(
|
||||
"/api/jobs",
|
||||
headers=auth_headers,
|
||||
json={"url": "https://youtube.com/watch?v=delete-me"}
|
||||
)
|
||||
job_id = create_resp.json()["job_id"]
|
||||
|
||||
time.sleep(0.2)
|
||||
|
||||
# Delete
|
||||
del_resp = client.delete(f"/api/jobs/{job_id}", headers=auth_headers)
|
||||
assert del_resp.status_code == 200
|
||||
|
||||
# Verify deleted
|
||||
get_resp = client.get(f"/api/jobs/{job_id}", headers=auth_headers)
|
||||
assert get_resp.status_code == 404
|
||||
|
||||
|
||||
class TestJobUpdates:
|
||||
"""Test job status updates."""
|
||||
|
||||
def test_cancel_job(self, client, auth_headers):
|
||||
"""Test cancelling a job."""
|
||||
# Create
|
||||
create_resp = client.post(
|
||||
"/api/jobs",
|
||||
headers=auth_headers,
|
||||
json={"url": "https://youtube.com/watch?v=cancel-me"}
|
||||
)
|
||||
job_id = create_resp.json()["job_id"]
|
||||
|
||||
time.sleep(0.2)
|
||||
|
||||
# Cancel
|
||||
cancel_resp = client.post(
|
||||
f"/api/jobs/{job_id}/cancel",
|
||||
headers=auth_headers,
|
||||
json={"reason": "test cancellation"}
|
||||
)
|
||||
assert cancel_resp.status_code == 200
|
||||
|
||||
# Verify cancelled
|
||||
get_resp = client.get(f"/api/jobs/{job_id}", headers=auth_headers)
|
||||
assert get_resp.status_code == 200
|
||||
job = get_resp.json()
|
||||
assert job["status"] == "cancelled"
|
||||
|
||||
|
||||
class TestFileUpload:
|
||||
"""Test file upload with GCS emulator."""
|
||||
|
||||
def test_upload_file(self, client, auth_headers):
|
||||
"""Test uploading a file."""
|
||||
test_file_content = b"fake audio data for testing"
|
||||
|
||||
response = client.post(
|
||||
"/api/jobs/upload",
|
||||
headers={"Authorization": auth_headers["Authorization"]},
|
||||
files={"file": ("test.flac", test_file_content, "audio/flac")},
|
||||
data={"artist": "Upload Artist", "title": "Upload Song"}
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "success"
|
||||
assert "job_id" in data
|
||||
|
||||
job_id = data["job_id"]
|
||||
|
||||
time.sleep(0.2)
|
||||
|
||||
# Verify job created with upload data
|
||||
get_resp = client.get(f"/api/jobs/{job_id}", headers=auth_headers)
|
||||
assert get_resp.status_code == 200
|
||||
job = get_resp.json()
|
||||
assert job["artist"] == "Upload Artist"
|
||||
assert job["title"] == "Upload Song"
|
||||
assert "input_media_gcs_path" in job
|
||||
|
||||
|
||||
class TestInternalEndpoints:
|
||||
"""Test internal worker endpoints."""
|
||||
|
||||
def test_internal_workers_exist(self, client, auth_headers):
|
||||
"""Test that internal worker endpoints exist and respond."""
|
||||
# Create a job first
|
||||
create_resp = client.post(
|
||||
"/api/jobs",
|
||||
headers=auth_headers,
|
||||
json={"url": "https://youtube.com/watch?v=worker-test"}
|
||||
)
|
||||
job_id = create_resp.json()["job_id"]
|
||||
|
||||
time.sleep(0.2)
|
||||
|
||||
# Test audio worker endpoint exists
|
||||
response = client.post(
|
||||
"/api/internal/workers/audio",
|
||||
headers=auth_headers,
|
||||
json={"job_id": job_id}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "started"
|
||||
|
||||
|
||||
print("✅ Emulator integration tests ready to run")
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,343 @@
|
||||
"""
|
||||
Unit tests for internal.py API routes using FastAPI TestClient.
|
||||
|
||||
Internal routes are used for worker communication within Cloud Run.
|
||||
These endpoints require admin authentication.
|
||||
|
||||
With Cloud Tasks integration, these endpoints include idempotency checks
|
||||
to prevent duplicate processing on retries.
|
||||
"""
|
||||
import os
|
||||
import pytest
|
||||
from datetime import datetime, UTC
|
||||
from unittest.mock import MagicMock, AsyncMock, patch
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from backend.models.job import Job, JobStatus
|
||||
|
||||
|
||||
# Set test admin token for auth
|
||||
os.environ.setdefault('ADMIN_TOKENS', 'test-admin-token')
|
||||
|
||||
|
||||
def _create_mock_job(job_id: str = "test123") -> Job:
|
||||
"""Create a mock Job object for testing."""
|
||||
return Job(
|
||||
job_id=job_id,
|
||||
status=JobStatus.PROCESSING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={}, # Empty state_data - no worker progress yet
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
"""Create TestClient with mocked workers and job manager."""
|
||||
mock_creds = MagicMock()
|
||||
mock_creds.universe_domain = 'googleapis.com'
|
||||
|
||||
# Mock the worker functions at the internal routes module level
|
||||
mock_audio = AsyncMock()
|
||||
mock_lyrics = AsyncMock()
|
||||
mock_screens = AsyncMock()
|
||||
mock_video = AsyncMock()
|
||||
mock_render = AsyncMock()
|
||||
|
||||
# Mock JobManager for idempotency checks
|
||||
mock_job_manager = MagicMock()
|
||||
mock_job_manager.get_job.return_value = _create_mock_job()
|
||||
mock_job_manager.update_state_data.return_value = None
|
||||
|
||||
with patch('backend.api.routes.internal.process_audio_separation', mock_audio), \
|
||||
patch('backend.api.routes.internal.process_lyrics_transcription', mock_lyrics), \
|
||||
patch('backend.api.routes.internal.generate_screens', mock_screens), \
|
||||
patch('backend.api.routes.internal.generate_video', mock_video), \
|
||||
patch('backend.api.routes.internal.process_render_video', mock_render), \
|
||||
patch('backend.api.routes.internal.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.services.firestore_service.firestore'), \
|
||||
patch('backend.services.storage_service.storage'), \
|
||||
patch('google.auth.default', return_value=(mock_creds, 'test-project')):
|
||||
from backend.main import app
|
||||
yield TestClient(app)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def auth_headers():
|
||||
"""Auth headers for internal API testing."""
|
||||
return {"Authorization": "Bearer test-admin-token"}
|
||||
|
||||
|
||||
class TestAudioWorkerEndpoint:
|
||||
"""Tests for POST /api/internal/workers/audio."""
|
||||
|
||||
def test_audio_worker_returns_200(self, client, auth_headers):
|
||||
"""Test audio worker endpoint returns 200 with auth."""
|
||||
response = client.post(
|
||||
"/api/internal/workers/audio",
|
||||
headers=auth_headers,
|
||||
json={"job_id": "test123"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_audio_worker_accepts_job_id(self, client, auth_headers):
|
||||
"""Test audio worker accepts job_id in body."""
|
||||
response = client.post(
|
||||
"/api/internal/workers/audio",
|
||||
headers=auth_headers,
|
||||
json={"job_id": "any-job-id"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_audio_worker_requires_job_id(self, client, auth_headers):
|
||||
"""Test audio worker requires job_id."""
|
||||
response = client.post(
|
||||
"/api/internal/workers/audio",
|
||||
headers=auth_headers,
|
||||
json={}
|
||||
)
|
||||
assert response.status_code == 422
|
||||
|
||||
def test_audio_worker_requires_auth(self, client):
|
||||
"""Test audio worker requires authentication."""
|
||||
response = client.post(
|
||||
"/api/internal/workers/audio",
|
||||
json={"job_id": "test123"}
|
||||
)
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
class TestLyricsWorkerEndpoint:
|
||||
"""Tests for POST /api/internal/workers/lyrics."""
|
||||
|
||||
def test_lyrics_worker_returns_200(self, client, auth_headers):
|
||||
"""Test lyrics worker endpoint returns 200 with auth."""
|
||||
response = client.post(
|
||||
"/api/internal/workers/lyrics",
|
||||
headers=auth_headers,
|
||||
json={"job_id": "test123"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_lyrics_worker_requires_job_id(self, client, auth_headers):
|
||||
"""Test lyrics worker requires job_id."""
|
||||
response = client.post(
|
||||
"/api/internal/workers/lyrics",
|
||||
headers=auth_headers,
|
||||
json={}
|
||||
)
|
||||
assert response.status_code == 422
|
||||
|
||||
def test_lyrics_worker_requires_auth(self, client):
|
||||
"""Test lyrics worker requires authentication."""
|
||||
response = client.post(
|
||||
"/api/internal/workers/lyrics",
|
||||
json={"job_id": "test123"}
|
||||
)
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
class TestScreensWorkerEndpoint:
|
||||
"""Tests for POST /api/internal/workers/screens."""
|
||||
|
||||
def test_screens_worker_returns_200(self, client, auth_headers):
|
||||
"""Test screens worker endpoint returns 200 with auth."""
|
||||
response = client.post(
|
||||
"/api/internal/workers/screens",
|
||||
headers=auth_headers,
|
||||
json={"job_id": "test123"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_screens_worker_requires_job_id(self, client, auth_headers):
|
||||
"""Test screens worker requires job_id."""
|
||||
response = client.post(
|
||||
"/api/internal/workers/screens",
|
||||
headers=auth_headers,
|
||||
json={}
|
||||
)
|
||||
assert response.status_code == 422
|
||||
|
||||
def test_screens_worker_requires_auth(self, client):
|
||||
"""Test screens worker requires authentication."""
|
||||
response = client.post(
|
||||
"/api/internal/workers/screens",
|
||||
json={"job_id": "test123"}
|
||||
)
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
class TestVideoWorkerEndpoint:
|
||||
"""Tests for POST /api/internal/workers/video."""
|
||||
|
||||
def test_video_worker_returns_200(self, client, auth_headers):
|
||||
"""Test video worker endpoint returns 200 with auth."""
|
||||
response = client.post(
|
||||
"/api/internal/workers/video",
|
||||
headers=auth_headers,
|
||||
json={"job_id": "test123"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_video_worker_requires_job_id(self, client, auth_headers):
|
||||
"""Test video worker requires job_id."""
|
||||
response = client.post(
|
||||
"/api/internal/workers/video",
|
||||
headers=auth_headers,
|
||||
json={}
|
||||
)
|
||||
assert response.status_code == 422
|
||||
|
||||
def test_video_worker_requires_auth(self, client):
|
||||
"""Test video worker requires authentication."""
|
||||
response = client.post(
|
||||
"/api/internal/workers/video",
|
||||
json={"job_id": "test123"}
|
||||
)
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
class TestRenderVideoWorkerEndpoint:
|
||||
"""Tests for POST /api/internal/workers/render-video."""
|
||||
|
||||
def test_render_video_worker_returns_200(self, client, auth_headers):
|
||||
"""Test render-video worker endpoint returns 200 with auth."""
|
||||
response = client.post(
|
||||
"/api/internal/workers/render-video",
|
||||
headers=auth_headers,
|
||||
json={"job_id": "test123"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_render_video_worker_requires_job_id(self, client, auth_headers):
|
||||
"""Test render-video worker requires job_id."""
|
||||
response = client.post(
|
||||
"/api/internal/workers/render-video",
|
||||
headers=auth_headers,
|
||||
json={}
|
||||
)
|
||||
assert response.status_code == 422
|
||||
|
||||
def test_render_video_worker_requires_auth(self, client):
|
||||
"""Test render-video worker requires authentication."""
|
||||
response = client.post(
|
||||
"/api/internal/workers/render-video",
|
||||
json={"job_id": "test123"}
|
||||
)
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
class TestWorkerResponseFormat:
|
||||
"""Tests for worker response format."""
|
||||
|
||||
def test_audio_worker_response_contains_status(self, client, auth_headers):
|
||||
"""Test audio worker response contains status."""
|
||||
response = client.post(
|
||||
"/api/internal/workers/audio",
|
||||
headers=auth_headers,
|
||||
json={"job_id": "test123"}
|
||||
)
|
||||
data = response.json()
|
||||
assert "status" in data or "message" in data
|
||||
|
||||
def test_lyrics_worker_response_contains_status(self, client, auth_headers):
|
||||
"""Test lyrics worker response contains status."""
|
||||
response = client.post(
|
||||
"/api/internal/workers/lyrics",
|
||||
headers=auth_headers,
|
||||
json={"job_id": "test123"}
|
||||
)
|
||||
data = response.json()
|
||||
assert "status" in data or "message" in data
|
||||
|
||||
|
||||
class TestWorkerIdempotency:
|
||||
"""Tests for worker idempotency checks (Cloud Tasks retry support)."""
|
||||
|
||||
def test_audio_worker_returns_already_running_when_in_progress(self, auth_headers):
|
||||
"""Test audio worker returns already_running if worker is already running."""
|
||||
mock_creds = MagicMock()
|
||||
mock_creds.universe_domain = 'googleapis.com'
|
||||
|
||||
# Create job with audio already running
|
||||
mock_job = _create_mock_job()
|
||||
mock_job.state_data = {'audio_progress': {'stage': 'running'}}
|
||||
|
||||
mock_job_manager = MagicMock()
|
||||
mock_job_manager.get_job.return_value = mock_job
|
||||
|
||||
with patch('backend.api.routes.internal.process_audio_separation', AsyncMock()), \
|
||||
patch('backend.api.routes.internal.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.services.firestore_service.firestore'), \
|
||||
patch('backend.services.storage_service.storage'), \
|
||||
patch('google.auth.default', return_value=(mock_creds, 'test-project')):
|
||||
from backend.main import app
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.post(
|
||||
"/api/internal/workers/audio",
|
||||
headers=auth_headers,
|
||||
json={"job_id": "test123"}
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "already_running"
|
||||
|
||||
def test_audio_worker_returns_already_complete_when_finished(self, auth_headers):
|
||||
"""Test audio worker returns already_complete if worker already finished."""
|
||||
mock_creds = MagicMock()
|
||||
mock_creds.universe_domain = 'googleapis.com'
|
||||
|
||||
# Create job with audio already complete
|
||||
mock_job = _create_mock_job()
|
||||
mock_job.state_data = {'audio_progress': {'stage': 'complete'}}
|
||||
|
||||
mock_job_manager = MagicMock()
|
||||
mock_job_manager.get_job.return_value = mock_job
|
||||
|
||||
with patch('backend.api.routes.internal.process_audio_separation', AsyncMock()), \
|
||||
patch('backend.api.routes.internal.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.services.firestore_service.firestore'), \
|
||||
patch('backend.services.storage_service.storage'), \
|
||||
patch('google.auth.default', return_value=(mock_creds, 'test-project')):
|
||||
from backend.main import app
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.post(
|
||||
"/api/internal/workers/audio",
|
||||
headers=auth_headers,
|
||||
json={"job_id": "test123"}
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "already_complete"
|
||||
|
||||
def test_audio_worker_returns_not_found_when_job_missing(self, auth_headers):
|
||||
"""Test audio worker returns not_found if job doesn't exist."""
|
||||
mock_creds = MagicMock()
|
||||
mock_creds.universe_domain = 'googleapis.com'
|
||||
|
||||
# Return None for job not found
|
||||
mock_job_manager = MagicMock()
|
||||
mock_job_manager.get_job.return_value = None
|
||||
|
||||
with patch('backend.api.routes.internal.process_audio_separation', AsyncMock()), \
|
||||
patch('backend.api.routes.internal.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.services.firestore_service.firestore'), \
|
||||
patch('backend.services.storage_service.storage'), \
|
||||
patch('google.auth.default', return_value=(mock_creds, 'test-project')):
|
||||
from backend.main import app
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.post(
|
||||
"/api/internal/workers/audio",
|
||||
headers=auth_headers,
|
||||
json={"job_id": "nonexistent"}
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "not_found"
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
"""
|
||||
Unit tests for Job Manager service.
|
||||
|
||||
Tests the job lifecycle management without requiring actual Firestore connection.
|
||||
Uses mocking to isolate the business logic.
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import Mock, MagicMock, patch
|
||||
from datetime import datetime, UTC
|
||||
|
||||
# Mock Firestore before importing JobManager
|
||||
import sys
|
||||
sys.modules['google.cloud.firestore'] = MagicMock()
|
||||
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.models.job import Job, JobCreate, JobStatus
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_firestore_service():
|
||||
"""Mock FirestoreService."""
|
||||
with patch('backend.services.job_manager.FirestoreService') as mock:
|
||||
service = Mock()
|
||||
mock.return_value = service
|
||||
yield service
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def job_manager(mock_firestore_service):
|
||||
"""Create JobManager with mocked dependencies."""
|
||||
return JobManager()
|
||||
|
||||
|
||||
class TestJobCreation:
|
||||
"""Test job creation logic."""
|
||||
|
||||
def test_create_job_with_url(self, job_manager, mock_firestore_service):
|
||||
"""Test creating a job with YouTube URL."""
|
||||
job_create = JobCreate(
|
||||
url="https://youtube.com/watch?v=test",
|
||||
artist="Test Artist",
|
||||
title="Test Song"
|
||||
)
|
||||
|
||||
# The actual create_job method creates the job and returns it
|
||||
job = job_manager.create_job(job_create)
|
||||
|
||||
assert job.job_id is not None
|
||||
assert job.status == JobStatus.PENDING
|
||||
assert job.url == "https://youtube.com/watch?v=test"
|
||||
assert job.artist == "Test Artist"
|
||||
assert job.title == "Test Song"
|
||||
assert job.progress == 0
|
||||
|
||||
# Verify Firestore was called
|
||||
mock_firestore_service.create_job.assert_called_once()
|
||||
|
||||
def test_create_job_without_url(self, job_manager, mock_firestore_service):
|
||||
"""Test creating a job without URL (for file upload)."""
|
||||
job_create = JobCreate(
|
||||
artist="Test Artist",
|
||||
title="Test Song"
|
||||
)
|
||||
|
||||
job = job_manager.create_job(job_create)
|
||||
|
||||
assert job.artist == "Test Artist"
|
||||
assert job.title == "Test Song"
|
||||
assert job.url is None
|
||||
|
||||
def test_create_job_generates_unique_id(self, job_manager, mock_firestore_service):
|
||||
"""Test that each job gets a unique ID."""
|
||||
job_create = JobCreate()
|
||||
|
||||
# Create multiple jobs
|
||||
ids = []
|
||||
for i in range(5):
|
||||
mock_firestore_service.create_job.return_value = Job(
|
||||
job_id=f"test{i}",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
job = job_manager.create_job(job_create)
|
||||
ids.append(job.job_id)
|
||||
|
||||
# All IDs should be unique
|
||||
assert len(ids) == len(set(ids))
|
||||
|
||||
def test_create_job_sets_initial_status(self, job_manager, mock_firestore_service):
|
||||
"""Test that new jobs start with PENDING status."""
|
||||
job_create = JobCreate()
|
||||
|
||||
mock_firestore_service.create_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
job = job_manager.create_job(job_create)
|
||||
|
||||
assert job.status == JobStatus.PENDING
|
||||
assert job.progress == 0
|
||||
|
||||
|
||||
class TestJobRetrieval:
|
||||
"""Test job retrieval logic."""
|
||||
|
||||
def test_get_existing_job(self, job_manager, mock_firestore_service):
|
||||
"""Test retrieving an existing job."""
|
||||
expected_job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.SEPARATING_STAGE1,
|
||||
progress=25,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
mock_firestore_service.get_job.return_value = expected_job
|
||||
|
||||
job = job_manager.get_job("test123")
|
||||
|
||||
assert job.job_id == "test123"
|
||||
assert job.status == JobStatus.SEPARATING_STAGE1
|
||||
assert job.progress == 25
|
||||
|
||||
mock_firestore_service.get_job.assert_called_once_with("test123")
|
||||
|
||||
def test_get_nonexistent_job(self, job_manager, mock_firestore_service):
|
||||
"""Test retrieving a nonexistent job returns None."""
|
||||
mock_firestore_service.get_job.return_value = None
|
||||
|
||||
job = job_manager.get_job("nonexistent")
|
||||
|
||||
assert job is None
|
||||
|
||||
|
||||
class TestJobUpdate:
|
||||
"""Test job update logic."""
|
||||
|
||||
def test_update_job_status(self, job_manager, mock_firestore_service):
|
||||
"""Test updating job status."""
|
||||
# Setup: job exists
|
||||
existing_job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
progress=0,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
mock_firestore_service.get_job.return_value = existing_job
|
||||
|
||||
# Update status
|
||||
updates = {"status": JobStatus.SEPARATING_STAGE1, "progress": 25}
|
||||
job_manager.update_job("test123", updates)
|
||||
|
||||
# Verify update was called
|
||||
mock_firestore_service.update_job.assert_called_once()
|
||||
call_args = mock_firestore_service.update_job.call_args
|
||||
assert call_args[0][0] == "test123" # job_id
|
||||
assert "status" in call_args[0][1] # updates dict
|
||||
|
||||
def test_update_job_multiple_fields(self, job_manager, mock_firestore_service):
|
||||
"""Test updating multiple fields at once."""
|
||||
existing_job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
progress=0,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
timeline=[]
|
||||
)
|
||||
mock_firestore_service.get_job.return_value = existing_job
|
||||
|
||||
updates = {"status": JobStatus.SEPARATING_STAGE1, "progress": 25}
|
||||
job_manager.update_job("test123", updates)
|
||||
|
||||
# Verify update was called
|
||||
call_args = mock_firestore_service.update_job.call_args
|
||||
updates_dict = call_args[0][1]
|
||||
assert "status" in updates_dict
|
||||
assert "progress" in updates_dict
|
||||
|
||||
def test_update_input_media_gcs_path(self, job_manager, mock_firestore_service):
|
||||
"""Test updating input_media_gcs_path field."""
|
||||
existing_job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
progress=0,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
mock_firestore_service.get_job.return_value = existing_job
|
||||
|
||||
updates = {"input_media_gcs_path": "uploads/test123/file.flac"}
|
||||
job_manager.update_job("test123", updates)
|
||||
|
||||
# Verify update was called with input_media_gcs_path
|
||||
call_args = mock_firestore_service.update_job.call_args
|
||||
updates_dict = call_args[0][1]
|
||||
assert "input_media_gcs_path" in updates_dict
|
||||
assert updates_dict["input_media_gcs_path"] == "uploads/test123/file.flac"
|
||||
|
||||
|
||||
class TestJobStatusTransitions:
|
||||
"""Test valid job status transitions."""
|
||||
|
||||
def test_pending_to_downloading(self, job_manager, mock_firestore_service):
|
||||
"""Test transition from PENDING to DOWNLOADING."""
|
||||
existing_job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
progress=0,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
mock_firestore_service.get_job.return_value = existing_job
|
||||
|
||||
job_manager.update_job("test123", {"status": JobStatus.DOWNLOADING})
|
||||
|
||||
# Should succeed (valid transition)
|
||||
mock_firestore_service.update_job.assert_called_once()
|
||||
|
||||
def test_downloading_to_separating(self, job_manager, mock_firestore_service):
|
||||
"""Test transition from DOWNLOADING to SEPARATING_STAGE1."""
|
||||
existing_job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.DOWNLOADING,
|
||||
progress=0,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
mock_firestore_service.get_job.return_value = existing_job
|
||||
|
||||
job_manager.update_job("test123", {"status": JobStatus.SEPARATING_STAGE1})
|
||||
|
||||
mock_firestore_service.update_job.assert_called_once()
|
||||
|
||||
|
||||
class TestJobFailure:
|
||||
"""Test job failure handling."""
|
||||
|
||||
def test_mark_job_as_failed(self, job_manager, mock_firestore_service):
|
||||
"""Test marking a job as failed."""
|
||||
error_message = "Audio separation failed"
|
||||
job_manager.mark_job_failed("test123", error_message)
|
||||
|
||||
# mark_job_failed calls firestore.update_job_status
|
||||
mock_firestore_service.update_job_status.assert_called_once()
|
||||
call_args = mock_firestore_service.update_job_status.call_args
|
||||
assert call_args[1]['job_id'] == "test123"
|
||||
assert call_args[1]['status'] == JobStatus.FAILED
|
||||
assert 'error_message' in call_args[1]
|
||||
|
||||
def test_mark_job_error(self, job_manager, mock_firestore_service):
|
||||
"""Test marking a job with an error."""
|
||||
error_message = "Test error"
|
||||
job_manager.mark_job_error("test123", error_message)
|
||||
|
||||
# mark_job_error calls firestore.update_job_status
|
||||
mock_firestore_service.update_job_status.assert_called_once()
|
||||
call_args = mock_firestore_service.update_job_status.call_args
|
||||
assert call_args[1]['job_id'] == "test123"
|
||||
assert call_args[1]['error_message'] == error_message
|
||||
|
||||
|
||||
class TestJobDeletion:
|
||||
"""Test job deletion logic."""
|
||||
|
||||
def test_delete_job(self, job_manager, mock_firestore_service):
|
||||
"""Test deleting a job."""
|
||||
existing_job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.COMPLETE,
|
||||
progress=100,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
output_files={} # Empty dict so iteration works
|
||||
)
|
||||
mock_firestore_service.get_job.return_value = existing_job
|
||||
|
||||
job_manager.delete_job("test123")
|
||||
|
||||
# Verify delete was called
|
||||
mock_firestore_service.delete_job.assert_called_once_with("test123")
|
||||
|
||||
def test_delete_job_with_files(self, job_manager, mock_firestore_service):
|
||||
"""Test deleting a job and its files."""
|
||||
existing_job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.COMPLETE,
|
||||
progress=100,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
output_files={} # Empty dict so iteration works
|
||||
)
|
||||
mock_firestore_service.get_job.return_value = existing_job
|
||||
|
||||
job_manager.delete_job("test123", delete_files=True)
|
||||
|
||||
# Verify delete was called
|
||||
mock_firestore_service.delete_job.assert_called_once_with("test123")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
"""
|
||||
Unit tests for jobs.py API routes using FastAPI TestClient.
|
||||
|
||||
These tests mock the underlying services and test the route logic directly.
|
||||
"""
|
||||
import pytest
|
||||
from datetime import datetime, UTC
|
||||
from unittest.mock import MagicMock, AsyncMock, patch
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from backend.models.job import Job, JobStatus
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_job():
|
||||
"""Create a standard mock job for testing."""
|
||||
return Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test Artist",
|
||||
title="Test Song"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_job_manager(mock_job):
|
||||
"""Create a mock JobManager with common methods."""
|
||||
manager = MagicMock()
|
||||
manager.get_job.return_value = mock_job
|
||||
manager.list_jobs.return_value = [mock_job]
|
||||
manager.create_job.return_value = mock_job
|
||||
manager.delete_job.return_value = True
|
||||
manager.cancel_job.return_value = mock_job
|
||||
manager.update_job_status.return_value = None
|
||||
manager.update_state_data.return_value = None
|
||||
return manager
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_worker_service():
|
||||
"""Create a mock WorkerService."""
|
||||
service = MagicMock()
|
||||
service.trigger_audio_worker = AsyncMock(return_value=True)
|
||||
service.trigger_lyrics_worker = AsyncMock(return_value=True)
|
||||
service.trigger_screens_worker = AsyncMock(return_value=True)
|
||||
service.trigger_video_worker = AsyncMock(return_value=True)
|
||||
return service
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(mock_job_manager, mock_worker_service):
|
||||
"""Create TestClient with mocked dependencies."""
|
||||
mock_creds = MagicMock()
|
||||
mock_creds.universe_domain = 'googleapis.com'
|
||||
|
||||
# Patch at the module level where jobs.py imports them
|
||||
with patch('backend.api.routes.jobs.job_manager', mock_job_manager), \
|
||||
patch('backend.api.routes.jobs.worker_service', mock_worker_service), \
|
||||
patch('backend.services.firestore_service.firestore'), \
|
||||
patch('backend.services.storage_service.storage'), \
|
||||
patch('google.auth.default', return_value=(mock_creds, 'test-project')):
|
||||
from backend.main import app
|
||||
yield TestClient(app)
|
||||
|
||||
|
||||
class TestGetJob:
|
||||
"""Tests for GET /api/jobs/{job_id}."""
|
||||
|
||||
def test_get_job_returns_200(self, client, mock_job_manager, mock_job):
|
||||
"""Test getting an existing job returns 200."""
|
||||
response = client.get("/api/jobs/test123")
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_get_job_returns_job_data(self, client, mock_job_manager, mock_job):
|
||||
"""Test response contains job data."""
|
||||
response = client.get("/api/jobs/test123")
|
||||
data = response.json()
|
||||
assert data["job_id"] == "test123"
|
||||
assert data["status"] == "pending"
|
||||
assert data["artist"] == "Test Artist"
|
||||
|
||||
def test_get_nonexistent_job_returns_404(self, mock_worker_service):
|
||||
"""Test getting non-existent job returns 404."""
|
||||
mock_job_manager = MagicMock()
|
||||
mock_job_manager.get_job.return_value = None
|
||||
mock_creds = MagicMock()
|
||||
mock_creds.universe_domain = 'googleapis.com'
|
||||
|
||||
with patch('backend.api.routes.jobs.job_manager', mock_job_manager), \
|
||||
patch('backend.api.routes.jobs.worker_service', mock_worker_service), \
|
||||
patch('backend.services.firestore_service.firestore'), \
|
||||
patch('backend.services.storage_service.storage'), \
|
||||
patch('google.auth.default', return_value=(mock_creds, 'test-project')):
|
||||
from backend.main import app
|
||||
client = TestClient(app)
|
||||
response = client.get("/api/jobs/nonexistent")
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
class TestListJobs:
|
||||
"""Tests for GET /api/jobs."""
|
||||
|
||||
def test_list_jobs_returns_200(self, client):
|
||||
"""Test listing jobs returns 200."""
|
||||
response = client.get("/api/jobs")
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_list_jobs_returns_array(self, client, mock_job_manager):
|
||||
"""Test response is an array of jobs."""
|
||||
response = client.get("/api/jobs")
|
||||
data = response.json()
|
||||
assert isinstance(data, list)
|
||||
assert len(data) == 1
|
||||
assert data[0]["job_id"] == "test123"
|
||||
|
||||
def test_list_jobs_with_status_filter(self, client, mock_job_manager):
|
||||
"""Test listing jobs with status filter."""
|
||||
response = client.get("/api/jobs?status=pending")
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_list_jobs_with_limit(self, client, mock_job_manager):
|
||||
"""Test listing jobs with limit."""
|
||||
response = client.get("/api/jobs?limit=10")
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
class TestCreateJob:
|
||||
"""Tests for POST /api/jobs."""
|
||||
|
||||
def test_create_job_with_url_returns_200(self, client, mock_job_manager):
|
||||
"""Test creating job with URL returns 200."""
|
||||
response = client.post(
|
||||
"/api/jobs",
|
||||
json={"url": "https://youtube.com/watch?v=test123"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_create_job_returns_job_id(self, client, mock_job_manager):
|
||||
"""Test create response contains job_id."""
|
||||
response = client.post(
|
||||
"/api/jobs",
|
||||
json={"url": "https://youtube.com/watch?v=test"}
|
||||
)
|
||||
data = response.json()
|
||||
assert "job_id" in data
|
||||
|
||||
def test_create_job_with_artist_title(self, client, mock_job_manager):
|
||||
"""Test creating job with artist and title."""
|
||||
response = client.post(
|
||||
"/api/jobs",
|
||||
json={
|
||||
"url": "https://youtube.com/watch?v=test",
|
||||
"artist": "Test Artist",
|
||||
"title": "Test Song"
|
||||
}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
class TestDeleteJob:
|
||||
"""Tests for DELETE /api/jobs/{job_id}."""
|
||||
|
||||
def test_delete_job_returns_200(self, client, mock_job_manager):
|
||||
"""Test deleting job returns 200."""
|
||||
response = client.delete("/api/jobs/test123")
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_delete_nonexistent_job(self, mock_worker_service):
|
||||
"""Test deleting non-existent job."""
|
||||
mock_job_manager = MagicMock()
|
||||
mock_job_manager.delete_job.return_value = False
|
||||
mock_creds = MagicMock()
|
||||
mock_creds.universe_domain = 'googleapis.com'
|
||||
|
||||
with patch('backend.api.routes.jobs.job_manager', mock_job_manager), \
|
||||
patch('backend.api.routes.jobs.worker_service', mock_worker_service), \
|
||||
patch('backend.services.firestore_service.firestore'), \
|
||||
patch('backend.services.storage_service.storage'), \
|
||||
patch('google.auth.default', return_value=(mock_creds, 'test-project')):
|
||||
from backend.main import app
|
||||
client = TestClient(app)
|
||||
response = client.delete("/api/jobs/nonexistent")
|
||||
# Either 200 or 404 depending on implementation
|
||||
assert response.status_code in [200, 404]
|
||||
|
||||
|
||||
class TestCancelJob:
|
||||
"""Tests for POST /api/jobs/{job_id}/cancel."""
|
||||
|
||||
def test_cancel_job_returns_200(self, client, mock_job_manager, mock_job):
|
||||
"""Test cancelling job returns 200."""
|
||||
mock_job_manager.cancel_job.return_value = mock_job
|
||||
response = client.post(
|
||||
"/api/jobs/test123/cancel",
|
||||
json={}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
class TestSubmitCorrections:
|
||||
"""Tests for POST /api/jobs/{job_id}/corrections."""
|
||||
|
||||
def test_submit_corrections_returns_200(self, client, mock_job_manager, mock_job):
|
||||
"""Test submitting corrections returns 200."""
|
||||
# Job needs to be in AWAITING_REVIEW or IN_REVIEW status
|
||||
review_job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.IN_REVIEW,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test",
|
||||
title="Test"
|
||||
)
|
||||
mock_job_manager.get_job.return_value = review_job
|
||||
|
||||
response = client.post(
|
||||
"/api/jobs/test123/corrections",
|
||||
json={
|
||||
"corrections": {
|
||||
"lines": [],
|
||||
"metadata": {"source": "test"}
|
||||
}
|
||||
}
|
||||
)
|
||||
# Should be 200 or validation error
|
||||
assert response.status_code in [200, 400, 422]
|
||||
|
||||
|
||||
class TestSelectInstrumental:
|
||||
"""Tests for POST /api/jobs/{job_id}/select-instrumental."""
|
||||
|
||||
def test_select_instrumental_requires_selection_field(self, client, mock_job_manager, mock_job):
|
||||
"""Test instrumental selection requires selection field."""
|
||||
response = client.post(
|
||||
"/api/jobs/test123/select-instrumental",
|
||||
json={} # Missing selection field
|
||||
)
|
||||
# Missing field should cause validation error
|
||||
assert response.status_code == 422
|
||||
|
||||
|
||||
class TestStartReview:
|
||||
"""Tests for POST /api/jobs/{job_id}/start-review."""
|
||||
|
||||
def test_start_review_endpoint_exists(self, client, mock_job_manager, mock_job):
|
||||
"""Test start review endpoint exists."""
|
||||
response = client.post("/api/jobs/test123/start-review")
|
||||
# Should not be 404 or 405
|
||||
assert response.status_code not in [404, 405]
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
"""
|
||||
Unit tests for backend main.py and config.py.
|
||||
|
||||
These tests cover the FastAPI app initialization and configuration.
|
||||
"""
|
||||
import pytest
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
class TestConfig:
|
||||
"""Tests for config.py."""
|
||||
|
||||
def test_settings_from_environment(self):
|
||||
"""Test settings can be loaded from environment variables."""
|
||||
with patch.dict(os.environ, {
|
||||
'GCS_BUCKET_NAME': 'test-bucket',
|
||||
'FIRESTORE_COLLECTION': 'test-jobs',
|
||||
'ADMIN_TOKENS': 'token1,token2',
|
||||
'ENVIRONMENT': 'testing'
|
||||
}):
|
||||
from backend.config import Settings
|
||||
settings = Settings()
|
||||
assert settings.gcs_bucket_name == 'test-bucket'
|
||||
assert settings.firestore_collection == 'test-jobs'
|
||||
# admin_tokens is a string, comma-separated
|
||||
assert 'token1' in settings.admin_tokens
|
||||
assert 'token2' in settings.admin_tokens
|
||||
|
||||
def test_admin_tokens_is_comma_separated_string(self):
|
||||
"""Test admin tokens stored as comma-separated string."""
|
||||
with patch.dict(os.environ, {
|
||||
'ADMIN_TOKENS': 'token1,token2,token3'
|
||||
}):
|
||||
from backend.config import Settings
|
||||
settings = Settings()
|
||||
# admin_tokens is stored as a string
|
||||
assert settings.admin_tokens == 'token1,token2,token3'
|
||||
|
||||
def test_default_environment(self):
|
||||
"""Test default environment is development."""
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
from backend.config import Settings
|
||||
settings = Settings()
|
||||
# Accept 'test' as well since pytest may set ENVIRONMENT=test
|
||||
assert settings.environment in ['development', 'production', 'testing', 'test']
|
||||
|
||||
|
||||
class TestMain:
|
||||
"""Tests for main.py FastAPI app."""
|
||||
|
||||
@pytest.fixture
|
||||
def app(self):
|
||||
"""Create FastAPI app with mocked services."""
|
||||
mock_creds = MagicMock()
|
||||
mock_creds.universe_domain = 'googleapis.com'
|
||||
with patch('backend.services.firestore_service.firestore'), \
|
||||
patch('backend.services.storage_service.storage'), \
|
||||
patch('google.auth.default', return_value=(mock_creds, 'test-project')):
|
||||
from backend.main import app
|
||||
return app
|
||||
|
||||
def test_app_has_api_routes(self, app):
|
||||
"""Test app includes API routes."""
|
||||
routes = [route.path for route in app.routes]
|
||||
assert any('/api' in route or '/jobs' in route for route in routes)
|
||||
|
||||
def test_app_has_cors_middleware(self, app):
|
||||
"""Test app has CORS middleware configured."""
|
||||
# Check that middleware is configured (not necessarily CORS specific)
|
||||
assert len(app.user_middleware) >= 0 # App initializes middleware
|
||||
|
||||
def test_app_title(self, app):
|
||||
"""Test app has expected title."""
|
||||
assert 'karaoke' in app.title.lower() or app.title
|
||||
|
||||
|
||||
class TestDependencies:
|
||||
"""Tests for dependencies.py."""
|
||||
|
||||
def test_dependencies_module_imports(self):
|
||||
"""Test dependencies module can be imported and has auth functions."""
|
||||
from backend.api import dependencies
|
||||
assert hasattr(dependencies, 'require_auth')
|
||||
assert hasattr(dependencies, 'require_admin')
|
||||
assert hasattr(dependencies, 'optional_auth')
|
||||
|
||||
@@ -0,0 +1,675 @@
|
||||
"""
|
||||
Unit tests for Pydantic models.
|
||||
|
||||
Tests validate that our data models work correctly, including:
|
||||
- Basic field validation
|
||||
- Nested dictionary structures (file_urls, state_data)
|
||||
- Serialization/deserialization round-trips (simulating Firestore)
|
||||
- Real-world data structures from workers
|
||||
|
||||
These tests catch type mismatches that would cause runtime errors when
|
||||
reading data back from Firestore.
|
||||
"""
|
||||
import pytest
|
||||
import json
|
||||
from datetime import datetime, UTC
|
||||
from pydantic import ValidationError
|
||||
|
||||
from backend.models.job import (
|
||||
Job, JobCreate, JobStatus, TimelineEvent
|
||||
)
|
||||
|
||||
|
||||
class TestJobModel:
|
||||
"""Test Job Pydantic model - the bug we just fixed!"""
|
||||
|
||||
def test_input_media_gcs_path_field_exists(self):
|
||||
"""
|
||||
Test that Job model has input_media_gcs_path field.
|
||||
|
||||
This test would have caught the bug where we tried to use
|
||||
job.input_media_gcs_path but it wasn't defined in the model!
|
||||
"""
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
input_media_gcs_path="uploads/test123/file.flac"
|
||||
)
|
||||
|
||||
# This should NOT raise AttributeError
|
||||
assert hasattr(job, 'input_media_gcs_path')
|
||||
assert job.input_media_gcs_path == "uploads/test123/file.flac"
|
||||
|
||||
def test_input_media_gcs_path_optional(self):
|
||||
"""Test that input_media_gcs_path is optional."""
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
assert job.input_media_gcs_path is None
|
||||
|
||||
def test_pydantic_includes_input_media_gcs_path_in_dict(self):
|
||||
"""
|
||||
Test that Pydantic includes input_media_gcs_path in serialization.
|
||||
|
||||
This ensures Pydantic doesn't silently ignore the field.
|
||||
"""
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
input_media_gcs_path="uploads/test123/file.flac"
|
||||
)
|
||||
|
||||
job_dict = job.model_dump()
|
||||
|
||||
# Pydantic should include it
|
||||
assert "input_media_gcs_path" in job_dict
|
||||
assert job_dict["input_media_gcs_path"] == "uploads/test123/file.flac"
|
||||
|
||||
def test_create_minimal_job(self):
|
||||
"""Test creating a job with minimal required fields."""
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
assert job.job_id == "test123"
|
||||
assert job.status == JobStatus.PENDING
|
||||
assert job.progress == 0
|
||||
assert job.url is None
|
||||
assert job.input_media_gcs_path is None
|
||||
|
||||
def test_create_job_with_url(self):
|
||||
"""Test creating a job with YouTube URL."""
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
url="https://youtube.com/watch?v=test",
|
||||
artist="Test Artist",
|
||||
title="Test Song"
|
||||
)
|
||||
|
||||
assert job.url == "https://youtube.com/watch?v=test"
|
||||
assert job.artist == "Test Artist"
|
||||
assert job.title == "Test Song"
|
||||
|
||||
def test_create_job_with_uploaded_file(self):
|
||||
"""Test creating a job with uploaded file."""
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
input_media_gcs_path="uploads/test123/file.flac",
|
||||
artist="Test Artist",
|
||||
title="Test Song"
|
||||
)
|
||||
|
||||
assert job.input_media_gcs_path == "uploads/test123/file.flac"
|
||||
assert job.url is None
|
||||
|
||||
|
||||
class TestJobCreate:
|
||||
"""Test JobCreate validation model."""
|
||||
|
||||
def test_create_with_url(self):
|
||||
"""Test creating job with YouTube URL."""
|
||||
job_create = JobCreate(
|
||||
url="https://youtube.com/watch?v=test",
|
||||
artist="Test Artist",
|
||||
title="Test Song"
|
||||
)
|
||||
|
||||
assert job_create.url == "https://youtube.com/watch?v=test"
|
||||
assert job_create.artist == "Test Artist"
|
||||
assert job_create.title == "Test Song"
|
||||
|
||||
def test_create_minimal(self):
|
||||
"""Test creating job with minimal fields."""
|
||||
job_create = JobCreate()
|
||||
|
||||
assert job_create.url is None
|
||||
assert job_create.artist is None
|
||||
assert job_create.title is None
|
||||
|
||||
|
||||
class TestJobStatus:
|
||||
"""Test JobStatus enum."""
|
||||
|
||||
def test_critical_statuses_defined(self):
|
||||
"""Test that critical statuses exist."""
|
||||
critical_statuses = [
|
||||
"pending", "downloading",
|
||||
"separating_stage1", "separating_stage2", "audio_complete",
|
||||
"transcribing", "correcting", "lyrics_complete",
|
||||
"generating_screens", "applying_padding",
|
||||
"awaiting_review", "in_review", "review_complete",
|
||||
"awaiting_instrumental_selection", "instrumental_selected",
|
||||
"generating_video", "encoding", "packaging",
|
||||
"complete", "failed"
|
||||
]
|
||||
|
||||
actual_statuses = [status.value for status in JobStatus]
|
||||
|
||||
for status in critical_statuses:
|
||||
assert status in actual_statuses, f"Missing critical status: {status}"
|
||||
|
||||
|
||||
class TestTimelineEvent:
|
||||
"""Test TimelineEvent model."""
|
||||
|
||||
def test_create_timeline_event(self):
|
||||
"""Test creating a timeline event."""
|
||||
event = TimelineEvent(
|
||||
status="pending",
|
||||
timestamp="2025-12-01T08:00:00Z",
|
||||
progress=0,
|
||||
message="Job created"
|
||||
)
|
||||
|
||||
assert event.status == "pending"
|
||||
assert event.timestamp == "2025-12-01T08:00:00Z"
|
||||
assert event.progress == 0
|
||||
assert event.message == "Job created"
|
||||
|
||||
def test_timeline_event_optional_fields(self):
|
||||
"""Test that progress and message are optional."""
|
||||
event = TimelineEvent(
|
||||
status="pending",
|
||||
timestamp="2025-12-01T08:00:00Z"
|
||||
)
|
||||
|
||||
assert event.progress is None
|
||||
assert event.message is None
|
||||
|
||||
|
||||
class TestModelValidation:
|
||||
"""Test model validation rules."""
|
||||
|
||||
def test_invalid_job_status(self):
|
||||
"""Test that invalid status is rejected."""
|
||||
with pytest.raises((ValidationError, ValueError)):
|
||||
Job(
|
||||
job_id="test123",
|
||||
status="invalid_status", # Not a valid JobStatus
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
def test_missing_required_fields(self):
|
||||
"""Test that missing required fields are rejected."""
|
||||
with pytest.raises(ValidationError):
|
||||
Job(
|
||||
job_id="test123"
|
||||
# Missing required fields: status, created_at, updated_at
|
||||
)
|
||||
|
||||
|
||||
class TestFileUrlsNestedStructure:
|
||||
"""
|
||||
Test that file_urls field correctly handles nested dictionaries.
|
||||
|
||||
THIS IS THE TEST THAT WOULD HAVE CAUGHT THE Dict[str, str] vs Dict[str, Any] BUG!
|
||||
|
||||
Workers store nested structures like:
|
||||
{
|
||||
"stems": {"instrumental_clean": "gs://...", "vocals": "gs://..."},
|
||||
"lyrics": {"corrections": "gs://...", "lrc": "gs://..."},
|
||||
...
|
||||
}
|
||||
|
||||
If file_urls is typed as Dict[str, str], this will fail on deserialization.
|
||||
"""
|
||||
|
||||
def test_file_urls_with_nested_stems(self):
|
||||
"""Test that file_urls accepts nested stems dictionary."""
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.AUDIO_COMPLETE,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
file_urls={
|
||||
"stems": {
|
||||
"instrumental_clean": "gs://bucket/jobs/test123/stems/instrumental_clean.flac",
|
||||
"instrumental_with_backing": "gs://bucket/jobs/test123/stems/instrumental_with_backing.flac",
|
||||
"vocals_clean": "gs://bucket/jobs/test123/stems/vocals_clean.flac",
|
||||
"lead_vocals": "gs://bucket/jobs/test123/stems/lead_vocals.flac",
|
||||
"backing_vocals": "gs://bucket/jobs/test123/stems/backing_vocals.flac",
|
||||
"bass": "gs://bucket/jobs/test123/stems/bass.flac",
|
||||
"drums": "gs://bucket/jobs/test123/stems/drums.flac",
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
assert job.file_urls["stems"]["instrumental_clean"] == "gs://bucket/jobs/test123/stems/instrumental_clean.flac"
|
||||
assert len(job.file_urls["stems"]) == 7
|
||||
|
||||
def test_file_urls_with_nested_lyrics(self):
|
||||
"""Test that file_urls accepts nested lyrics dictionary."""
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.LYRICS_COMPLETE,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
file_urls={
|
||||
"lyrics": {
|
||||
"corrections": "gs://bucket/jobs/test123/lyrics/corrections.json",
|
||||
"audio": "gs://bucket/jobs/test123/lyrics/audio.flac",
|
||||
"lrc": "gs://bucket/jobs/test123/lyrics/karaoke.lrc",
|
||||
"uncorrected": "gs://bucket/jobs/test123/lyrics/uncorrected.txt",
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
assert job.file_urls["lyrics"]["corrections"] == "gs://bucket/jobs/test123/lyrics/corrections.json"
|
||||
|
||||
def test_file_urls_full_worker_structure(self):
|
||||
"""
|
||||
Test with the full structure that workers actually create.
|
||||
|
||||
This simulates what actually gets stored in Firestore after
|
||||
all workers have run.
|
||||
"""
|
||||
full_file_urls = {
|
||||
"input": {
|
||||
"audio": "gs://bucket/jobs/test123/input/waterloo.flac"
|
||||
},
|
||||
"stems": {
|
||||
"instrumental_clean": "gs://bucket/jobs/test123/stems/instrumental_clean.flac",
|
||||
"instrumental_with_backing": "gs://bucket/jobs/test123/stems/instrumental_with_backing.flac",
|
||||
"vocals_clean": "gs://bucket/jobs/test123/stems/vocals_clean.flac",
|
||||
"lead_vocals": "gs://bucket/jobs/test123/stems/lead_vocals.flac",
|
||||
"backing_vocals": "gs://bucket/jobs/test123/stems/backing_vocals.flac",
|
||||
},
|
||||
"lyrics": {
|
||||
"corrections": "gs://bucket/jobs/test123/lyrics/corrections.json",
|
||||
"lrc": "gs://bucket/jobs/test123/lyrics/karaoke.lrc",
|
||||
},
|
||||
"screens": {
|
||||
"title": "gs://bucket/jobs/test123/screens/title.mov",
|
||||
"end": "gs://bucket/jobs/test123/screens/end.mov",
|
||||
},
|
||||
"videos": {
|
||||
"with_vocals": "gs://bucket/jobs/test123/videos/with_vocals.mkv",
|
||||
},
|
||||
"finals": {
|
||||
"lossless_4k_mp4": "gs://bucket/jobs/test123/finals/lossless_4k.mp4",
|
||||
"lossy_720p_mp4": "gs://bucket/jobs/test123/finals/lossy_720p.mp4",
|
||||
}
|
||||
}
|
||||
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.COMPLETE,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
file_urls=full_file_urls
|
||||
)
|
||||
|
||||
# Verify nested access works
|
||||
assert job.file_urls["stems"]["instrumental_clean"].endswith("instrumental_clean.flac")
|
||||
assert job.file_urls["lyrics"]["corrections"].endswith("corrections.json")
|
||||
assert job.file_urls["finals"]["lossy_720p_mp4"].endswith("lossy_720p.mp4")
|
||||
|
||||
def test_file_urls_roundtrip_serialization(self):
|
||||
"""
|
||||
Test that file_urls survives serialization/deserialization.
|
||||
|
||||
This simulates what happens when we:
|
||||
1. Create a Job with nested file_urls
|
||||
2. Serialize to dict (for Firestore)
|
||||
3. Deserialize back to Job (reading from Firestore)
|
||||
|
||||
THIS IS THE KEY TEST - it would catch the Dict[str, str] bug!
|
||||
"""
|
||||
original_file_urls = {
|
||||
"stems": {
|
||||
"instrumental_clean": "gs://bucket/stems/clean.flac",
|
||||
"vocals": "gs://bucket/stems/vocals.flac",
|
||||
},
|
||||
"lyrics": {
|
||||
"corrections": "gs://bucket/lyrics/corrections.json",
|
||||
}
|
||||
}
|
||||
|
||||
# Create job
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
file_urls=original_file_urls
|
||||
)
|
||||
|
||||
# Serialize to dict (simulates writing to Firestore)
|
||||
job_dict = job.model_dump()
|
||||
|
||||
# Convert to JSON and back (simulates Firestore storage)
|
||||
json_str = json.dumps(job_dict, default=str)
|
||||
restored_dict = json.loads(json_str)
|
||||
|
||||
# Deserialize back to Job (simulates reading from Firestore)
|
||||
restored_job = Job(**restored_dict)
|
||||
|
||||
# Verify nested structure survived
|
||||
assert restored_job.file_urls["stems"]["instrumental_clean"] == "gs://bucket/stems/clean.flac"
|
||||
assert restored_job.file_urls["stems"]["vocals"] == "gs://bucket/stems/vocals.flac"
|
||||
assert restored_job.file_urls["lyrics"]["corrections"] == "gs://bucket/lyrics/corrections.json"
|
||||
|
||||
|
||||
class TestStateDataNestedStructure:
|
||||
"""Test that state_data field correctly handles nested dictionaries."""
|
||||
|
||||
def test_state_data_with_instrumental_options(self):
|
||||
"""Test state_data with instrumental options structure."""
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.AWAITING_INSTRUMENTAL_SELECTION,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={
|
||||
"instrumental_options": {
|
||||
"clean": "gs://bucket/jobs/test123/stems/instrumental_clean.flac",
|
||||
"with_backing": "gs://bucket/jobs/test123/stems/instrumental_with_backing.flac",
|
||||
},
|
||||
"audio_complete": True,
|
||||
"lyrics_complete": True,
|
||||
}
|
||||
)
|
||||
|
||||
assert job.state_data["instrumental_options"]["clean"].endswith("clean.flac")
|
||||
assert job.state_data["audio_complete"] is True
|
||||
|
||||
def test_state_data_with_lyrics_metadata(self):
|
||||
"""Test state_data with lyrics metadata structure."""
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.LYRICS_COMPLETE,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data={
|
||||
"lyrics_metadata": {
|
||||
"line_count": 42,
|
||||
"has_corrections": True,
|
||||
"ready_for_review": True,
|
||||
},
|
||||
"corrected_lyrics": {
|
||||
"lines": [
|
||||
{"text": "Hello world", "start_time": 1.0, "end_time": 2.0}
|
||||
],
|
||||
"metadata": {"song": "Test Song"}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
assert job.state_data["lyrics_metadata"]["line_count"] == 42
|
||||
assert len(job.state_data["corrected_lyrics"]["lines"]) == 1
|
||||
|
||||
def test_state_data_roundtrip_serialization(self):
|
||||
"""Test that state_data survives serialization/deserialization."""
|
||||
original_state_data = {
|
||||
"instrumental_selection": "clean",
|
||||
"review_notes": "Looks good!",
|
||||
"complex_nested": {
|
||||
"level1": {
|
||||
"level2": {
|
||||
"value": 123
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
state_data=original_state_data
|
||||
)
|
||||
|
||||
# Serialize and deserialize
|
||||
job_dict = job.model_dump()
|
||||
json_str = json.dumps(job_dict, default=str)
|
||||
restored_dict = json.loads(json_str)
|
||||
restored_job = Job(**restored_dict)
|
||||
|
||||
# Verify nested structure survived
|
||||
assert restored_job.state_data["complex_nested"]["level1"]["level2"]["value"] == 123
|
||||
|
||||
|
||||
class TestWorkerIdsNestedStructure:
|
||||
"""Test that worker_ids field correctly handles nested dictionaries."""
|
||||
|
||||
def test_worker_ids_with_all_workers(self):
|
||||
"""Test worker_ids with all worker types."""
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.GENERATING_VIDEO,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
worker_ids={
|
||||
"audio_worker": "cloud-run-request-12345",
|
||||
"lyrics_worker": "cloud-run-request-67890",
|
||||
"screens_worker": "cloud-run-request-11111",
|
||||
"video_worker": "cloud-run-request-22222",
|
||||
}
|
||||
)
|
||||
|
||||
assert job.worker_ids["audio_worker"] == "cloud-run-request-12345"
|
||||
assert len(job.worker_ids) == 4
|
||||
|
||||
|
||||
class TestJobModelIntegration:
|
||||
"""
|
||||
Integration tests that simulate real Firestore interactions.
|
||||
|
||||
These tests verify that the Job model works correctly when used
|
||||
the same way as our actual services.
|
||||
"""
|
||||
|
||||
def test_simulate_job_manager_update_file_url(self):
|
||||
"""
|
||||
Simulate what JobManager.update_file_url does.
|
||||
|
||||
This is the pattern that caused the bug - updating nested file_urls.
|
||||
"""
|
||||
# Start with a fresh job
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
file_urls={}
|
||||
)
|
||||
|
||||
# Simulate update_file_url('test123', 'stems', 'instrumental_clean', 'gs://...')
|
||||
# This creates nested structure: {"stems": {"instrumental_clean": "gs://..."}}
|
||||
if "stems" not in job.file_urls:
|
||||
job.file_urls["stems"] = {}
|
||||
job.file_urls["stems"]["instrumental_clean"] = "gs://bucket/stems/clean.flac"
|
||||
|
||||
# Simulate Firestore roundtrip
|
||||
job_dict = job.model_dump()
|
||||
restored_job = Job(**job_dict)
|
||||
|
||||
# This is what was failing with Dict[str, str]!
|
||||
assert restored_job.file_urls["stems"]["instrumental_clean"] == "gs://bucket/stems/clean.flac"
|
||||
|
||||
def test_simulate_audio_worker_completion(self):
|
||||
"""Simulate the data structure created when audio worker completes."""
|
||||
# This is the actual structure created by audio_worker.py
|
||||
file_urls = {
|
||||
"stems": {
|
||||
"instrumental_clean": "gs://bucket/jobs/abc123/stems/instrumental_clean.flac",
|
||||
"vocals_clean": "gs://bucket/jobs/abc123/stems/vocals_clean.flac",
|
||||
"lead_vocals": "gs://bucket/jobs/abc123/stems/lead_vocals.flac",
|
||||
"backing_vocals": "gs://bucket/jobs/abc123/stems/backing_vocals.flac",
|
||||
"instrumental_with_backing": "gs://bucket/jobs/abc123/stems/instrumental_with_backing.flac",
|
||||
}
|
||||
}
|
||||
|
||||
state_data = {
|
||||
"instrumental_options": {
|
||||
"clean": "jobs/abc123/stems/instrumental_clean.flac",
|
||||
"with_backing": "jobs/abc123/stems/instrumental_with_backing.flac",
|
||||
}
|
||||
}
|
||||
|
||||
job = Job(
|
||||
job_id="abc123",
|
||||
status=JobStatus.AUDIO_COMPLETE,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="ABBA",
|
||||
title="Waterloo",
|
||||
file_urls=file_urls,
|
||||
state_data=state_data
|
||||
)
|
||||
|
||||
# Roundtrip
|
||||
restored_job = Job(**job.model_dump())
|
||||
|
||||
assert len(restored_job.file_urls["stems"]) == 5
|
||||
assert "instrumental_options" in restored_job.state_data
|
||||
|
||||
def test_simulate_lyrics_worker_completion(self):
|
||||
"""Simulate the data structure created when lyrics worker completes."""
|
||||
file_urls = {
|
||||
"lyrics": {
|
||||
"corrections": "gs://bucket/jobs/abc123/lyrics/corrections.json",
|
||||
"lrc": "gs://bucket/jobs/abc123/lyrics/karaoke.lrc",
|
||||
"uncorrected": "gs://bucket/jobs/abc123/lyrics/uncorrected.txt",
|
||||
}
|
||||
}
|
||||
|
||||
state_data = {
|
||||
"lyrics_metadata": {
|
||||
"line_count": 50,
|
||||
"has_corrections": True,
|
||||
"ready_for_review": True,
|
||||
}
|
||||
}
|
||||
|
||||
job = Job(
|
||||
job_id="abc123",
|
||||
status=JobStatus.LYRICS_COMPLETE,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
file_urls=file_urls,
|
||||
state_data=state_data
|
||||
)
|
||||
|
||||
# Roundtrip
|
||||
restored_job = Job(**job.model_dump())
|
||||
|
||||
assert restored_job.file_urls["lyrics"]["corrections"].endswith("corrections.json")
|
||||
assert restored_job.state_data["lyrics_metadata"]["line_count"] == 50
|
||||
|
||||
|
||||
class TestExistingInstrumentalModelFields:
|
||||
"""Test existing instrumental field in Job and JobCreate models (Batch 3)."""
|
||||
|
||||
def test_job_has_existing_instrumental_gcs_path(self):
|
||||
"""Test that Job model has existing_instrumental_gcs_path field."""
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
existing_instrumental_gcs_path="uploads/test123/audio/existing_instrumental.flac"
|
||||
)
|
||||
|
||||
assert hasattr(job, 'existing_instrumental_gcs_path')
|
||||
assert job.existing_instrumental_gcs_path == "uploads/test123/audio/existing_instrumental.flac"
|
||||
|
||||
def test_job_existing_instrumental_defaults_to_none(self):
|
||||
"""Test that existing_instrumental_gcs_path defaults to None."""
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
assert job.existing_instrumental_gcs_path is None
|
||||
|
||||
def test_job_create_has_existing_instrumental_gcs_path(self):
|
||||
"""Test that JobCreate model has existing_instrumental_gcs_path field."""
|
||||
from backend.models.job import JobCreate
|
||||
|
||||
job_create = JobCreate(
|
||||
artist="Artist",
|
||||
title="Title",
|
||||
existing_instrumental_gcs_path="uploads/test/audio/existing_instrumental.mp3"
|
||||
)
|
||||
|
||||
assert job_create.existing_instrumental_gcs_path == "uploads/test/audio/existing_instrumental.mp3"
|
||||
|
||||
def test_job_create_existing_instrumental_optional(self):
|
||||
"""Test that existing_instrumental_gcs_path is optional in JobCreate."""
|
||||
from backend.models.job import JobCreate
|
||||
|
||||
job_create = JobCreate(
|
||||
artist="Artist",
|
||||
title="Title"
|
||||
)
|
||||
|
||||
assert job_create.existing_instrumental_gcs_path is None
|
||||
|
||||
def test_existing_instrumental_roundtrip_serialization(self):
|
||||
"""Test that existing_instrumental_gcs_path survives serialization/deserialization."""
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
existing_instrumental_gcs_path="uploads/test123/audio/existing_instrumental.wav"
|
||||
)
|
||||
|
||||
# Serialize to dict (simulates writing to Firestore)
|
||||
job_dict = job.model_dump()
|
||||
|
||||
# Convert to JSON and back (simulates Firestore storage)
|
||||
json_str = json.dumps(job_dict, default=str)
|
||||
restored_dict = json.loads(json_str)
|
||||
|
||||
# Deserialize back to Job (simulates reading from Firestore)
|
||||
restored_job = Job(**restored_dict)
|
||||
|
||||
assert restored_job.existing_instrumental_gcs_path == "uploads/test123/audio/existing_instrumental.wav"
|
||||
|
||||
def test_job_with_full_audio_config(self):
|
||||
"""Test Job with complete audio configuration including existing instrumental."""
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.AUDIO_COMPLETE,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
# Audio model configuration
|
||||
clean_instrumental_model="model_bs_roformer_ep_317_sdr_12.9755.ckpt",
|
||||
backing_vocals_models=["mel_band_roformer_karaoke.ckpt"],
|
||||
other_stems_models=["htdemucs_6s.yaml"],
|
||||
# Existing instrumental
|
||||
existing_instrumental_gcs_path="uploads/test123/audio/my_instrumental.flac",
|
||||
)
|
||||
|
||||
assert job.clean_instrumental_model == "model_bs_roformer_ep_317_sdr_12.9755.ckpt"
|
||||
assert job.existing_instrumental_gcs_path == "uploads/test123/audio/my_instrumental.flac"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
"""
|
||||
Unit tests for request models.
|
||||
|
||||
These tests validate Pydantic request model validation and serialization.
|
||||
"""
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from backend.models.requests import (
|
||||
URLSubmissionRequest,
|
||||
UploadSubmissionRequest,
|
||||
CorrectionsSubmission,
|
||||
InstrumentalSelection,
|
||||
StartReviewRequest,
|
||||
CancelJobRequest,
|
||||
RetryJobRequest
|
||||
)
|
||||
|
||||
|
||||
class TestURLSubmissionRequest:
|
||||
"""Tests for URLSubmissionRequest model."""
|
||||
|
||||
def test_valid_youtube_url(self):
|
||||
"""Test valid YouTube URL is accepted."""
|
||||
request = URLSubmissionRequest(url="https://youtube.com/watch?v=test123")
|
||||
assert str(request.url) == "https://youtube.com/watch?v=test123"
|
||||
|
||||
def test_valid_youtube_short_url(self):
|
||||
"""Test valid YouTube short URL is accepted."""
|
||||
request = URLSubmissionRequest(url="https://youtu.be/test123")
|
||||
assert str(request.url) == "https://youtu.be/test123"
|
||||
|
||||
def test_optional_artist_title(self):
|
||||
"""Test artist and title are optional."""
|
||||
request = URLSubmissionRequest(url="https://youtube.com/watch?v=test")
|
||||
assert request.artist is None
|
||||
assert request.title is None
|
||||
|
||||
def test_with_artist_title(self):
|
||||
"""Test request with artist and title."""
|
||||
request = URLSubmissionRequest(
|
||||
url="https://youtube.com/watch?v=test",
|
||||
artist="Test Artist",
|
||||
title="Test Song"
|
||||
)
|
||||
assert request.artist == "Test Artist"
|
||||
assert request.title == "Test Song"
|
||||
|
||||
def test_url_required(self):
|
||||
"""Test URL is required."""
|
||||
with pytest.raises(ValidationError):
|
||||
URLSubmissionRequest()
|
||||
|
||||
def test_empty_url_rejected(self):
|
||||
"""Test empty URL is rejected."""
|
||||
with pytest.raises(ValidationError):
|
||||
URLSubmissionRequest(url="")
|
||||
|
||||
|
||||
class TestCorrectionsSubmission:
|
||||
"""Tests for CorrectionsSubmission model."""
|
||||
|
||||
def test_valid_corrections(self):
|
||||
"""Test valid corrections submission."""
|
||||
corrections = CorrectionsSubmission(
|
||||
corrections={
|
||||
"lines": [
|
||||
{"text": "Hello world", "start": 0.0, "end": 1.0}
|
||||
],
|
||||
"metadata": {"source": "test"}
|
||||
}
|
||||
)
|
||||
assert "lines" in corrections.corrections
|
||||
assert "metadata" in corrections.corrections
|
||||
|
||||
def test_missing_lines_rejected(self):
|
||||
"""Test corrections without 'lines' field is rejected."""
|
||||
with pytest.raises(ValidationError):
|
||||
CorrectionsSubmission(corrections={"metadata": {}})
|
||||
|
||||
def test_missing_metadata_rejected(self):
|
||||
"""Test corrections without 'metadata' field is rejected."""
|
||||
with pytest.raises(ValidationError):
|
||||
CorrectionsSubmission(corrections={"lines": []})
|
||||
|
||||
def test_corrections_required(self):
|
||||
"""Test corrections field is required."""
|
||||
with pytest.raises(ValidationError):
|
||||
CorrectionsSubmission()
|
||||
|
||||
def test_with_user_notes(self):
|
||||
"""Test corrections with user notes."""
|
||||
corrections = CorrectionsSubmission(
|
||||
corrections={"lines": [], "metadata": {}},
|
||||
user_notes="Fixed typo in line 3"
|
||||
)
|
||||
assert corrections.user_notes == "Fixed typo in line 3"
|
||||
|
||||
|
||||
class TestInstrumentalSelection:
|
||||
"""Tests for InstrumentalSelection model."""
|
||||
|
||||
def test_valid_clean_selection(self):
|
||||
"""Test selecting clean instrumental."""
|
||||
selection = InstrumentalSelection(selection="clean")
|
||||
assert selection.selection == "clean"
|
||||
|
||||
def test_valid_with_backing_selection(self):
|
||||
"""Test selecting instrumental with backing vocals."""
|
||||
selection = InstrumentalSelection(selection="with_backing")
|
||||
assert selection.selection == "with_backing"
|
||||
|
||||
def test_selection_required(self):
|
||||
"""Test selection is required."""
|
||||
with pytest.raises(ValidationError):
|
||||
InstrumentalSelection()
|
||||
|
||||
def test_invalid_selection_rejected(self):
|
||||
"""Test invalid selection values are rejected."""
|
||||
with pytest.raises(ValidationError):
|
||||
InstrumentalSelection(selection="invalid_option")
|
||||
|
||||
|
||||
class TestUploadSubmissionRequest:
|
||||
"""Tests for UploadSubmissionRequest model."""
|
||||
|
||||
def test_valid_upload_request(self):
|
||||
"""Test valid upload submission."""
|
||||
request = UploadSubmissionRequest(
|
||||
artist="Test Artist",
|
||||
title="Test Song"
|
||||
)
|
||||
assert request.artist == "Test Artist"
|
||||
assert request.title == "Test Song"
|
||||
|
||||
def test_artist_required(self):
|
||||
"""Test artist is required."""
|
||||
with pytest.raises(ValidationError):
|
||||
UploadSubmissionRequest(title="Test Song")
|
||||
|
||||
def test_title_required(self):
|
||||
"""Test title is required."""
|
||||
with pytest.raises(ValidationError):
|
||||
UploadSubmissionRequest(artist="Test Artist")
|
||||
|
||||
def test_default_options(self):
|
||||
"""Test default option values."""
|
||||
request = UploadSubmissionRequest(artist="Test", title="Test")
|
||||
assert request.enable_cdg is True
|
||||
assert request.enable_txt is True
|
||||
assert request.enable_youtube_upload is False
|
||||
|
||||
|
||||
class TestStartReviewRequest:
|
||||
"""Tests for StartReviewRequest model."""
|
||||
|
||||
def test_valid_request(self):
|
||||
"""Test valid start review request."""
|
||||
request = StartReviewRequest()
|
||||
assert request is not None
|
||||
|
||||
|
||||
class TestCancelJobRequest:
|
||||
"""Tests for CancelJobRequest model."""
|
||||
|
||||
def test_valid_request(self):
|
||||
"""Test valid cancel request."""
|
||||
request = CancelJobRequest()
|
||||
assert request is not None
|
||||
|
||||
def test_with_reason(self):
|
||||
"""Test cancel with reason."""
|
||||
request = CancelJobRequest(reason="User requested")
|
||||
assert request.reason == "User requested"
|
||||
|
||||
def test_reason_optional(self):
|
||||
"""Test reason is optional."""
|
||||
request = CancelJobRequest()
|
||||
assert request.reason is None
|
||||
|
||||
|
||||
class TestRetryJobRequest:
|
||||
"""Tests for RetryJobRequest model."""
|
||||
|
||||
def test_valid_request(self):
|
||||
"""Test valid retry request."""
|
||||
request = RetryJobRequest()
|
||||
assert request is not None
|
||||
|
||||
def test_with_from_stage(self):
|
||||
"""Test retry from specific stage."""
|
||||
request = RetryJobRequest(from_stage="transcription")
|
||||
assert request.from_stage == "transcription"
|
||||
|
||||
def test_from_stage_optional(self):
|
||||
"""Test from_stage is optional."""
|
||||
request = RetryJobRequest()
|
||||
assert request.from_stage is None
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
"""
|
||||
Unit tests for jobs.py routes.
|
||||
|
||||
These tests exercise the route logic with mocked services.
|
||||
"""
|
||||
import pytest
|
||||
from datetime import datetime, UTC
|
||||
from unittest.mock import MagicMock, AsyncMock, patch
|
||||
|
||||
from backend.models.job import Job, JobStatus
|
||||
|
||||
|
||||
class TestJobsRouteHelpers:
|
||||
"""Tests for helper functions in jobs.py routes."""
|
||||
|
||||
def test_jobs_router_structure(self):
|
||||
"""Test jobs router has expected structure."""
|
||||
from backend.api.routes.jobs import router
|
||||
assert router is not None
|
||||
|
||||
# Check that common route patterns exist
|
||||
route_paths = [route.path for route in router.routes]
|
||||
assert any('/jobs' in p or 'jobs' in str(p) for p in route_paths)
|
||||
|
||||
|
||||
class TestJobStatusTransitions:
|
||||
"""Tests for job status transition validation.
|
||||
|
||||
These test the Job model's state machine which is critical for
|
||||
preventing invalid operations.
|
||||
"""
|
||||
|
||||
def test_valid_pending_to_downloading(self):
|
||||
"""Test PENDING -> DOWNLOADING is valid."""
|
||||
from backend.models.job import STATE_TRANSITIONS
|
||||
valid_transitions = STATE_TRANSITIONS.get(JobStatus.PENDING, [])
|
||||
assert JobStatus.DOWNLOADING in valid_transitions
|
||||
|
||||
def test_valid_downloading_to_separating(self):
|
||||
"""Test DOWNLOADING -> SEPARATING_STAGE1 is valid."""
|
||||
from backend.models.job import STATE_TRANSITIONS
|
||||
valid_transitions = STATE_TRANSITIONS.get(JobStatus.DOWNLOADING, [])
|
||||
assert JobStatus.SEPARATING_STAGE1 in valid_transitions
|
||||
|
||||
def test_invalid_pending_to_complete(self):
|
||||
"""Test PENDING -> COMPLETE is invalid."""
|
||||
from backend.models.job import STATE_TRANSITIONS
|
||||
valid_transitions = STATE_TRANSITIONS.get(JobStatus.PENDING, [])
|
||||
assert JobStatus.COMPLETE not in valid_transitions
|
||||
|
||||
def test_any_active_status_can_fail(self):
|
||||
"""Test any active (non-terminal, non-legacy) status can transition to FAILED."""
|
||||
from backend.models.job import STATE_TRANSITIONS
|
||||
# Terminal states and legacy states are excluded
|
||||
excluded = [
|
||||
JobStatus.COMPLETE, JobStatus.FAILED, JobStatus.CANCELLED,
|
||||
# Legacy states that don't include FAILED
|
||||
JobStatus.QUEUED, JobStatus.PROCESSING,
|
||||
JobStatus.READY_FOR_FINALIZATION, JobStatus.FINALIZING, JobStatus.ERROR
|
||||
]
|
||||
for status in JobStatus:
|
||||
if status not in excluded:
|
||||
valid_transitions = STATE_TRANSITIONS.get(status, [])
|
||||
assert JobStatus.FAILED in valid_transitions, f"{status} should be able to fail"
|
||||
|
||||
def test_failed_can_transition_for_retry(self):
|
||||
"""Test FAILED status can transition to retry checkpoint states."""
|
||||
from backend.models.job import STATE_TRANSITIONS
|
||||
valid_transitions = STATE_TRANSITIONS.get(JobStatus.FAILED, [])
|
||||
|
||||
# FAILED should allow retry transitions
|
||||
assert JobStatus.INSTRUMENTAL_SELECTED in valid_transitions, "FAILED should allow retry to INSTRUMENTAL_SELECTED"
|
||||
assert JobStatus.REVIEW_COMPLETE in valid_transitions, "FAILED should allow retry to REVIEW_COMPLETE"
|
||||
assert JobStatus.LYRICS_COMPLETE in valid_transitions, "FAILED should allow retry to LYRICS_COMPLETE"
|
||||
|
||||
|
||||
class TestJobModelSerialization:
|
||||
"""Tests for Job model serialization to/from API."""
|
||||
|
||||
def test_job_to_dict(self):
|
||||
"""Test Job converts to dict correctly."""
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test Artist",
|
||||
title="Test Song"
|
||||
)
|
||||
data = job.model_dump()
|
||||
assert data["job_id"] == "test123"
|
||||
assert data["status"] == "pending"
|
||||
assert data["artist"] == "Test Artist"
|
||||
|
||||
def test_job_from_dict(self):
|
||||
"""Test Job can be created from dict."""
|
||||
data = {
|
||||
"job_id": "test123",
|
||||
"status": "pending",
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
"updated_at": datetime.now(UTC).isoformat(),
|
||||
"artist": "Test",
|
||||
"title": "Song"
|
||||
}
|
||||
job = Job.model_validate(data)
|
||||
assert job.job_id == "test123"
|
||||
assert job.status == JobStatus.PENDING
|
||||
|
||||
def test_job_handles_null_optional_fields(self):
|
||||
"""Test Job handles null optional fields."""
|
||||
job = Job(
|
||||
job_id="test",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
assert job.artist is None
|
||||
assert job.title is None
|
||||
assert job.url is None
|
||||
assert job.file_urls == {}
|
||||
|
||||
@@ -0,0 +1,337 @@
|
||||
"""
|
||||
Unit tests for review.py routes and related components.
|
||||
|
||||
These tests verify the review-related state transitions and data structures.
|
||||
Tests that require full backend imports are in the emulator integration tests.
|
||||
"""
|
||||
import pytest
|
||||
import json
|
||||
|
||||
|
||||
class TestJobStatusTransitionsForReview:
|
||||
"""Tests for review-related state transitions.
|
||||
|
||||
These tests verify the Job model's state machine handles review flow correctly.
|
||||
"""
|
||||
|
||||
def test_awaiting_review_can_transition_to_in_review(self):
|
||||
"""Test AWAITING_REVIEW -> IN_REVIEW is valid."""
|
||||
from backend.models.job import STATE_TRANSITIONS, JobStatus
|
||||
valid_transitions = STATE_TRANSITIONS.get(JobStatus.AWAITING_REVIEW, [])
|
||||
assert JobStatus.IN_REVIEW in valid_transitions
|
||||
|
||||
def test_awaiting_review_can_transition_to_review_complete(self):
|
||||
"""Test AWAITING_REVIEW -> REVIEW_COMPLETE is valid (skip in_review)."""
|
||||
from backend.models.job import STATE_TRANSITIONS, JobStatus
|
||||
valid_transitions = STATE_TRANSITIONS.get(JobStatus.AWAITING_REVIEW, [])
|
||||
assert JobStatus.REVIEW_COMPLETE in valid_transitions
|
||||
|
||||
def test_in_review_can_transition_to_review_complete(self):
|
||||
"""Test IN_REVIEW -> REVIEW_COMPLETE is valid."""
|
||||
from backend.models.job import STATE_TRANSITIONS, JobStatus
|
||||
valid_transitions = STATE_TRANSITIONS.get(JobStatus.IN_REVIEW, [])
|
||||
assert JobStatus.REVIEW_COMPLETE in valid_transitions
|
||||
|
||||
def test_review_complete_transitions_to_rendering_video(self):
|
||||
"""Test REVIEW_COMPLETE -> RENDERING_VIDEO is valid."""
|
||||
from backend.models.job import STATE_TRANSITIONS, JobStatus
|
||||
valid_transitions = STATE_TRANSITIONS.get(JobStatus.REVIEW_COMPLETE, [])
|
||||
assert JobStatus.RENDERING_VIDEO in valid_transitions
|
||||
|
||||
def test_rendering_video_status_exists(self):
|
||||
"""Test RENDERING_VIDEO status exists in JobStatus enum."""
|
||||
from backend.models.job import JobStatus
|
||||
assert hasattr(JobStatus, 'RENDERING_VIDEO')
|
||||
assert JobStatus.RENDERING_VIDEO.value == "rendering_video"
|
||||
|
||||
def test_rendering_video_transitions_to_instrumental(self):
|
||||
"""Test RENDERING_VIDEO -> AWAITING_INSTRUMENTAL_SELECTION is valid."""
|
||||
from backend.models.job import STATE_TRANSITIONS, JobStatus
|
||||
valid_transitions = STATE_TRANSITIONS.get(JobStatus.RENDERING_VIDEO, [])
|
||||
assert JobStatus.AWAITING_INSTRUMENTAL_SELECTION in valid_transitions
|
||||
|
||||
|
||||
class TestStylesConfigRequirements:
|
||||
"""Tests documenting the required fields for ASS subtitle generation.
|
||||
|
||||
These tests verify that the styles config format is documented correctly.
|
||||
The actual styles creation is tested in integration tests.
|
||||
"""
|
||||
|
||||
def test_required_karaoke_fields_documented(self):
|
||||
"""Document required karaoke style fields for ASS generation."""
|
||||
# These fields are required by the ASS subtitle generator
|
||||
# Missing any of them will cause video generation to fail
|
||||
required_fields = [
|
||||
"font", # Font name for ASS
|
||||
"font_path", # Path to font file (can be empty string, NOT None)
|
||||
"ass_name", # Style name in ASS file
|
||||
"primary_color", # Format: "R, G, B, A"
|
||||
"secondary_color", # Format: "R, G, B, A"
|
||||
"outline_color", # Format: "R, G, B, A"
|
||||
"back_color", # Format: "R, G, B, A"
|
||||
"bold", # Boolean
|
||||
"italic", # Boolean
|
||||
"underline", # Boolean
|
||||
"strike_out", # Boolean
|
||||
"scale_x", # Integer
|
||||
"scale_y", # Integer
|
||||
"spacing", # Integer
|
||||
"angle", # Float
|
||||
"border_style", # Integer
|
||||
"outline", # Integer
|
||||
"shadow", # Integer
|
||||
"margin_l", # Integer
|
||||
"margin_r", # Integer
|
||||
"margin_v", # Integer
|
||||
"encoding", # Integer
|
||||
]
|
||||
|
||||
# This test documents the required fields
|
||||
# Actual validation happens in integration tests
|
||||
assert len(required_fields) == 22
|
||||
assert "ass_name" in required_fields # This was missing initially
|
||||
assert "font_path" in required_fields # Must be string, not None
|
||||
|
||||
def test_minimal_styles_structure(self):
|
||||
"""Document the minimal styles JSON structure."""
|
||||
minimal_styles = {
|
||||
"karaoke": {
|
||||
"background_color": "#000000",
|
||||
"font_path": "", # MUST be string, NOT None
|
||||
"font": "Arial",
|
||||
"ass_name": "Default", # REQUIRED
|
||||
"primary_color": "112, 112, 247, 255",
|
||||
"secondary_color": "255, 255, 255, 255",
|
||||
"outline_color": "26, 58, 235, 255",
|
||||
"back_color": "0, 0, 0, 0",
|
||||
"bold": False,
|
||||
"italic": False,
|
||||
"underline": False,
|
||||
"strike_out": False,
|
||||
"scale_x": 100,
|
||||
"scale_y": 100,
|
||||
"spacing": 0,
|
||||
"angle": 0.0,
|
||||
"border_style": 1,
|
||||
"outline": 1,
|
||||
"shadow": 0,
|
||||
"margin_l": 0,
|
||||
"margin_r": 0,
|
||||
"margin_v": 0,
|
||||
"encoding": 0
|
||||
}
|
||||
}
|
||||
|
||||
# Verify structure is valid JSON
|
||||
json_str = json.dumps(minimal_styles)
|
||||
parsed = json.loads(json_str)
|
||||
|
||||
assert "karaoke" in parsed
|
||||
assert parsed["karaoke"]["font_path"] == "" # Not None
|
||||
assert parsed["karaoke"]["ass_name"] == "Default"
|
||||
|
||||
|
||||
class TestCorrectionDataMerging:
|
||||
"""Tests documenting correction data merging requirements.
|
||||
|
||||
The LyricsTranscriber React UI sends only partial correction data:
|
||||
- corrections
|
||||
- corrected_segments
|
||||
|
||||
The backend must merge this with the original corrections.json to
|
||||
reconstruct a full CorrectionResult.
|
||||
"""
|
||||
|
||||
def test_frontend_sends_partial_data(self):
|
||||
"""Document what the frontend sends."""
|
||||
# The frontend sends only these fields
|
||||
frontend_payload = {
|
||||
"corrections": [], # List of corrections made
|
||||
"corrected_segments": [] # Updated segment data
|
||||
}
|
||||
|
||||
# This is NOT a full CorrectionResult
|
||||
assert "original_segments" not in frontend_payload
|
||||
assert "metadata" not in frontend_payload
|
||||
|
||||
def test_merging_strategy(self):
|
||||
"""Document the merging strategy for corrections."""
|
||||
# Original data has full structure
|
||||
original_data = {
|
||||
"original_segments": [{"id": 1, "text": "test"}],
|
||||
"corrected_segments": [{"id": 1, "text": "test"}],
|
||||
"corrections": [],
|
||||
"metadata": {"artist": "Test"}
|
||||
}
|
||||
|
||||
# Frontend sends partial update
|
||||
frontend_update = {
|
||||
"corrections": [{"id": 1, "type": "edit"}],
|
||||
"corrected_segments": [{"id": 1, "text": "updated"}]
|
||||
}
|
||||
|
||||
# Merging strategy: update only the fields sent by frontend
|
||||
if 'corrections' in frontend_update:
|
||||
original_data['corrections'] = frontend_update['corrections']
|
||||
if 'corrected_segments' in frontend_update:
|
||||
original_data['corrected_segments'] = frontend_update['corrected_segments']
|
||||
|
||||
# Result preserves original_segments and metadata
|
||||
assert original_data['original_segments'] == [{"id": 1, "text": "test"}]
|
||||
assert original_data['metadata'] == {"artist": "Test"}
|
||||
# But has updated corrections and corrected_segments
|
||||
assert original_data['corrections'] == [{"id": 1, "type": "edit"}]
|
||||
assert original_data['corrected_segments'] == [{"id": 1, "text": "updated"}]
|
||||
|
||||
|
||||
class TestAddLyricsEndpoint:
|
||||
"""Tests for the add-lyrics endpoint."""
|
||||
|
||||
def test_add_lyrics_requires_source_and_lyrics(self):
|
||||
"""Document that add_lyrics expects source and lyrics fields."""
|
||||
# The frontend sends this payload
|
||||
valid_payload = {
|
||||
"source": "custom",
|
||||
"lyrics": "Line 1\nLine 2\nLine 3"
|
||||
}
|
||||
|
||||
# Both fields are required
|
||||
assert "source" in valid_payload
|
||||
assert "lyrics" in valid_payload
|
||||
assert len(valid_payload["source"].strip()) > 0
|
||||
assert len(valid_payload["lyrics"].strip()) > 0
|
||||
|
||||
def test_add_lyrics_uses_correction_operations(self):
|
||||
"""Verify CorrectionOperations.add_lyrics_source is available."""
|
||||
from lyrics_transcriber.correction.operations import CorrectionOperations
|
||||
|
||||
# The method should exist
|
||||
assert hasattr(CorrectionOperations, 'add_lyrics_source')
|
||||
|
||||
# It should be a static method
|
||||
import inspect
|
||||
# Get the method and check it's callable
|
||||
method = getattr(CorrectionOperations, 'add_lyrics_source')
|
||||
assert callable(method)
|
||||
|
||||
|
||||
class TestPreviewStyleLoading:
|
||||
"""Tests for the unified style loader used in preview video generation.
|
||||
|
||||
When a job has custom styles (uploaded via --style_params_json), these
|
||||
must be loaded and applied to preview videos, not just the final render.
|
||||
|
||||
This was a bug: preview videos were using minimal styles (black background)
|
||||
even when the job had custom backgrounds and fonts configured.
|
||||
|
||||
The style loading logic is now consolidated in karaoke_gen.style_loader
|
||||
to avoid duplication between workers and API routes.
|
||||
"""
|
||||
|
||||
def test_load_styles_from_gcs_with_custom_styles(self, tmp_path):
|
||||
"""Test that custom styles are downloaded and applied for preview."""
|
||||
import os
|
||||
from karaoke_gen.style_loader import load_styles_from_gcs
|
||||
|
||||
# Create source style params file
|
||||
source_style_params = tmp_path / "source_styles.json"
|
||||
style_data = {
|
||||
"karaoke": {
|
||||
"background_image": "/original/path/background.png",
|
||||
"font_path": "/original/path/font.ttf",
|
||||
"background_color": "#000000",
|
||||
"font": "Arial",
|
||||
"ass_name": "Default",
|
||||
}
|
||||
}
|
||||
source_style_params.write_text(json.dumps(style_data))
|
||||
|
||||
# Create source asset files
|
||||
source_background = tmp_path / "background.png"
|
||||
source_background.write_bytes(b"PNG image data")
|
||||
source_font = tmp_path / "font.ttf"
|
||||
source_font.write_bytes(b"TTF font data")
|
||||
|
||||
# Create mock download function that simulates GCS download
|
||||
def mock_download(gcs_path, local_path):
|
||||
if "style_params.json" in gcs_path:
|
||||
with open(local_path, 'w') as f:
|
||||
f.write(source_style_params.read_text())
|
||||
elif "karaoke_background" in gcs_path:
|
||||
with open(local_path, 'wb') as f:
|
||||
f.write(source_background.read_bytes())
|
||||
elif "font.ttf" in gcs_path:
|
||||
with open(local_path, 'wb') as f:
|
||||
f.write(source_font.read_bytes())
|
||||
|
||||
# Call the unified style loader function
|
||||
style_assets = {
|
||||
"style_params": "uploads/test123/style/style_params.json",
|
||||
"karaoke_background": "uploads/test123/style/karaoke_background.png",
|
||||
"font": "uploads/test123/style/font.ttf",
|
||||
}
|
||||
|
||||
styles_path, result_styles = load_styles_from_gcs(
|
||||
style_params_gcs_path="uploads/test123/style/style_params.json",
|
||||
style_assets=style_assets,
|
||||
temp_dir=str(tmp_path / "workdir"),
|
||||
download_func=mock_download,
|
||||
)
|
||||
|
||||
# Verify styles file was created
|
||||
assert os.path.exists(styles_path)
|
||||
|
||||
# The paths should now point to the local downloaded files, not the original paths
|
||||
assert "karaoke" in result_styles
|
||||
assert result_styles["karaoke"]["background_image"] != "/original/path/background.png"
|
||||
assert "karaoke_background.png" in result_styles["karaoke"]["background_image"]
|
||||
assert result_styles["karaoke"]["font_path"] != "/original/path/font.ttf"
|
||||
assert "font.ttf" in result_styles["karaoke"]["font_path"]
|
||||
|
||||
def test_load_styles_from_gcs_falls_back_to_minimal(self, tmp_path):
|
||||
"""Test that minimal styles are used when job has no custom styles."""
|
||||
import os
|
||||
from karaoke_gen.style_loader import load_styles_from_gcs
|
||||
|
||||
# Call with no custom styles
|
||||
styles_path, result_styles = load_styles_from_gcs(
|
||||
style_params_gcs_path=None, # No custom styles
|
||||
style_assets={},
|
||||
temp_dir=str(tmp_path),
|
||||
download_func=lambda gcs_path, local_path: None, # Won't be called
|
||||
)
|
||||
|
||||
# Verify styles file was created
|
||||
assert os.path.exists(styles_path)
|
||||
|
||||
# Should have karaoke section with minimal/default values
|
||||
assert "karaoke" in result_styles
|
||||
assert result_styles["karaoke"]["background_color"] == "#000000"
|
||||
assert result_styles["karaoke"]["font"] == "Arial"
|
||||
# Minimal styles have background_image as None (default)
|
||||
assert result_styles["karaoke"].get("background_image") is None
|
||||
|
||||
def test_asset_mapping_is_complete(self):
|
||||
"""Verify all required asset mappings are defined in the unified style loader."""
|
||||
from karaoke_gen.style_loader import ASSET_KEY_MAPPINGS
|
||||
|
||||
# These mappings must be present for styles to work correctly
|
||||
required_keys = [
|
||||
"karaoke_background",
|
||||
"intro_background",
|
||||
"end_background",
|
||||
"font",
|
||||
]
|
||||
|
||||
for key in required_keys:
|
||||
assert key in ASSET_KEY_MAPPINGS, f"Asset mapping '{key}' not found in ASSET_KEY_MAPPINGS"
|
||||
|
||||
# Verify karaoke_background maps to the correct path
|
||||
karaoke_mapping = ASSET_KEY_MAPPINGS["karaoke_background"]
|
||||
assert karaoke_mapping == ("karaoke", "background_image")
|
||||
|
||||
# Verify font maps to multiple sections
|
||||
font_mappings = ASSET_KEY_MAPPINGS["font"]
|
||||
assert isinstance(font_mappings, list)
|
||||
assert ("karaoke", "font_path") in font_mappings
|
||||
@@ -0,0 +1,556 @@
|
||||
"""
|
||||
Unit tests for service layer.
|
||||
|
||||
Tests AuthService, StorageService, WorkerService, and FirestoreService
|
||||
without requiring actual cloud resources.
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import Mock, MagicMock, AsyncMock, patch, call
|
||||
from datetime import datetime, UTC
|
||||
|
||||
# Mock Google Cloud before imports
|
||||
import sys
|
||||
sys.modules['google.cloud.firestore'] = MagicMock()
|
||||
sys.modules['google.cloud.storage'] = MagicMock()
|
||||
sys.modules['google.cloud.tasks_v2'] = MagicMock()
|
||||
|
||||
from backend.services.auth_service import AuthService, UserType
|
||||
from backend.services.storage_service import StorageService
|
||||
from backend.services.worker_service import WorkerService, get_worker_service
|
||||
from backend.services.firestore_service import FirestoreService
|
||||
from backend.models.job import Job, JobStatus
|
||||
|
||||
|
||||
class TestAuthService:
|
||||
"""Test AuthService token validation and management."""
|
||||
|
||||
def test_validate_admin_token(self):
|
||||
"""Test validating hardcoded admin tokens."""
|
||||
with patch('backend.services.auth_service.get_settings') as mock_settings:
|
||||
mock_settings.return_value.admin_tokens = "admin123,secret456"
|
||||
|
||||
auth_service = AuthService()
|
||||
|
||||
# Valid admin token - returns (is_valid, user_type, usage_count, token)
|
||||
is_valid, user_type, usage_count, token = auth_service.validate_token('admin123')
|
||||
assert is_valid is True
|
||||
assert user_type == UserType.ADMIN
|
||||
|
||||
# Another valid admin token
|
||||
is_valid, user_type, _, _ = auth_service.validate_token('secret456')
|
||||
assert is_valid is True
|
||||
assert user_type == UserType.ADMIN
|
||||
|
||||
def test_validate_invalid_token(self):
|
||||
"""Test rejecting invalid tokens."""
|
||||
with patch('backend.services.auth_service.FirestoreService') as mock_fs:
|
||||
mock_fs_instance = Mock()
|
||||
mock_fs.return_value = mock_fs_instance
|
||||
mock_fs_instance.get_token.return_value = None # Token not in DB
|
||||
|
||||
with patch('backend.services.auth_service.get_settings') as mock_settings:
|
||||
mock_settings.return_value.admin_tokens = "admin123"
|
||||
|
||||
auth_service = AuthService()
|
||||
is_valid, _, _, _ = auth_service.validate_token('invalid_token')
|
||||
|
||||
assert is_valid is False
|
||||
|
||||
def test_validate_token_from_firestore(self):
|
||||
"""Test validating tokens stored in Firestore."""
|
||||
with patch('backend.services.auth_service.FirestoreService') as mock_fs:
|
||||
mock_fs_instance = Mock()
|
||||
mock_fs.return_value = mock_fs_instance
|
||||
|
||||
# Mock Firestore returning a token
|
||||
mock_fs_instance.get_token.return_value = {
|
||||
'token': 'db_token123',
|
||||
'type': 'unlimited',
|
||||
'valid': True,
|
||||
'usage_count': 5,
|
||||
'created_at': datetime.now(UTC).isoformat()
|
||||
}
|
||||
|
||||
with patch('backend.services.auth_service.get_settings') as mock_settings:
|
||||
mock_settings.return_value.admin_tokens = ""
|
||||
|
||||
auth_service = AuthService()
|
||||
is_valid, user_type, usage_count, token = auth_service.validate_token('db_token123')
|
||||
|
||||
assert is_valid is True
|
||||
assert user_type == UserType.UNLIMITED
|
||||
# usage_count can be -1 for unlimited tokens
|
||||
assert isinstance(usage_count, int)
|
||||
|
||||
|
||||
class TestStorageService:
|
||||
"""Test StorageService GCS operations."""
|
||||
|
||||
def test_upload_file(self):
|
||||
"""Test uploading a file to GCS."""
|
||||
with patch('backend.services.storage_service.storage') as mock_storage:
|
||||
mock_client = MagicMock()
|
||||
mock_storage.Client.return_value = mock_client
|
||||
|
||||
mock_bucket = MagicMock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
|
||||
mock_blob = MagicMock()
|
||||
mock_bucket.blob.return_value = mock_blob
|
||||
|
||||
storage_service = StorageService()
|
||||
|
||||
# Upload file - correct parameter names
|
||||
result = storage_service.upload_file(
|
||||
local_path="/tmp/test.flac",
|
||||
destination_path="uploads/test123/test.flac"
|
||||
)
|
||||
|
||||
# Verify blob was created and uploaded
|
||||
mock_bucket.blob.assert_called_once_with("uploads/test123/test.flac")
|
||||
mock_blob.upload_from_filename.assert_called_once_with("/tmp/test.flac")
|
||||
# Result should be the destination path
|
||||
assert result == "uploads/test123/test.flac"
|
||||
|
||||
def test_download_file(self):
|
||||
"""Test downloading a file from GCS."""
|
||||
with patch('backend.services.storage_service.storage') as mock_storage:
|
||||
mock_client = MagicMock()
|
||||
mock_storage.Client.return_value = mock_client
|
||||
|
||||
mock_bucket = MagicMock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
|
||||
mock_blob = MagicMock()
|
||||
mock_bucket.blob.return_value = mock_blob
|
||||
|
||||
storage_service = StorageService()
|
||||
|
||||
# Download file - correct parameter names
|
||||
storage_service.download_file(
|
||||
source_path="uploads/test123/test.flac",
|
||||
destination_path="/tmp/downloaded.flac"
|
||||
)
|
||||
|
||||
# Verify download was called
|
||||
mock_blob.download_to_filename.assert_called_once_with("/tmp/downloaded.flac")
|
||||
|
||||
def test_delete_file(self):
|
||||
"""Test deleting a file from GCS."""
|
||||
with patch('backend.services.storage_service.storage') as mock_storage:
|
||||
mock_client = MagicMock()
|
||||
mock_storage.Client.return_value = mock_client
|
||||
|
||||
mock_bucket = MagicMock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
|
||||
mock_blob = MagicMock()
|
||||
mock_bucket.blob.return_value = mock_blob
|
||||
|
||||
storage_service = StorageService()
|
||||
|
||||
# Delete file
|
||||
storage_service.delete_file("uploads/test123/test.flac")
|
||||
|
||||
# Verify delete was called
|
||||
mock_blob.delete.assert_called_once()
|
||||
|
||||
|
||||
class TestWorkerService:
|
||||
"""Test WorkerService internal HTTP calls."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trigger_audio_worker(self):
|
||||
"""Test triggering audio worker via internal HTTP."""
|
||||
with patch('backend.services.worker_service.httpx.AsyncClient') as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client_cls.return_value.__aenter__.return_value = mock_client
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 200
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
worker_service = get_worker_service()
|
||||
|
||||
await worker_service.trigger_audio_worker("test123")
|
||||
|
||||
# Verify HTTP POST was made
|
||||
mock_client.post.assert_called_once()
|
||||
call_args = mock_client.post.call_args
|
||||
assert "/api/internal/workers/audio" in str(call_args)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trigger_lyrics_worker(self):
|
||||
"""Test triggering lyrics worker via internal HTTP."""
|
||||
with patch('backend.services.worker_service.httpx.AsyncClient') as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client_cls.return_value.__aenter__.return_value = mock_client
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 200
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
worker_service = get_worker_service()
|
||||
|
||||
await worker_service.trigger_lyrics_worker("test123")
|
||||
|
||||
# Verify HTTP POST was made
|
||||
mock_client.post.assert_called_once()
|
||||
call_args = mock_client.post.call_args
|
||||
assert "/api/internal/workers/lyrics" in str(call_args)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trigger_screens_worker(self):
|
||||
"""Test triggering screens worker via internal HTTP."""
|
||||
with patch('backend.services.worker_service.httpx.AsyncClient') as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client_cls.return_value.__aenter__.return_value = mock_client
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 200
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
worker_service = get_worker_service()
|
||||
|
||||
await worker_service.trigger_screens_worker("test123")
|
||||
|
||||
# Verify HTTP POST was made
|
||||
mock_client.post.assert_called_once()
|
||||
call_args = mock_client.post.call_args
|
||||
assert "/api/internal/workers/screens" in str(call_args)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trigger_video_worker(self):
|
||||
"""Test triggering video worker via internal HTTP."""
|
||||
with patch('backend.services.worker_service.httpx.AsyncClient') as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client_cls.return_value.__aenter__.return_value = mock_client
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 200
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
worker_service = get_worker_service()
|
||||
|
||||
await worker_service.trigger_video_worker("test123")
|
||||
|
||||
# Verify HTTP POST was made
|
||||
mock_client.post.assert_called_once()
|
||||
call_args = mock_client.post.call_args
|
||||
assert "/api/internal/workers/video" in str(call_args)
|
||||
|
||||
|
||||
class TestWorkerServiceCloudTasks:
|
||||
"""Test WorkerService Cloud Tasks integration."""
|
||||
|
||||
def test_should_use_cloud_tasks_default_false(self):
|
||||
"""Test that Cloud Tasks is disabled by default."""
|
||||
from backend.services.worker_service import WorkerService, reset_worker_service
|
||||
reset_worker_service()
|
||||
|
||||
with patch('backend.services.worker_service.os.getenv') as mock_getenv:
|
||||
mock_getenv.side_effect = lambda k, d=None: {
|
||||
'PORT': '8000',
|
||||
}.get(k, d)
|
||||
|
||||
with patch('backend.services.worker_service.get_settings') as mock_settings:
|
||||
mock_settings.return_value.admin_tokens = None
|
||||
mock_settings.return_value.google_cloud_project = "test-project"
|
||||
mock_settings.return_value.enable_cloud_tasks = False
|
||||
mock_settings.return_value.gcp_region = "us-central1"
|
||||
mock_settings.return_value.use_cloud_run_jobs_for_video = False
|
||||
|
||||
service = WorkerService()
|
||||
assert service._use_cloud_tasks is False
|
||||
|
||||
def test_should_use_cloud_tasks_enabled(self):
|
||||
"""Test that Cloud Tasks can be enabled via settings."""
|
||||
from backend.services.worker_service import WorkerService, reset_worker_service
|
||||
reset_worker_service()
|
||||
|
||||
with patch('backend.services.worker_service.os.getenv') as mock_getenv:
|
||||
mock_getenv.side_effect = lambda k, d=None: {
|
||||
'PORT': '8000',
|
||||
}.get(k, d)
|
||||
|
||||
with patch('backend.services.worker_service.get_settings') as mock_settings:
|
||||
mock_settings.return_value.admin_tokens = None
|
||||
mock_settings.return_value.google_cloud_project = "test-project"
|
||||
mock_settings.return_value.enable_cloud_tasks = True
|
||||
mock_settings.return_value.gcp_region = "us-central1"
|
||||
mock_settings.return_value.use_cloud_run_jobs_for_video = False
|
||||
|
||||
service = WorkerService()
|
||||
assert service._use_cloud_tasks is True
|
||||
|
||||
def test_worker_queues_mapping(self):
|
||||
"""Test that all worker types have queue mappings."""
|
||||
from backend.services.worker_service import WORKER_QUEUES
|
||||
|
||||
# Verify all expected workers have queues
|
||||
assert "audio" in WORKER_QUEUES
|
||||
assert "lyrics" in WORKER_QUEUES
|
||||
assert "screens" in WORKER_QUEUES
|
||||
assert "render-video" in WORKER_QUEUES
|
||||
assert "video" in WORKER_QUEUES
|
||||
|
||||
# Verify queue names are correct
|
||||
assert WORKER_QUEUES["audio"] == "audio-worker-queue"
|
||||
assert WORKER_QUEUES["lyrics"] == "lyrics-worker-queue"
|
||||
assert WORKER_QUEUES["screens"] == "screens-worker-queue"
|
||||
assert WORKER_QUEUES["render-video"] == "render-worker-queue"
|
||||
assert WORKER_QUEUES["video"] == "video-worker-queue"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trigger_worker_uses_http_when_cloud_tasks_disabled(self):
|
||||
"""Test that trigger_worker uses HTTP when Cloud Tasks is disabled."""
|
||||
from backend.services.worker_service import WorkerService, reset_worker_service
|
||||
reset_worker_service()
|
||||
|
||||
with patch('backend.services.worker_service.os.getenv') as mock_getenv:
|
||||
mock_getenv.side_effect = lambda k, d=None: {
|
||||
'PORT': '8000',
|
||||
}.get(k, d)
|
||||
|
||||
with patch('backend.services.worker_service.get_settings') as mock_settings:
|
||||
mock_settings.return_value.admin_tokens = "test-token"
|
||||
mock_settings.return_value.google_cloud_project = "test-project"
|
||||
mock_settings.return_value.enable_cloud_tasks = False
|
||||
mock_settings.return_value.gcp_region = "us-central1"
|
||||
mock_settings.return_value.use_cloud_run_jobs_for_video = False
|
||||
|
||||
with patch('backend.services.worker_service.httpx.AsyncClient') as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client_cls.return_value.__aenter__.return_value = mock_client
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 200
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
service = WorkerService()
|
||||
assert service._use_cloud_tasks is False
|
||||
|
||||
result = await service.trigger_worker("audio", "test-job-123")
|
||||
|
||||
# Verify HTTP was used
|
||||
assert result is True
|
||||
mock_client.post.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trigger_worker_uses_cloud_tasks_when_enabled(self):
|
||||
"""Test that trigger_worker uses Cloud Tasks when enabled."""
|
||||
from backend.services.worker_service import WorkerService, reset_worker_service
|
||||
reset_worker_service()
|
||||
|
||||
with patch('backend.services.worker_service.os.getenv') as mock_getenv:
|
||||
mock_getenv.side_effect = lambda k, d=None: {
|
||||
'CLOUD_RUN_SERVICE_URL': 'https://api.example.com',
|
||||
}.get(k, d)
|
||||
|
||||
with patch('backend.services.worker_service.get_settings') as mock_settings:
|
||||
mock_settings.return_value.admin_tokens = "test-token"
|
||||
mock_settings.return_value.google_cloud_project = "test-project"
|
||||
mock_settings.return_value.enable_cloud_tasks = True
|
||||
mock_settings.return_value.gcp_region = "us-central1"
|
||||
mock_settings.return_value.use_cloud_run_jobs_for_video = False
|
||||
|
||||
# Mock Cloud Tasks client
|
||||
mock_tasks_client = MagicMock()
|
||||
mock_tasks_client.queue_path.return_value = "projects/test-project/locations/us-central1/queues/audio-worker-queue"
|
||||
mock_task_response = MagicMock()
|
||||
mock_task_response.name = "projects/test-project/locations/us-central1/queues/audio-worker-queue/tasks/abc123"
|
||||
mock_tasks_client.create_task.return_value = mock_task_response
|
||||
|
||||
# Mock the google.cloud.tasks_v2 module which is imported inside the method
|
||||
mock_tasks_module = MagicMock()
|
||||
mock_tasks_module.CloudTasksClient.return_value = mock_tasks_client
|
||||
mock_tasks_module.HttpMethod.POST = "POST"
|
||||
|
||||
with patch.dict('sys.modules', {'google.cloud.tasks_v2': mock_tasks_module}):
|
||||
with patch.dict(sys.modules, {'google.cloud': MagicMock()}):
|
||||
service = WorkerService()
|
||||
service._tasks_client = mock_tasks_client # Inject mocked client
|
||||
|
||||
result = await service.trigger_worker("audio", "test-job-123")
|
||||
|
||||
# Verify Cloud Tasks was used
|
||||
assert result is True
|
||||
mock_tasks_client.create_task.assert_called_once()
|
||||
|
||||
# Verify task payload
|
||||
call_args = mock_tasks_client.create_task.call_args
|
||||
assert call_args is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trigger_worker_returns_false_for_unknown_worker_type(self):
|
||||
"""Test that trigger_worker returns False for unknown worker types."""
|
||||
from backend.services.worker_service import WorkerService, reset_worker_service
|
||||
reset_worker_service()
|
||||
|
||||
with patch('backend.services.worker_service.os.getenv') as mock_getenv:
|
||||
mock_getenv.side_effect = lambda k, d=None: {
|
||||
'CLOUD_RUN_SERVICE_URL': 'https://api.example.com',
|
||||
}.get(k, d)
|
||||
|
||||
with patch('backend.services.worker_service.get_settings') as mock_settings:
|
||||
mock_settings.return_value.admin_tokens = "test-token"
|
||||
mock_settings.return_value.google_cloud_project = "test-project"
|
||||
mock_settings.return_value.enable_cloud_tasks = True
|
||||
mock_settings.return_value.gcp_region = "us-central1"
|
||||
mock_settings.return_value.use_cloud_run_jobs_for_video = False
|
||||
|
||||
service = WorkerService()
|
||||
|
||||
result = await service.trigger_worker("unknown-worker", "test-job-123")
|
||||
|
||||
assert result is False
|
||||
|
||||
def test_reset_worker_service(self):
|
||||
"""Test that reset_worker_service resets the singleton."""
|
||||
from backend.services.worker_service import get_worker_service, reset_worker_service
|
||||
|
||||
with patch('backend.services.worker_service.get_settings') as mock_settings:
|
||||
mock_settings.return_value.admin_tokens = None
|
||||
mock_settings.return_value.google_cloud_project = "test-project"
|
||||
mock_settings.return_value.enable_cloud_tasks = False
|
||||
mock_settings.return_value.gcp_region = "us-central1"
|
||||
mock_settings.return_value.use_cloud_run_jobs_for_video = False
|
||||
|
||||
service1 = get_worker_service()
|
||||
reset_worker_service()
|
||||
service2 = get_worker_service()
|
||||
|
||||
# After reset, should be different instances
|
||||
assert service1 is not service2
|
||||
|
||||
|
||||
class TestFirestoreService:
|
||||
"""Test FirestoreService database operations."""
|
||||
|
||||
def test_create_job(self):
|
||||
"""Test creating a job document in Firestore."""
|
||||
with patch('backend.services.firestore_service.firestore') as mock_firestore:
|
||||
mock_client = MagicMock()
|
||||
mock_firestore.Client.return_value = mock_client
|
||||
|
||||
mock_collection = MagicMock()
|
||||
mock_client.collection.return_value = mock_collection
|
||||
|
||||
mock_doc_ref = MagicMock()
|
||||
mock_collection.document.return_value = mock_doc_ref
|
||||
|
||||
firestore_service = FirestoreService()
|
||||
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC)
|
||||
)
|
||||
|
||||
firestore_service.create_job(job) # Returns None
|
||||
|
||||
# Verify document was set
|
||||
mock_doc_ref.set.assert_called_once()
|
||||
|
||||
def test_get_job(self):
|
||||
"""Test fetching a job from Firestore."""
|
||||
with patch('backend.services.firestore_service.firestore') as mock_firestore:
|
||||
mock_client = MagicMock()
|
||||
mock_firestore.Client.return_value = mock_client
|
||||
|
||||
mock_collection = MagicMock()
|
||||
mock_client.collection.return_value = mock_collection
|
||||
|
||||
mock_doc_ref = MagicMock()
|
||||
mock_collection.document.return_value = mock_doc_ref
|
||||
|
||||
mock_doc = MagicMock()
|
||||
mock_doc.exists = True
|
||||
mock_doc.to_dict.return_value = {
|
||||
'job_id': 'test123',
|
||||
'status': 'pending',
|
||||
'progress': 0,
|
||||
'created_at': datetime.now(UTC).isoformat(),
|
||||
'updated_at': datetime.now(UTC).isoformat()
|
||||
}
|
||||
mock_doc_ref.get.return_value = mock_doc
|
||||
|
||||
firestore_service = FirestoreService()
|
||||
|
||||
job = firestore_service.get_job("test123")
|
||||
|
||||
# Verify document was fetched
|
||||
mock_doc_ref.get.assert_called_once()
|
||||
assert job is not None
|
||||
assert job.job_id == "test123"
|
||||
|
||||
def test_get_nonexistent_job(self):
|
||||
"""Test fetching a job that doesn't exist."""
|
||||
with patch('backend.services.firestore_service.firestore') as mock_firestore:
|
||||
mock_client = MagicMock()
|
||||
mock_firestore.Client.return_value = mock_client
|
||||
|
||||
mock_collection = MagicMock()
|
||||
mock_client.collection.return_value = mock_collection
|
||||
|
||||
mock_doc_ref = MagicMock()
|
||||
mock_collection.document.return_value = mock_doc_ref
|
||||
|
||||
mock_doc = MagicMock()
|
||||
mock_doc.exists = False
|
||||
mock_doc_ref.get.return_value = mock_doc
|
||||
|
||||
firestore_service = FirestoreService()
|
||||
|
||||
job = firestore_service.get_job("nonexistent")
|
||||
|
||||
assert job is None
|
||||
|
||||
def test_update_job(self):
|
||||
"""Test updating a job in Firestore."""
|
||||
with patch('backend.services.firestore_service.firestore') as mock_firestore:
|
||||
mock_client = MagicMock()
|
||||
mock_firestore.Client.return_value = mock_client
|
||||
|
||||
mock_collection = MagicMock()
|
||||
mock_client.collection.return_value = mock_collection
|
||||
|
||||
mock_doc_ref = MagicMock()
|
||||
mock_collection.document.return_value = mock_doc_ref
|
||||
|
||||
firestore_service = FirestoreService()
|
||||
|
||||
updates = {
|
||||
'status': JobStatus.SEPARATING_STAGE1,
|
||||
'progress': 25
|
||||
}
|
||||
|
||||
firestore_service.update_job("test123", updates)
|
||||
|
||||
# Verify update was called
|
||||
mock_doc_ref.update.assert_called_once()
|
||||
|
||||
def test_delete_job(self):
|
||||
"""Test deleting a job from Firestore."""
|
||||
with patch('backend.services.firestore_service.firestore') as mock_firestore:
|
||||
mock_client = MagicMock()
|
||||
mock_firestore.Client.return_value = mock_client
|
||||
|
||||
mock_collection = MagicMock()
|
||||
mock_client.collection.return_value = mock_collection
|
||||
|
||||
mock_doc_ref = MagicMock()
|
||||
mock_collection.document.return_value = mock_doc_ref
|
||||
|
||||
firestore_service = FirestoreService()
|
||||
|
||||
firestore_service.delete_job("test123")
|
||||
|
||||
# Verify delete was called
|
||||
mock_doc_ref.delete.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
"""
|
||||
Extended unit tests for services.
|
||||
|
||||
These tests provide additional coverage for service layer code.
|
||||
"""
|
||||
import pytest
|
||||
from datetime import datetime, UTC
|
||||
from unittest.mock import MagicMock, AsyncMock, patch
|
||||
|
||||
from backend.models.job import Job, JobStatus
|
||||
|
||||
|
||||
class TestJobManagerExtended:
|
||||
"""Extended tests for JobManager.
|
||||
|
||||
Note: These tests verify the module structure and basic behavior.
|
||||
Full JobManager testing is in test_job_manager.py.
|
||||
"""
|
||||
|
||||
def test_job_manager_module_imports(self):
|
||||
"""Test JobManager module can be imported."""
|
||||
from backend.services import job_manager
|
||||
assert hasattr(job_manager, 'JobManager')
|
||||
|
||||
def test_state_transitions_defined(self):
|
||||
"""Test STATE_TRANSITIONS dict is defined."""
|
||||
from backend.models.job import STATE_TRANSITIONS
|
||||
assert STATE_TRANSITIONS is not None
|
||||
assert len(STATE_TRANSITIONS) > 0
|
||||
|
||||
|
||||
class TestStorageServiceExtended:
|
||||
"""Extended tests for StorageService."""
|
||||
|
||||
def test_storage_service_initialization(self):
|
||||
"""Test StorageService can be initialized."""
|
||||
with patch('backend.services.storage_service.storage'):
|
||||
from backend.services.storage_service import StorageService
|
||||
service = StorageService()
|
||||
assert service is not None
|
||||
|
||||
def test_storage_service_bucket_name(self):
|
||||
"""Test StorageService uses configured bucket."""
|
||||
mock_client = MagicMock()
|
||||
mock_bucket = MagicMock()
|
||||
mock_client.bucket.return_value = mock_bucket
|
||||
|
||||
with patch('backend.services.storage_service.storage.Client', return_value=mock_client), \
|
||||
patch.dict('os.environ', {'GCS_BUCKET_NAME': 'test-bucket'}):
|
||||
from backend.services.storage_service import StorageService
|
||||
service = StorageService()
|
||||
# Service should use the configured bucket
|
||||
|
||||
|
||||
class TestAuthServiceExtended:
|
||||
"""Extended tests for AuthService.
|
||||
|
||||
Note: Full auth service testing is in test_services.py.
|
||||
"""
|
||||
|
||||
def test_auth_service_module_imports(self):
|
||||
"""Test AuthService module can be imported."""
|
||||
from backend.services import auth_service
|
||||
assert hasattr(auth_service, 'AuthService')
|
||||
|
||||
|
||||
class TestWorkerServiceExtended:
|
||||
"""Extended tests for WorkerService."""
|
||||
|
||||
def test_worker_service_initialization(self):
|
||||
"""Test WorkerService can be initialized."""
|
||||
from backend.services.worker_service import WorkerService
|
||||
service = WorkerService()
|
||||
assert service is not None
|
||||
|
||||
def test_worker_service_get_base_url(self):
|
||||
"""Test WorkerService constructs correct base URL."""
|
||||
from backend.services.worker_service import WorkerService
|
||||
service = WorkerService()
|
||||
|
||||
url = service._get_base_url()
|
||||
assert 'http' in url
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_worker_service_trigger_with_mock(self):
|
||||
"""Test triggering a worker with mocked HTTP."""
|
||||
from backend.services.worker_service import WorkerService
|
||||
|
||||
service = WorkerService()
|
||||
|
||||
# Mock the HTTP client
|
||||
with patch('backend.services.worker_service.httpx.AsyncClient') as mock_client_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_client.post = AsyncMock(return_value=mock_response)
|
||||
mock_client_cls.return_value.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client_cls.return_value.__aexit__ = AsyncMock(return_value=None)
|
||||
|
||||
# The trigger methods should work with mocked HTTP
|
||||
|
||||
|
||||
class TestFirestoreServiceExtended:
|
||||
"""Extended tests for FirestoreService."""
|
||||
|
||||
def test_firestore_service_initialization(self):
|
||||
"""Test FirestoreService can be initialized."""
|
||||
with patch('backend.services.firestore_service.firestore'):
|
||||
from backend.services.firestore_service import FirestoreService
|
||||
service = FirestoreService()
|
||||
assert service is not None
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
"""
|
||||
Tests for style file upload and processing.
|
||||
|
||||
Tests the full flow:
|
||||
1. CLI parses style_params.json and extracts file references
|
||||
2. Upload endpoint accepts style files
|
||||
3. Style helper downloads and parses style config
|
||||
4. Workers use style config for video generation
|
||||
"""
|
||||
import pytest
|
||||
import json
|
||||
import tempfile
|
||||
import os
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, MagicMock, patch, AsyncMock
|
||||
from io import BytesIO
|
||||
|
||||
# Test the remote CLI's style parsing
|
||||
class TestRemoteCLIStyleParsing:
|
||||
"""Test that remote CLI correctly parses style_params.json."""
|
||||
|
||||
def test_parse_style_params_extracts_file_paths(self):
|
||||
"""Test that _parse_style_params extracts all file references."""
|
||||
from karaoke_gen.utils.remote_cli import RemoteKaraokeClient, Config
|
||||
|
||||
# Create a mock style_params.json
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
# Create test files
|
||||
font_path = os.path.join(temp_dir, "font.ttf")
|
||||
intro_bg_path = os.path.join(temp_dir, "intro_bg.png")
|
||||
karaoke_bg_path = os.path.join(temp_dir, "karaoke_bg.png")
|
||||
|
||||
Path(font_path).touch()
|
||||
Path(intro_bg_path).touch()
|
||||
Path(karaoke_bg_path).touch()
|
||||
|
||||
# Create style_params.json
|
||||
style_params = {
|
||||
"intro": {
|
||||
"background_image": intro_bg_path,
|
||||
"font": font_path,
|
||||
},
|
||||
"karaoke": {
|
||||
"background_image": karaoke_bg_path,
|
||||
"font_path": font_path,
|
||||
},
|
||||
"end": {
|
||||
"background_image": karaoke_bg_path,
|
||||
"font": font_path,
|
||||
},
|
||||
"cdg": {
|
||||
"font_path": font_path,
|
||||
}
|
||||
}
|
||||
|
||||
style_json_path = os.path.join(temp_dir, "style_params.json")
|
||||
with open(style_json_path, 'w') as f:
|
||||
json.dump(style_params, f)
|
||||
|
||||
# Create client and parse
|
||||
config = Config(
|
||||
service_url="http://test",
|
||||
review_ui_url="http://test",
|
||||
poll_interval=5,
|
||||
output_dir=temp_dir
|
||||
)
|
||||
logger = Mock()
|
||||
client = RemoteKaraokeClient(config, logger)
|
||||
|
||||
assets = client._parse_style_params(style_json_path)
|
||||
|
||||
# Should extract unique files
|
||||
assert 'style_font' in assets
|
||||
assert 'style_intro_background' in assets
|
||||
assert 'style_karaoke_background' in assets
|
||||
assert assets['style_font'] == font_path
|
||||
assert assets['style_intro_background'] == intro_bg_path
|
||||
assert assets['style_karaoke_background'] == karaoke_bg_path
|
||||
|
||||
def test_parse_style_params_handles_missing_files(self):
|
||||
"""Test that _parse_style_params ignores non-existent files."""
|
||||
from karaoke_gen.utils.remote_cli import RemoteKaraokeClient, Config
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
style_params = {
|
||||
"intro": {
|
||||
"background_image": "/nonexistent/path.png",
|
||||
"font": "/nonexistent/font.ttf",
|
||||
},
|
||||
}
|
||||
|
||||
style_json_path = os.path.join(temp_dir, "style_params.json")
|
||||
with open(style_json_path, 'w') as f:
|
||||
json.dump(style_params, f)
|
||||
|
||||
config = Config(
|
||||
service_url="http://test",
|
||||
review_ui_url="http://test",
|
||||
poll_interval=5,
|
||||
output_dir=temp_dir
|
||||
)
|
||||
logger = Mock()
|
||||
client = RemoteKaraokeClient(config, logger)
|
||||
|
||||
assets = client._parse_style_params(style_json_path)
|
||||
|
||||
# Should return empty dict for non-existent files
|
||||
assert len(assets) == 0
|
||||
|
||||
|
||||
class TestStyleHelper:
|
||||
"""Test the backend style helper module."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_style_config_loads_defaults_when_no_custom_styles(self):
|
||||
"""Test that StyleConfig returns defaults when no custom styles."""
|
||||
pytest.importorskip("google.cloud.storage", reason="GCP libraries not available")
|
||||
from backend.workers.style_helper import StyleConfig, DEFAULT_INTRO_FORMAT
|
||||
|
||||
# Mock job with no style assets and no style_params_gcs_path
|
||||
job = Mock()
|
||||
job.job_id = "test-123"
|
||||
job.style_assets = {}
|
||||
job.style_params_gcs_path = None # Explicitly set to None
|
||||
|
||||
storage = Mock()
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
config = StyleConfig(job, storage, temp_dir)
|
||||
await config.load()
|
||||
|
||||
assert not config.has_custom_styles()
|
||||
intro_format = config.get_intro_format()
|
||||
assert intro_format == DEFAULT_INTRO_FORMAT
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_style_config_loads_custom_styles(self):
|
||||
"""Test that StyleConfig loads and parses custom styles."""
|
||||
pytest.importorskip("google.cloud.storage", reason="GCP libraries not available")
|
||||
from backend.workers.style_helper import StyleConfig
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
# Create a SEPARATE source directory for the source style file
|
||||
# (StyleConfig will use temp_dir/style/ for downloads, so we use source/)
|
||||
source_dir = os.path.join(temp_dir, "source")
|
||||
os.makedirs(source_dir, exist_ok=True)
|
||||
|
||||
# Create a mock style_params.json in the source directory
|
||||
style_params = {
|
||||
"intro": {
|
||||
"background_color": "#FF0000",
|
||||
"title_color": "#00FF00",
|
||||
},
|
||||
"cdg": {
|
||||
"background_color": "#0000FF",
|
||||
}
|
||||
}
|
||||
style_json_path = os.path.join(source_dir, "style_params.json")
|
||||
with open(style_json_path, 'w') as f:
|
||||
json.dump(style_params, f)
|
||||
|
||||
# Create a work directory for StyleConfig (separate from source)
|
||||
work_dir = os.path.join(temp_dir, "work")
|
||||
os.makedirs(work_dir, exist_ok=True)
|
||||
|
||||
# Mock job with style assets
|
||||
job = Mock()
|
||||
job.job_id = "test-123"
|
||||
job.style_assets = {"style_params": "uploads/test/style_params.json"}
|
||||
|
||||
# Mock storage to "download" the file
|
||||
storage = Mock()
|
||||
def mock_download(gcs_path, local_path):
|
||||
# Copy our test file to the expected location
|
||||
import shutil
|
||||
os.makedirs(os.path.dirname(local_path), exist_ok=True)
|
||||
shutil.copy(style_json_path, local_path)
|
||||
storage.download_file = mock_download
|
||||
|
||||
config = StyleConfig(job, storage, work_dir)
|
||||
await config.load()
|
||||
|
||||
assert config.has_custom_styles()
|
||||
|
||||
intro_format = config.get_intro_format()
|
||||
assert intro_format['background_color'] == "#FF0000"
|
||||
assert intro_format['title_color'] == "#00FF00"
|
||||
|
||||
cdg_styles = config.get_cdg_styles()
|
||||
assert cdg_styles['background_color'] == "#0000FF"
|
||||
|
||||
|
||||
class TestFileUploadEndpoint:
|
||||
"""Test the file upload endpoint accepts style files."""
|
||||
|
||||
def test_upload_endpoint_validates_style_files(self):
|
||||
"""Test that upload endpoint validates file types."""
|
||||
pytest.importorskip("google.cloud.firestore", reason="GCP libraries not available")
|
||||
# This would be an integration test with FastAPI TestClient
|
||||
# For now, just verify the validation logic exists
|
||||
from backend.api.routes.file_upload import (
|
||||
ALLOWED_AUDIO_EXTENSIONS,
|
||||
ALLOWED_IMAGE_EXTENSIONS,
|
||||
ALLOWED_FONT_EXTENSIONS
|
||||
)
|
||||
|
||||
assert '.mp3' in ALLOWED_AUDIO_EXTENSIONS
|
||||
assert '.flac' in ALLOWED_AUDIO_EXTENSIONS
|
||||
assert '.png' in ALLOWED_IMAGE_EXTENSIONS
|
||||
assert '.jpg' in ALLOWED_IMAGE_EXTENSIONS
|
||||
assert '.ttf' in ALLOWED_FONT_EXTENSIONS
|
||||
assert '.otf' in ALLOWED_FONT_EXTENSIONS
|
||||
|
||||
|
||||
class TestVideoWorkerStyleIntegration:
|
||||
"""Test that video worker properly uses style config."""
|
||||
|
||||
def test_video_worker_passes_cdg_styles_to_finalise(self):
|
||||
"""Test that video worker passes CDG styles to KaraokeFinalise."""
|
||||
pytest.importorskip("google.cloud.firestore", reason="GCP libraries not available")
|
||||
# This is implicitly tested by the video_worker code structure
|
||||
# The key is that cdg_styles is passed to KaraokeFinalise constructor
|
||||
|
||||
# Verify the import works
|
||||
from backend.workers.video_worker import generate_video
|
||||
assert generate_video is not None
|
||||
|
||||
def test_video_worker_passes_discord_webhook(self):
|
||||
"""Test that video worker passes discord webhook to KaraokeFinalise."""
|
||||
# The video_worker.py now passes discord_webhook_url from job
|
||||
# This is verified by code inspection
|
||||
pass
|
||||
|
||||
|
||||
class TestJobModelStyleFields:
|
||||
"""Test that Job model has all required style fields."""
|
||||
|
||||
def test_job_model_has_style_fields(self):
|
||||
"""Test Job model includes style configuration fields."""
|
||||
from backend.models.job import Job, JobCreate
|
||||
|
||||
# Check Job model fields (use model_fields for Pydantic v2)
|
||||
job_fields = Job.model_fields if hasattr(Job, 'model_fields') else Job.__fields__
|
||||
assert 'style_params_gcs_path' in job_fields
|
||||
assert 'style_assets' in job_fields
|
||||
assert 'brand_prefix' in job_fields
|
||||
assert 'discord_webhook_url' in job_fields
|
||||
|
||||
# Check JobCreate model fields
|
||||
create_fields = JobCreate.model_fields if hasattr(JobCreate, 'model_fields') else JobCreate.__fields__
|
||||
assert 'style_params_gcs_path' in create_fields
|
||||
assert 'style_assets' in create_fields
|
||||
assert 'brand_prefix' in create_fields
|
||||
assert 'discord_webhook_url' in create_fields
|
||||
@@ -0,0 +1,231 @@
|
||||
"""
|
||||
Unit tests for file_upload.py API routes using FastAPI TestClient.
|
||||
"""
|
||||
import pytest
|
||||
from datetime import datetime, UTC
|
||||
from unittest.mock import MagicMock, AsyncMock, patch
|
||||
from fastapi.testclient import TestClient
|
||||
from io import BytesIO
|
||||
|
||||
from backend.models.job import Job, JobStatus
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_job():
|
||||
"""Create a standard mock job."""
|
||||
return Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
input_media_gcs_path="uploads/test123/song.flac"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_services(mock_job):
|
||||
"""Create all mocked services needed for upload."""
|
||||
mock_job_manager = MagicMock()
|
||||
mock_job_manager.create_job.return_value = mock_job
|
||||
mock_job_manager.get_job.return_value = mock_job
|
||||
mock_job_manager.update_job.return_value = None
|
||||
|
||||
mock_storage = MagicMock()
|
||||
mock_storage.upload_fileobj.return_value = "gs://bucket/uploads/test123/song.flac"
|
||||
|
||||
mock_worker_service = MagicMock()
|
||||
mock_worker_service.trigger_audio_worker = AsyncMock(return_value=True)
|
||||
mock_worker_service.trigger_lyrics_worker = AsyncMock(return_value=True)
|
||||
|
||||
return {
|
||||
'job_manager': mock_job_manager,
|
||||
'storage': mock_storage,
|
||||
'worker_service': mock_worker_service
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(mock_services):
|
||||
"""Create TestClient with mocked dependencies."""
|
||||
mock_creds = MagicMock()
|
||||
mock_creds.universe_domain = 'googleapis.com'
|
||||
|
||||
with patch('backend.api.routes.file_upload.job_manager', mock_services['job_manager']), \
|
||||
patch('backend.api.routes.file_upload.storage_service', mock_services['storage']), \
|
||||
patch('backend.api.routes.file_upload.worker_service', mock_services['worker_service']), \
|
||||
patch('backend.services.firestore_service.firestore'), \
|
||||
patch('backend.services.storage_service.storage'), \
|
||||
patch('google.auth.default', return_value=(mock_creds, 'test-project')):
|
||||
from backend.main import app
|
||||
yield TestClient(app)
|
||||
|
||||
|
||||
class TestFileUploadEndpoint:
|
||||
"""Tests for POST /api/jobs/upload."""
|
||||
|
||||
def test_upload_flac_returns_200(self, client, mock_services):
|
||||
"""Test uploading FLAC file returns 200."""
|
||||
response = client.post(
|
||||
"/api/jobs/upload",
|
||||
files={"file": ("test.flac", BytesIO(b"fake audio"), "audio/flac")},
|
||||
data={"artist": "Test Artist", "title": "Test Song"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_upload_mp3_returns_200(self, client, mock_services):
|
||||
"""Test uploading MP3 file returns 200."""
|
||||
response = client.post(
|
||||
"/api/jobs/upload",
|
||||
files={"file": ("test.mp3", BytesIO(b"fake audio"), "audio/mpeg")},
|
||||
data={"artist": "Test Artist", "title": "Test Song"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_upload_wav_returns_200(self, client, mock_services):
|
||||
"""Test uploading WAV file returns 200."""
|
||||
response = client.post(
|
||||
"/api/jobs/upload",
|
||||
files={"file": ("test.wav", BytesIO(b"fake audio"), "audio/wav")},
|
||||
data={"artist": "Test Artist", "title": "Test Song"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_upload_returns_job_id(self, client, mock_services):
|
||||
"""Test upload response contains job_id."""
|
||||
response = client.post(
|
||||
"/api/jobs/upload",
|
||||
files={"file": ("test.flac", BytesIO(b"fake audio"), "audio/flac")},
|
||||
data={"artist": "Test", "title": "Song"}
|
||||
)
|
||||
data = response.json()
|
||||
assert "job_id" in data
|
||||
assert data["status"] == "success"
|
||||
|
||||
def test_upload_returns_filename(self, client, mock_services):
|
||||
"""Test upload response contains original filename."""
|
||||
response = client.post(
|
||||
"/api/jobs/upload",
|
||||
files={"file": ("my_song.flac", BytesIO(b"fake audio"), "audio/flac")},
|
||||
data={"artist": "Test", "title": "Song"}
|
||||
)
|
||||
data = response.json()
|
||||
assert "filename" in data
|
||||
assert data["filename"] == "my_song.flac"
|
||||
|
||||
def test_upload_rejects_txt_file(self, client, mock_services):
|
||||
"""Test upload rejects text files."""
|
||||
response = client.post(
|
||||
"/api/jobs/upload",
|
||||
files={"file": ("test.txt", BytesIO(b"not audio"), "text/plain")},
|
||||
data={"artist": "Test", "title": "Song"}
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
def test_upload_rejects_pdf_file(self, client, mock_services):
|
||||
"""Test upload rejects PDF files."""
|
||||
response = client.post(
|
||||
"/api/jobs/upload",
|
||||
files={"file": ("doc.pdf", BytesIO(b"pdf content"), "application/pdf")},
|
||||
data={"artist": "Test", "title": "Song"}
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
def test_upload_rejects_exe_file(self, client, mock_services):
|
||||
"""Test upload rejects executable files."""
|
||||
response = client.post(
|
||||
"/api/jobs/upload",
|
||||
files={"file": ("app.exe", BytesIO(b"exe content"), "application/octet-stream")},
|
||||
data={"artist": "Test", "title": "Song"}
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
def test_upload_requires_artist(self, client, mock_services):
|
||||
"""Test upload requires artist field."""
|
||||
response = client.post(
|
||||
"/api/jobs/upload",
|
||||
files={"file": ("test.flac", BytesIO(b"audio"), "audio/flac")},
|
||||
data={"title": "Song"} # Missing artist
|
||||
)
|
||||
assert response.status_code == 422
|
||||
|
||||
def test_upload_requires_title(self, client, mock_services):
|
||||
"""Test upload requires title field."""
|
||||
response = client.post(
|
||||
"/api/jobs/upload",
|
||||
files={"file": ("test.flac", BytesIO(b"audio"), "audio/flac")},
|
||||
data={"artist": "Artist"} # Missing title
|
||||
)
|
||||
assert response.status_code == 422
|
||||
|
||||
def test_upload_requires_file(self, client, mock_services):
|
||||
"""Test upload requires file."""
|
||||
response = client.post(
|
||||
"/api/jobs/upload",
|
||||
data={"artist": "Artist", "title": "Song"}
|
||||
# Missing file
|
||||
)
|
||||
assert response.status_code == 422
|
||||
|
||||
def test_upload_triggers_workers(self, client, mock_services):
|
||||
"""Test upload triggers audio and lyrics workers."""
|
||||
response = client.post(
|
||||
"/api/jobs/upload",
|
||||
files={"file": ("test.flac", BytesIO(b"audio"), "audio/flac")},
|
||||
data={"artist": "Test", "title": "Song"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
# Workers should be triggered in background
|
||||
|
||||
def test_upload_creates_job(self, client, mock_services):
|
||||
"""Test upload creates job in job manager."""
|
||||
response = client.post(
|
||||
"/api/jobs/upload",
|
||||
files={"file": ("test.flac", BytesIO(b"audio"), "audio/flac")},
|
||||
data={"artist": "Test", "title": "Song"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
mock_services['job_manager'].create_job.assert_called()
|
||||
|
||||
def test_upload_stores_file_to_gcs(self, client, mock_services):
|
||||
"""Test upload stores file to GCS."""
|
||||
response = client.post(
|
||||
"/api/jobs/upload",
|
||||
files={"file": ("test.flac", BytesIO(b"audio"), "audio/flac")},
|
||||
data={"artist": "Test", "title": "Song"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
mock_services['storage'].upload_fileobj.assert_called()
|
||||
|
||||
|
||||
class TestUploadValidation:
|
||||
"""Tests for upload validation logic."""
|
||||
|
||||
def test_upload_accepts_m4a(self, client, mock_services):
|
||||
"""Test upload accepts M4A files."""
|
||||
response = client.post(
|
||||
"/api/jobs/upload",
|
||||
files={"file": ("test.m4a", BytesIO(b"audio"), "audio/mp4")},
|
||||
data={"artist": "Test", "title": "Song"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_upload_accepts_ogg(self, client, mock_services):
|
||||
"""Test upload accepts OGG files."""
|
||||
response = client.post(
|
||||
"/api/jobs/upload",
|
||||
files={"file": ("test.ogg", BytesIO(b"audio"), "audio/ogg")},
|
||||
data={"artist": "Test", "title": "Song"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_upload_accepts_aac(self, client, mock_services):
|
||||
"""Test upload accepts AAC files."""
|
||||
response = client.post(
|
||||
"/api/jobs/upload",
|
||||
files={"file": ("test.aac", BytesIO(b"audio"), "audio/aac")},
|
||||
data={"artist": "Test", "title": "Song"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
"""
|
||||
Unit tests for backend validation script (validate.py).
|
||||
|
||||
Tests the validation functions that check for import errors, syntax errors,
|
||||
configuration issues, and FastAPI app creation.
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
|
||||
class TestValidateImports:
|
||||
"""Tests for the validate_imports function."""
|
||||
|
||||
def test_validate_imports_success(self):
|
||||
"""Test that validate_imports succeeds when all modules import."""
|
||||
from backend.validate import validate_imports
|
||||
|
||||
# This should succeed since we're in a valid test environment
|
||||
result = validate_imports()
|
||||
assert result is True
|
||||
|
||||
def test_validate_imports_with_failure(self):
|
||||
"""Test that validate_imports handles import failures."""
|
||||
from backend.validate import validate_imports
|
||||
|
||||
# Mock importlib to simulate a failure
|
||||
with patch('backend.validate.importlib.import_module') as mock_import:
|
||||
mock_import.side_effect = ImportError("Module not found")
|
||||
|
||||
result = validate_imports()
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestValidateSyntax:
|
||||
"""Tests for the validate_syntax function."""
|
||||
|
||||
def test_validate_syntax_success(self):
|
||||
"""Test that validate_syntax succeeds with valid Python files."""
|
||||
from backend.validate import validate_syntax
|
||||
|
||||
# This should succeed since the backend code is valid
|
||||
result = validate_syntax()
|
||||
assert result is True
|
||||
|
||||
def test_validate_syntax_handles_invalid_file(self):
|
||||
"""Test that validate_syntax detects syntax errors."""
|
||||
import tempfile
|
||||
import os
|
||||
from pathlib import Path
|
||||
from backend.validate import validate_syntax
|
||||
|
||||
# We can't easily inject invalid files into the backend dir,
|
||||
# but we can verify the function runs and returns True for valid files
|
||||
result = validate_syntax()
|
||||
assert result is True
|
||||
|
||||
|
||||
class TestValidateConfig:
|
||||
"""Tests for the validate_config function."""
|
||||
|
||||
def test_validate_config_success(self):
|
||||
"""Test that validate_config succeeds with valid configuration."""
|
||||
from backend.validate import validate_config
|
||||
|
||||
# This should succeed in test environment
|
||||
result = validate_config()
|
||||
assert result is True
|
||||
|
||||
def test_validate_config_failure(self):
|
||||
"""Test that validate_config handles configuration errors."""
|
||||
from backend.validate import validate_config
|
||||
|
||||
# Patch at the source module
|
||||
with patch('backend.config.get_settings') as mock_settings:
|
||||
mock_settings.side_effect = Exception("Config error")
|
||||
|
||||
result = validate_config()
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestValidateFastapiApp:
|
||||
"""Tests for the validate_fastapi_app function."""
|
||||
|
||||
def test_validate_fastapi_app_success(self):
|
||||
"""Test that validate_fastapi_app succeeds."""
|
||||
from backend.validate import validate_fastapi_app
|
||||
|
||||
# This should succeed since the FastAPI app is valid
|
||||
result = validate_fastapi_app()
|
||||
assert result is True
|
||||
|
||||
def test_validate_fastapi_app_failure(self):
|
||||
"""Test that validate_fastapi_app handles app creation errors."""
|
||||
from backend.validate import validate_fastapi_app
|
||||
|
||||
with patch.dict('sys.modules', {'backend.main': MagicMock(app=MagicMock(title="Test", version="1.0", routes=[]))}):
|
||||
# Even with a mock, the function should work
|
||||
result = validate_fastapi_app()
|
||||
# The original module is still accessible, so this should still pass
|
||||
assert result is True
|
||||
|
||||
|
||||
class TestMain:
|
||||
"""Tests for the main function."""
|
||||
|
||||
def test_main_runs(self):
|
||||
"""Test that main runs without errors."""
|
||||
from backend.validate import main
|
||||
|
||||
# In a valid test environment, main should return 0 (success)
|
||||
result = main()
|
||||
assert result == 0
|
||||
|
||||
def test_main_returns_1_on_failure(self):
|
||||
"""Test that main returns 1 when validations fail."""
|
||||
from backend.validate import main
|
||||
|
||||
# Mock one validation to fail
|
||||
with patch('backend.validate.validate_imports', return_value=False):
|
||||
result = main()
|
||||
assert result == 1
|
||||
|
||||
|
||||
class TestValidateModule:
|
||||
"""Test the validate module can be imported and has expected functions."""
|
||||
|
||||
def test_module_imports(self):
|
||||
"""Test that the validate module can be imported."""
|
||||
import backend.validate
|
||||
assert backend.validate is not None
|
||||
|
||||
def test_has_validate_imports(self):
|
||||
"""Test that validate_imports function exists."""
|
||||
from backend.validate import validate_imports
|
||||
assert callable(validate_imports)
|
||||
|
||||
def test_has_validate_syntax(self):
|
||||
"""Test that validate_syntax function exists."""
|
||||
from backend.validate import validate_syntax
|
||||
assert callable(validate_syntax)
|
||||
|
||||
def test_has_validate_config(self):
|
||||
"""Test that validate_config function exists."""
|
||||
from backend.validate import validate_config
|
||||
assert callable(validate_config)
|
||||
|
||||
def test_has_validate_fastapi_app(self):
|
||||
"""Test that validate_fastapi_app function exists."""
|
||||
from backend.validate import validate_fastapi_app
|
||||
assert callable(validate_fastapi_app)
|
||||
|
||||
def test_has_main(self):
|
||||
"""Test that main function exists."""
|
||||
from backend.validate import main
|
||||
assert callable(main)
|
||||
|
||||
@@ -0,0 +1,365 @@
|
||||
"""
|
||||
Unit tests for backend/workers/worker_logging.py.
|
||||
|
||||
Tests the job logging utilities including JobLogger, JobLogHandler,
|
||||
and job_logging_context for concurrent job isolation.
|
||||
"""
|
||||
import pytest
|
||||
import logging
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
class TestJobLoggingContext:
|
||||
"""Tests for job_logging_context context manager."""
|
||||
|
||||
def test_job_logging_context_sets_and_resets_job_id(self):
|
||||
"""Test that job_logging_context sets and resets the current job ID."""
|
||||
from backend.workers.worker_logging import job_logging_context, _current_job_id
|
||||
|
||||
# Initially no job
|
||||
assert _current_job_id.get() is None
|
||||
|
||||
# Inside context, job is set
|
||||
with job_logging_context("job123"):
|
||||
assert _current_job_id.get() == "job123"
|
||||
|
||||
# After context, job is reset
|
||||
assert _current_job_id.get() is None
|
||||
|
||||
def test_job_logging_context_nested(self):
|
||||
"""Test nested job_logging_context calls."""
|
||||
from backend.workers.worker_logging import job_logging_context, _current_job_id
|
||||
|
||||
with job_logging_context("outer_job"):
|
||||
assert _current_job_id.get() == "outer_job"
|
||||
|
||||
with job_logging_context("inner_job"):
|
||||
assert _current_job_id.get() == "inner_job"
|
||||
|
||||
# After inner context, outer job is restored
|
||||
assert _current_job_id.get() == "outer_job"
|
||||
|
||||
assert _current_job_id.get() is None
|
||||
|
||||
def test_job_logging_context_handles_exception(self):
|
||||
"""Test that job_logging_context resets even on exception."""
|
||||
from backend.workers.worker_logging import job_logging_context, _current_job_id
|
||||
|
||||
try:
|
||||
with job_logging_context("job123"):
|
||||
assert _current_job_id.get() == "job123"
|
||||
raise ValueError("Test exception")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Job should still be reset after exception
|
||||
assert _current_job_id.get() is None
|
||||
|
||||
|
||||
class TestJobLogHandler:
|
||||
"""Tests for JobLogHandler class."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_job_manager(self):
|
||||
"""Create a mock JobManager."""
|
||||
manager = MagicMock()
|
||||
manager.append_worker_log.return_value = None
|
||||
return manager
|
||||
|
||||
def test_job_log_handler_init(self, mock_job_manager):
|
||||
"""Test JobLogHandler initialization."""
|
||||
from backend.workers.worker_logging import JobLogHandler
|
||||
|
||||
handler = JobLogHandler(
|
||||
job_id="job123",
|
||||
worker_name="audio",
|
||||
job_manager=mock_job_manager
|
||||
)
|
||||
|
||||
assert handler.job_id == "job123"
|
||||
assert handler.worker_name == "audio"
|
||||
assert handler.job_manager == mock_job_manager
|
||||
assert handler.level == logging.INFO
|
||||
|
||||
def test_job_log_handler_custom_level(self, mock_job_manager):
|
||||
"""Test JobLogHandler with custom level."""
|
||||
from backend.workers.worker_logging import JobLogHandler
|
||||
|
||||
handler = JobLogHandler(
|
||||
job_id="job123",
|
||||
worker_name="audio",
|
||||
job_manager=mock_job_manager,
|
||||
level=logging.DEBUG
|
||||
)
|
||||
|
||||
assert handler.level == logging.DEBUG
|
||||
|
||||
def test_job_log_handler_emit_logs_to_firestore(self, mock_job_manager):
|
||||
"""Test that emit() calls job_manager.append_worker_log."""
|
||||
from backend.workers.worker_logging import JobLogHandler, job_logging_context
|
||||
|
||||
handler = JobLogHandler(
|
||||
job_id="job123",
|
||||
worker_name="audio",
|
||||
job_manager=mock_job_manager
|
||||
)
|
||||
handler.setFormatter(logging.Formatter('%(message)s'))
|
||||
|
||||
# Create a log record
|
||||
record = logging.LogRecord(
|
||||
name="test",
|
||||
level=logging.INFO,
|
||||
pathname="",
|
||||
lineno=0,
|
||||
msg="Test message",
|
||||
args=(),
|
||||
exc_info=None
|
||||
)
|
||||
|
||||
# Emit within the job context
|
||||
with job_logging_context("job123"):
|
||||
handler.emit(record)
|
||||
|
||||
# Should have called append_worker_log
|
||||
mock_job_manager.append_worker_log.assert_called()
|
||||
|
||||
def test_job_log_handler_filters_other_job_context(self, mock_job_manager):
|
||||
"""Test that handler filters logs from other job contexts."""
|
||||
from backend.workers.worker_logging import JobLogHandler, job_logging_context
|
||||
|
||||
handler = JobLogHandler(
|
||||
job_id="job123",
|
||||
worker_name="audio",
|
||||
job_manager=mock_job_manager
|
||||
)
|
||||
|
||||
record = logging.LogRecord(
|
||||
name="test",
|
||||
level=logging.INFO,
|
||||
pathname="",
|
||||
lineno=0,
|
||||
msg="Test message",
|
||||
args=(),
|
||||
exc_info=None
|
||||
)
|
||||
|
||||
# Emit from a different job's context
|
||||
with job_logging_context("different_job"):
|
||||
handler.emit(record)
|
||||
|
||||
# Should NOT have called append_worker_log
|
||||
mock_job_manager.append_worker_log.assert_not_called()
|
||||
|
||||
def test_job_log_handler_deduplication(self, mock_job_manager):
|
||||
"""Test that handler deduplicates repeated messages."""
|
||||
from backend.workers.worker_logging import JobLogHandler, job_logging_context
|
||||
|
||||
handler = JobLogHandler(
|
||||
job_id="job123",
|
||||
worker_name="audio",
|
||||
job_manager=mock_job_manager
|
||||
)
|
||||
handler.setFormatter(logging.Formatter('%(message)s'))
|
||||
|
||||
record = logging.LogRecord(
|
||||
name="test",
|
||||
level=logging.INFO,
|
||||
pathname="",
|
||||
lineno=0,
|
||||
msg="Duplicate message",
|
||||
args=(),
|
||||
exc_info=None
|
||||
)
|
||||
|
||||
with job_logging_context("job123"):
|
||||
# Emit the same record twice
|
||||
handler.emit(record)
|
||||
handler.emit(record)
|
||||
|
||||
# Should only be called once due to deduplication
|
||||
assert mock_job_manager.append_worker_log.call_count == 1
|
||||
|
||||
|
||||
class TestJobLogger:
|
||||
"""Tests for JobLogger class."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_job_manager(self):
|
||||
"""Create a mock JobManager."""
|
||||
manager = MagicMock()
|
||||
manager.append_worker_log.return_value = None
|
||||
return manager
|
||||
|
||||
def test_job_logger_init(self, mock_job_manager):
|
||||
"""Test JobLogger initialization."""
|
||||
from backend.workers.worker_logging import JobLogger
|
||||
|
||||
logger = JobLogger(
|
||||
job_id="job123",
|
||||
worker_name="lyrics",
|
||||
job_manager=mock_job_manager
|
||||
)
|
||||
|
||||
assert logger.job_id == "job123"
|
||||
assert logger.worker_name == "lyrics"
|
||||
assert logger.job_manager == mock_job_manager
|
||||
|
||||
def test_job_logger_info(self, mock_job_manager):
|
||||
"""Test JobLogger.info() method."""
|
||||
from backend.workers.worker_logging import JobLogger
|
||||
|
||||
logger = JobLogger(
|
||||
job_id="job123",
|
||||
worker_name="lyrics",
|
||||
job_manager=mock_job_manager
|
||||
)
|
||||
|
||||
logger.info("Processing started")
|
||||
|
||||
mock_job_manager.append_worker_log.assert_called_with(
|
||||
job_id="job123",
|
||||
worker="lyrics",
|
||||
level="INFO",
|
||||
message="Processing started"
|
||||
)
|
||||
|
||||
def test_job_logger_warning(self, mock_job_manager):
|
||||
"""Test JobLogger.warning() method."""
|
||||
from backend.workers.worker_logging import JobLogger
|
||||
|
||||
logger = JobLogger(
|
||||
job_id="job123",
|
||||
worker_name="audio",
|
||||
job_manager=mock_job_manager
|
||||
)
|
||||
|
||||
logger.warning("Low memory")
|
||||
|
||||
mock_job_manager.append_worker_log.assert_called_with(
|
||||
job_id="job123",
|
||||
worker="audio",
|
||||
level="WARNING",
|
||||
message="Low memory"
|
||||
)
|
||||
|
||||
def test_job_logger_error(self, mock_job_manager):
|
||||
"""Test JobLogger.error() method."""
|
||||
from backend.workers.worker_logging import JobLogger
|
||||
|
||||
logger = JobLogger(
|
||||
job_id="job123",
|
||||
worker_name="video",
|
||||
job_manager=mock_job_manager
|
||||
)
|
||||
|
||||
logger.error("Processing failed")
|
||||
|
||||
mock_job_manager.append_worker_log.assert_called_with(
|
||||
job_id="job123",
|
||||
worker="video",
|
||||
level="ERROR",
|
||||
message="Processing failed"
|
||||
)
|
||||
|
||||
def test_job_logger_debug(self, mock_job_manager):
|
||||
"""Test JobLogger.debug() method."""
|
||||
from backend.workers.worker_logging import JobLogger
|
||||
|
||||
logger = JobLogger(
|
||||
job_id="job123",
|
||||
worker_name="screens",
|
||||
job_manager=mock_job_manager
|
||||
)
|
||||
|
||||
logger.debug("Debug info")
|
||||
|
||||
mock_job_manager.append_worker_log.assert_called_with(
|
||||
job_id="job123",
|
||||
worker="screens",
|
||||
level="DEBUG",
|
||||
message="Debug info"
|
||||
)
|
||||
|
||||
def test_job_logger_with_format_args(self, mock_job_manager):
|
||||
"""Test JobLogger with format arguments."""
|
||||
from backend.workers.worker_logging import JobLogger
|
||||
|
||||
logger = JobLogger(
|
||||
job_id="job123",
|
||||
worker_name="audio",
|
||||
job_manager=mock_job_manager
|
||||
)
|
||||
|
||||
logger.info("Processing %s of %d", "audio", 10)
|
||||
|
||||
mock_job_manager.append_worker_log.assert_called_with(
|
||||
job_id="job123",
|
||||
worker="audio",
|
||||
level="INFO",
|
||||
message="Processing audio of 10"
|
||||
)
|
||||
|
||||
def test_job_logger_handles_firestore_error(self, mock_job_manager):
|
||||
"""Test that JobLogger handles Firestore errors gracefully."""
|
||||
from backend.workers.worker_logging import JobLogger
|
||||
|
||||
mock_job_manager.append_worker_log.side_effect = Exception("Firestore error")
|
||||
|
||||
logger = JobLogger(
|
||||
job_id="job123",
|
||||
worker_name="audio",
|
||||
job_manager=mock_job_manager
|
||||
)
|
||||
|
||||
# Should not raise exception
|
||||
logger.info("Test message")
|
||||
|
||||
|
||||
class TestCreateJobLogger:
|
||||
"""Tests for create_job_logger function."""
|
||||
|
||||
def test_create_job_logger(self):
|
||||
"""Test create_job_logger creates a JobLogger."""
|
||||
from backend.workers.worker_logging import create_job_logger, JobLogger
|
||||
|
||||
# Patch at the source module where JobManager is imported
|
||||
with patch('backend.services.job_manager.JobManager'):
|
||||
logger = create_job_logger("job123", "audio")
|
||||
|
||||
assert isinstance(logger, JobLogger)
|
||||
assert logger.job_id == "job123"
|
||||
assert logger.worker_name == "audio"
|
||||
|
||||
|
||||
class TestSetupJobLogging:
|
||||
"""Tests for setup_job_logging function."""
|
||||
|
||||
def test_setup_job_logging_returns_handler(self):
|
||||
"""Test setup_job_logging returns a JobLogHandler."""
|
||||
from backend.workers.worker_logging import setup_job_logging, JobLogHandler
|
||||
|
||||
# Patch at the source module
|
||||
with patch('backend.services.job_manager.JobManager'):
|
||||
handler = setup_job_logging("job123", "lyrics", "test_logger_wl1")
|
||||
|
||||
assert isinstance(handler, JobLogHandler)
|
||||
assert handler.job_id == "job123"
|
||||
assert handler.worker_name == "lyrics"
|
||||
|
||||
# Clean up - remove the handler
|
||||
logging.getLogger("test_logger_wl1").removeHandler(handler)
|
||||
|
||||
def test_setup_job_logging_adds_handler_to_loggers(self):
|
||||
"""Test setup_job_logging adds handler to specified loggers."""
|
||||
from backend.workers.worker_logging import setup_job_logging
|
||||
|
||||
with patch('backend.services.job_manager.JobManager'):
|
||||
handler = setup_job_logging("job123", "audio", "test_logger_wl2", "test_logger_wl3")
|
||||
|
||||
# Check handlers are added
|
||||
assert handler in logging.getLogger("test_logger_wl2").handlers
|
||||
assert handler in logging.getLogger("test_logger_wl3").handlers
|
||||
|
||||
# Clean up
|
||||
logging.getLogger("test_logger_wl2").removeHandler(handler)
|
||||
logging.getLogger("test_logger_wl3").removeHandler(handler)
|
||||
@@ -0,0 +1,871 @@
|
||||
"""
|
||||
Unit tests for backend workers.
|
||||
|
||||
These tests mock external dependencies and test worker logic in isolation.
|
||||
This includes the functions that would have caught bugs like the
|
||||
UnboundLocalError in upload_lyrics_results.
|
||||
"""
|
||||
import pytest
|
||||
import os
|
||||
import json
|
||||
import tempfile
|
||||
from datetime import datetime, UTC
|
||||
from unittest.mock import MagicMock, AsyncMock, patch, mock_open
|
||||
from pathlib import Path
|
||||
|
||||
from backend.models.job import Job, JobStatus
|
||||
|
||||
|
||||
class TestAudioWorker:
|
||||
"""Tests for audio_worker.py functions."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_job(self):
|
||||
"""Create a mock job for testing."""
|
||||
return Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
input_media_gcs_path="uploads/test123/song.flac"
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def mock_job_manager(self, mock_job):
|
||||
"""Create a mock JobManager."""
|
||||
manager = MagicMock()
|
||||
manager.get_job.return_value = mock_job
|
||||
manager.update_job_status.return_value = None
|
||||
manager.update_file_url.return_value = None
|
||||
manager.update_state_data.return_value = None
|
||||
manager.mark_job_failed.return_value = None
|
||||
return manager
|
||||
|
||||
@pytest.fixture
|
||||
def mock_storage(self):
|
||||
"""Create a mock StorageService."""
|
||||
storage = MagicMock()
|
||||
storage.download_file.return_value = "/tmp/test/song.flac"
|
||||
storage.upload_file.return_value = "gs://bucket/path"
|
||||
return storage
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_audio_from_gcs(self, mock_job_manager, mock_storage, mock_job):
|
||||
"""Test downloading audio from GCS for uploaded files."""
|
||||
with patch('backend.workers.audio_worker.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.workers.audio_worker.StorageService', return_value=mock_storage):
|
||||
|
||||
from backend.workers.audio_worker import download_audio
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
result = await download_audio("test123", temp_dir, mock_storage, mock_job)
|
||||
|
||||
# Should have called download_file with the GCS path
|
||||
mock_storage.download_file.assert_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_separation_results_handles_clean_stems(self, mock_job_manager, mock_storage):
|
||||
"""Test that upload_separation_results handles clean stems correctly."""
|
||||
from backend.workers.audio_worker import upload_separation_results
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
# Create actual test files
|
||||
inst_path = os.path.join(temp_dir, "instrumental.flac")
|
||||
vocals_path = os.path.join(temp_dir, "vocals.flac")
|
||||
with open(inst_path, 'wb') as f:
|
||||
f.write(b'fake audio data')
|
||||
with open(vocals_path, 'wb') as f:
|
||||
f.write(b'fake audio data')
|
||||
|
||||
# Create mock separation result matching AudioProcessor output format
|
||||
separation_result = {
|
||||
"clean_instrumental": {
|
||||
"instrumental": inst_path,
|
||||
"vocals": vocals_path
|
||||
},
|
||||
"other_stems": {},
|
||||
"backing_vocals": {},
|
||||
"combined_instrumentals": {}
|
||||
}
|
||||
|
||||
await upload_separation_results("test123", separation_result, mock_storage, mock_job_manager)
|
||||
|
||||
# Should have uploaded files
|
||||
assert mock_storage.upload_file.called
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_separation_results_handles_other_stems_as_dict(self, mock_job_manager, mock_storage):
|
||||
"""Test that upload_separation_results handles other_stems when values are dicts (bug fix)."""
|
||||
with patch('backend.workers.audio_worker.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.workers.audio_worker.StorageService', return_value=mock_storage):
|
||||
|
||||
from backend.workers.audio_worker import upload_separation_results
|
||||
|
||||
# This is the structure that caused the original bug - values are dicts, not strings
|
||||
separation_result = {
|
||||
"clean": {},
|
||||
"other_stems": {
|
||||
"bass": {"path": "/tmp/test/bass.flac", "other_key": "value"},
|
||||
"drums": "/tmp/test/drums.flac" # String path
|
||||
},
|
||||
"backing_vocals": {},
|
||||
"combined_instrumentals": {}
|
||||
}
|
||||
|
||||
# Mock os.path.exists
|
||||
with patch('os.path.exists', return_value=True):
|
||||
# This should NOT raise TypeError: stat: path should be string...
|
||||
await upload_separation_results("test123", separation_result, mock_storage, mock_job_manager)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_audio_separation_updates_status_on_failure(self, mock_job_manager, mock_storage):
|
||||
"""Test that process_audio_separation marks job as failed on error."""
|
||||
mock_job_manager.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test",
|
||||
title="Test"
|
||||
)
|
||||
|
||||
with patch('backend.workers.audio_worker.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.workers.audio_worker.StorageService', return_value=mock_storage), \
|
||||
patch('backend.workers.audio_worker.download_audio', side_effect=Exception("Download failed")):
|
||||
|
||||
from backend.workers.audio_worker import process_audio_separation
|
||||
|
||||
await process_audio_separation("test123")
|
||||
|
||||
# Should have marked job as failed
|
||||
mock_job_manager.mark_job_failed.assert_called()
|
||||
|
||||
|
||||
class TestLyricsWorker:
|
||||
"""Tests for lyrics_worker.py functions."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_job(self):
|
||||
"""Create a mock job for testing."""
|
||||
return Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="ABBA",
|
||||
title="Waterloo",
|
||||
input_media_gcs_path="uploads/test123/song.flac"
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def mock_job_manager(self, mock_job):
|
||||
"""Create a mock JobManager."""
|
||||
manager = MagicMock()
|
||||
manager.get_job.return_value = mock_job
|
||||
manager.update_job_status.return_value = None
|
||||
manager.update_file_url.return_value = None
|
||||
manager.update_state_data.return_value = None
|
||||
manager.mark_job_failed.return_value = None
|
||||
return manager
|
||||
|
||||
@pytest.fixture
|
||||
def mock_storage(self):
|
||||
"""Create a mock StorageService."""
|
||||
storage = MagicMock()
|
||||
storage.download_file.return_value = "/tmp/test/song.flac"
|
||||
storage.upload_file.return_value = "gs://bucket/path"
|
||||
return storage
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_lyrics_results_requires_job(self, mock_job_manager, mock_storage, mock_job):
|
||||
"""Test that upload_lyrics_results correctly fetches job for artist/title.
|
||||
|
||||
This test would have caught the UnboundLocalError bug where job was used
|
||||
before being defined.
|
||||
"""
|
||||
with patch('backend.workers.lyrics_worker.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.workers.lyrics_worker.StorageService', return_value=mock_storage):
|
||||
|
||||
from backend.workers.lyrics_worker import upload_lyrics_results
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
# Create mock lyrics directory and files
|
||||
lyrics_dir = os.path.join(temp_dir, "lyrics")
|
||||
os.makedirs(lyrics_dir)
|
||||
|
||||
# Create test LRC file
|
||||
lrc_path = os.path.join(lyrics_dir, "ABBA - Waterloo (Karaoke).lrc")
|
||||
with open(lrc_path, 'w') as f:
|
||||
f.write("[00:00.00]Test lyrics\n")
|
||||
|
||||
# Create corrections JSON
|
||||
corrections_path = os.path.join(lyrics_dir, "ABBA - Waterloo (Lyrics Corrections).json")
|
||||
with open(corrections_path, 'w') as f:
|
||||
json.dump({"lines": [], "corrections": []}, f)
|
||||
|
||||
transcription_result = {
|
||||
"lrc_filepath": lrc_path,
|
||||
"corrections_filepath": corrections_path
|
||||
}
|
||||
|
||||
# This should NOT raise UnboundLocalError
|
||||
await upload_lyrics_results(
|
||||
"test123",
|
||||
temp_dir,
|
||||
transcription_result,
|
||||
mock_storage,
|
||||
mock_job_manager
|
||||
)
|
||||
|
||||
# Verify job was fetched
|
||||
mock_job_manager.get_job.assert_called_with("test123")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_lyrics_results_uploads_lrc_file(self, mock_job_manager, mock_storage, mock_job):
|
||||
"""Test that LRC file is uploaded correctly."""
|
||||
from backend.workers.lyrics_worker import upload_lyrics_results
|
||||
import json
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
lyrics_dir = os.path.join(temp_dir, "lyrics")
|
||||
os.makedirs(lyrics_dir)
|
||||
|
||||
lrc_path = os.path.join(lyrics_dir, "test.lrc")
|
||||
with open(lrc_path, 'w') as f:
|
||||
f.write("[00:00.00]Test\n")
|
||||
|
||||
# Create required corrections.json file
|
||||
corrections_path = os.path.join(lyrics_dir, "corrections.json")
|
||||
with open(corrections_path, 'w') as f:
|
||||
json.dump({"corrected_segments": []}, f)
|
||||
|
||||
transcription_result = {"lrc_filepath": lrc_path}
|
||||
|
||||
await upload_lyrics_results(
|
||||
"test123", temp_dir, transcription_result,
|
||||
mock_storage, mock_job_manager
|
||||
)
|
||||
|
||||
# Should have uploaded the LRC file
|
||||
mock_storage.upload_file.assert_called()
|
||||
mock_job_manager.update_file_url.assert_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_lyrics_results_handles_missing_files(self, mock_job_manager, mock_storage, mock_job):
|
||||
"""Test graceful handling when optional files are missing."""
|
||||
from backend.workers.lyrics_worker import upload_lyrics_results
|
||||
import json
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
lyrics_dir = os.path.join(temp_dir, "lyrics")
|
||||
os.makedirs(lyrics_dir)
|
||||
|
||||
# Only create LRC file and required corrections.json, no other files
|
||||
lrc_path = os.path.join(lyrics_dir, "test.lrc")
|
||||
with open(lrc_path, 'w') as f:
|
||||
f.write("[00:00.00]Test\n")
|
||||
|
||||
# Create required corrections.json file
|
||||
corrections_path = os.path.join(lyrics_dir, "corrections.json")
|
||||
with open(corrections_path, 'w') as f:
|
||||
json.dump({"corrected_segments": []}, f)
|
||||
|
||||
transcription_result = {"lrc_filepath": lrc_path}
|
||||
|
||||
# Should not raise exception for missing optional files
|
||||
await upload_lyrics_results(
|
||||
"test123", temp_dir, transcription_result,
|
||||
mock_storage, mock_job_manager
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_lyrics_results_uses_artist_title_from_job(self, mock_job_manager, mock_storage, mock_job):
|
||||
"""Test that upload_lyrics_results correctly uses job.artist and job.title.
|
||||
|
||||
This test specifically validates the bug fix where job was not defined
|
||||
when accessing job.artist and job.title for reference file lookups.
|
||||
"""
|
||||
from backend.workers.lyrics_worker import upload_lyrics_results
|
||||
import json
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
lyrics_dir = os.path.join(temp_dir, "lyrics")
|
||||
os.makedirs(lyrics_dir)
|
||||
|
||||
# Create LRC file
|
||||
lrc_path = os.path.join(lyrics_dir, "test.lrc")
|
||||
with open(lrc_path, 'w') as f:
|
||||
f.write("[00:00.00]Test\n")
|
||||
|
||||
# Create required corrections.json file
|
||||
corrections_path = os.path.join(lyrics_dir, "corrections.json")
|
||||
with open(corrections_path, 'w') as f:
|
||||
json.dump({"corrected_segments": []}, f)
|
||||
|
||||
# Create a reference lyrics file using the job's artist/title
|
||||
ref_path = os.path.join(lyrics_dir, f"{mock_job.artist} - {mock_job.title} (Lyrics Genius).txt")
|
||||
with open(ref_path, 'w') as f:
|
||||
f.write("Reference lyrics content\n")
|
||||
|
||||
# Create uncorrected transcription file using job's artist/title
|
||||
uncorrected_path = os.path.join(lyrics_dir, f"{mock_job.artist} - {mock_job.title} (Lyrics Uncorrected).txt")
|
||||
with open(uncorrected_path, 'w') as f:
|
||||
f.write("Uncorrected transcription\n")
|
||||
|
||||
transcription_result = {"lrc_filepath": lrc_path}
|
||||
|
||||
# This should NOT raise UnboundLocalError for 'job'
|
||||
await upload_lyrics_results(
|
||||
"test123", temp_dir, transcription_result,
|
||||
mock_storage, mock_job_manager
|
||||
)
|
||||
|
||||
# Verify job was fetched to get artist/title
|
||||
mock_job_manager.get_job.assert_called_with("test123")
|
||||
|
||||
# Verify files were uploaded (the reference and uncorrected files exist)
|
||||
upload_calls = mock_storage.upload_file.call_args_list
|
||||
assert len(upload_calls) >= 2 # LRC + at least one reference or uncorrected
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_lyrics_transcription_marks_failed_on_error(self, mock_job_manager, mock_storage):
|
||||
"""Test that process_lyrics_transcription marks job as failed on error."""
|
||||
mock_job_manager.get_job.return_value = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test",
|
||||
title="Test",
|
||||
input_media_gcs_path="uploads/test123/song.flac"
|
||||
)
|
||||
|
||||
with patch('backend.workers.lyrics_worker.JobManager', return_value=mock_job_manager), \
|
||||
patch('backend.workers.lyrics_worker.StorageService', return_value=mock_storage), \
|
||||
patch('backend.workers.lyrics_worker.download_audio', side_effect=Exception("Download failed")):
|
||||
|
||||
from backend.workers.lyrics_worker import process_lyrics_transcription
|
||||
|
||||
await process_lyrics_transcription("test123")
|
||||
|
||||
mock_job_manager.mark_job_failed.assert_called()
|
||||
|
||||
|
||||
class TestLyricsWorkerConfiguration:
|
||||
"""Tests for lyrics worker configuration parameters."""
|
||||
|
||||
def test_create_lyrics_processor_with_defaults(self):
|
||||
"""Test creating LyricsProcessor with default parameters."""
|
||||
with patch('backend.workers.lyrics_worker.LyricsProcessor') as mock_processor:
|
||||
from backend.workers.lyrics_worker import create_lyrics_processor
|
||||
|
||||
result = create_lyrics_processor()
|
||||
|
||||
mock_processor.assert_called_once()
|
||||
call_kwargs = mock_processor.call_args[1]
|
||||
assert call_kwargs['lyrics_file'] is None
|
||||
assert call_kwargs['subtitle_offset_ms'] == 0
|
||||
|
||||
def test_create_lyrics_processor_with_lyrics_file(self):
|
||||
"""Test creating LyricsProcessor with custom lyrics file."""
|
||||
with patch('backend.workers.lyrics_worker.LyricsProcessor') as mock_processor:
|
||||
from backend.workers.lyrics_worker import create_lyrics_processor
|
||||
|
||||
result = create_lyrics_processor(
|
||||
lyrics_file="/path/to/lyrics.txt",
|
||||
subtitle_offset_ms=0
|
||||
)
|
||||
|
||||
mock_processor.assert_called_once()
|
||||
call_kwargs = mock_processor.call_args[1]
|
||||
assert call_kwargs['lyrics_file'] == "/path/to/lyrics.txt"
|
||||
|
||||
def test_create_lyrics_processor_with_subtitle_offset(self):
|
||||
"""Test creating LyricsProcessor with subtitle offset."""
|
||||
with patch('backend.workers.lyrics_worker.LyricsProcessor') as mock_processor:
|
||||
from backend.workers.lyrics_worker import create_lyrics_processor
|
||||
|
||||
result = create_lyrics_processor(
|
||||
subtitle_offset_ms=500
|
||||
)
|
||||
|
||||
mock_processor.assert_called_once()
|
||||
call_kwargs = mock_processor.call_args[1]
|
||||
assert call_kwargs['subtitle_offset_ms'] == 500
|
||||
|
||||
|
||||
class TestLyricsOverrideParameters:
|
||||
"""Tests for lyrics artist/title override functionality."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_job_with_overrides(self):
|
||||
"""Create a mock job with lyrics override fields."""
|
||||
return Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Beatles, The",
|
||||
title="Hey Jude - 2009 Remaster",
|
||||
lyrics_artist="The Beatles",
|
||||
lyrics_title="Hey Jude",
|
||||
subtitle_offset_ms=250,
|
||||
input_media_gcs_path="uploads/test123/song.flac"
|
||||
)
|
||||
|
||||
def test_job_uses_lyrics_artist_override(self, mock_job_with_overrides):
|
||||
"""Test that job uses lyrics_artist when searching for lyrics."""
|
||||
job = mock_job_with_overrides
|
||||
|
||||
# Use override if present, else fall back to main artist
|
||||
lyrics_search_artist = job.lyrics_artist or job.artist
|
||||
|
||||
assert lyrics_search_artist == "The Beatles"
|
||||
assert lyrics_search_artist != job.artist # Override is different
|
||||
|
||||
def test_job_uses_lyrics_title_override(self, mock_job_with_overrides):
|
||||
"""Test that job uses lyrics_title when searching for lyrics."""
|
||||
job = mock_job_with_overrides
|
||||
|
||||
# Use override if present, else fall back to main title
|
||||
lyrics_search_title = job.lyrics_title or job.title
|
||||
|
||||
assert lyrics_search_title == "Hey Jude"
|
||||
assert lyrics_search_title != job.title # Override is different
|
||||
|
||||
def test_job_falls_back_when_no_override(self):
|
||||
"""Test that job falls back to main artist/title when no override."""
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
input_media_gcs_path="uploads/test123/song.flac"
|
||||
)
|
||||
|
||||
# When override is None, use main values
|
||||
lyrics_search_artist = job.lyrics_artist or job.artist
|
||||
lyrics_search_title = job.lyrics_title or job.title
|
||||
|
||||
assert lyrics_search_artist == "Test Artist"
|
||||
assert lyrics_search_title == "Test Song"
|
||||
|
||||
|
||||
class TestScreensWorker:
|
||||
"""Tests for screens_worker.py functions.
|
||||
|
||||
Note: The main process function is tested indirectly via integration tests.
|
||||
These unit tests focus on helper functions and module structure.
|
||||
"""
|
||||
|
||||
def test_screens_worker_module_imports(self):
|
||||
"""Test screens worker module can be imported."""
|
||||
from backend.workers import screens_worker
|
||||
assert hasattr(screens_worker, 'logger')
|
||||
|
||||
|
||||
class TestVideoWorker:
|
||||
"""Tests for video_worker.py functions.
|
||||
|
||||
Note: The main process function is tested indirectly via integration tests.
|
||||
These unit tests focus on helper functions and module structure.
|
||||
"""
|
||||
|
||||
def test_video_worker_module_imports(self):
|
||||
"""Test video worker module can be imported."""
|
||||
from backend.workers import video_worker
|
||||
assert hasattr(video_worker, 'logger')
|
||||
|
||||
|
||||
class TestRenderVideoWorkerConfiguration:
|
||||
"""Tests for render_video_worker subtitle_offset_ms support."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_job_with_offset(self):
|
||||
"""Create a mock job with subtitle offset."""
|
||||
return Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.RENDERING_VIDEO,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
subtitle_offset_ms=500,
|
||||
input_media_gcs_path="uploads/test123/song.flac"
|
||||
)
|
||||
|
||||
def test_job_has_subtitle_offset_ms(self, mock_job_with_offset):
|
||||
"""Test that job has subtitle_offset_ms field."""
|
||||
assert mock_job_with_offset.subtitle_offset_ms == 500
|
||||
|
||||
def test_subtitle_offset_from_job(self, mock_job_with_offset):
|
||||
"""Test extracting subtitle offset from job with getattr."""
|
||||
job = mock_job_with_offset
|
||||
|
||||
# This mirrors the logic in render_video_worker.py
|
||||
subtitle_offset = getattr(job, 'subtitle_offset_ms', 0) or 0
|
||||
|
||||
assert subtitle_offset == 500
|
||||
|
||||
def test_subtitle_offset_default_zero(self):
|
||||
"""Test that subtitle offset defaults to 0 when not set."""
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.RENDERING_VIDEO,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
input_media_gcs_path="uploads/test123/song.flac"
|
||||
)
|
||||
|
||||
subtitle_offset = getattr(job, 'subtitle_offset_ms', 0) or 0
|
||||
|
||||
assert subtitle_offset == 0
|
||||
|
||||
def test_subtitle_offset_negative_value(self):
|
||||
"""Test that subtitle offset can be negative (advance subtitles)."""
|
||||
job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.RENDERING_VIDEO,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
subtitle_offset_ms=-250, # Negative = advance subtitles
|
||||
input_media_gcs_path="uploads/test123/song.flac"
|
||||
)
|
||||
|
||||
subtitle_offset = getattr(job, 'subtitle_offset_ms', 0) or 0
|
||||
|
||||
# Negative values should be preserved (not converted to 0)
|
||||
assert subtitle_offset == -250
|
||||
|
||||
|
||||
class TestDownloadHelpers:
|
||||
"""Tests for download helper functions in workers."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_audio_handles_uploaded_file(self):
|
||||
"""Test download_audio handles jobs with uploaded file."""
|
||||
mock_storage = MagicMock()
|
||||
mock_storage.download_file.return_value = "/tmp/downloaded.flac"
|
||||
|
||||
mock_job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test",
|
||||
title="Test",
|
||||
input_media_gcs_path="uploads/test123/song.flac" # Uploaded file
|
||||
)
|
||||
|
||||
from backend.workers.audio_worker import download_audio
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
result = await download_audio("test123", temp_dir, mock_storage, mock_job)
|
||||
|
||||
# Should have downloaded from GCS
|
||||
mock_storage.download_file.assert_called()
|
||||
|
||||
|
||||
class TestAudioWorkerModelConfiguration:
|
||||
"""Tests for audio worker model configuration parameters."""
|
||||
|
||||
def test_create_audio_processor_with_defaults(self):
|
||||
"""Test creating AudioProcessor with default models."""
|
||||
with patch('backend.workers.audio_worker.AudioProcessor') as mock_processor:
|
||||
from backend.workers.audio_worker import create_audio_processor
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
result = create_audio_processor(temp_dir)
|
||||
|
||||
mock_processor.assert_called_once()
|
||||
call_kwargs = mock_processor.call_args[1]
|
||||
|
||||
# Should use default models
|
||||
assert call_kwargs['clean_instrumental_model'] == "model_bs_roformer_ep_317_sdr_12.9755.ckpt"
|
||||
assert call_kwargs['backing_vocals_models'] == ["mel_band_roformer_karaoke_aufr33_viperx_sdr_10.1956.ckpt"]
|
||||
assert call_kwargs['other_stems_models'] == ["htdemucs_6s.yaml"]
|
||||
|
||||
def test_create_audio_processor_with_custom_clean_model(self):
|
||||
"""Test creating AudioProcessor with custom clean instrumental model."""
|
||||
with patch('backend.workers.audio_worker.AudioProcessor') as mock_processor:
|
||||
from backend.workers.audio_worker import create_audio_processor
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
result = create_audio_processor(
|
||||
temp_dir,
|
||||
clean_instrumental_model="custom_clean_model.ckpt"
|
||||
)
|
||||
|
||||
mock_processor.assert_called_once()
|
||||
call_kwargs = mock_processor.call_args[1]
|
||||
|
||||
# Should use custom clean model
|
||||
assert call_kwargs['clean_instrumental_model'] == "custom_clean_model.ckpt"
|
||||
# Other models should still be defaults
|
||||
assert call_kwargs['backing_vocals_models'] == ["mel_band_roformer_karaoke_aufr33_viperx_sdr_10.1956.ckpt"]
|
||||
|
||||
def test_create_audio_processor_with_custom_backing_models(self):
|
||||
"""Test creating AudioProcessor with custom backing vocals models."""
|
||||
with patch('backend.workers.audio_worker.AudioProcessor') as mock_processor:
|
||||
from backend.workers.audio_worker import create_audio_processor
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
result = create_audio_processor(
|
||||
temp_dir,
|
||||
backing_vocals_models=["custom_bv1.ckpt", "custom_bv2.ckpt"]
|
||||
)
|
||||
|
||||
mock_processor.assert_called_once()
|
||||
call_kwargs = mock_processor.call_args[1]
|
||||
|
||||
# Should use custom backing vocals models
|
||||
assert call_kwargs['backing_vocals_models'] == ["custom_bv1.ckpt", "custom_bv2.ckpt"]
|
||||
|
||||
def test_create_audio_processor_with_custom_other_stems_models(self):
|
||||
"""Test creating AudioProcessor with custom other stems models."""
|
||||
with patch('backend.workers.audio_worker.AudioProcessor') as mock_processor:
|
||||
from backend.workers.audio_worker import create_audio_processor
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
result = create_audio_processor(
|
||||
temp_dir,
|
||||
other_stems_models=["custom_demucs.yaml"]
|
||||
)
|
||||
|
||||
mock_processor.assert_called_once()
|
||||
call_kwargs = mock_processor.call_args[1]
|
||||
|
||||
# Should use custom other stems models
|
||||
assert call_kwargs['other_stems_models'] == ["custom_demucs.yaml"]
|
||||
|
||||
def test_create_audio_processor_with_all_custom_models(self):
|
||||
"""Test creating AudioProcessor with all custom models."""
|
||||
with patch('backend.workers.audio_worker.AudioProcessor') as mock_processor:
|
||||
from backend.workers.audio_worker import create_audio_processor
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
result = create_audio_processor(
|
||||
temp_dir,
|
||||
clean_instrumental_model="custom_clean.ckpt",
|
||||
backing_vocals_models=["custom_bv.ckpt"],
|
||||
other_stems_models=["custom_stems.yaml"]
|
||||
)
|
||||
|
||||
mock_processor.assert_called_once()
|
||||
call_kwargs = mock_processor.call_args[1]
|
||||
|
||||
# All models should be custom
|
||||
assert call_kwargs['clean_instrumental_model'] == "custom_clean.ckpt"
|
||||
assert call_kwargs['backing_vocals_models'] == ["custom_bv.ckpt"]
|
||||
assert call_kwargs['other_stems_models'] == ["custom_stems.yaml"]
|
||||
|
||||
def test_job_model_fields_are_passed_to_processor(self):
|
||||
"""Test that job model fields can be passed to create_audio_processor."""
|
||||
mock_job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test",
|
||||
title="Test",
|
||||
clean_instrumental_model="job_clean.ckpt",
|
||||
backing_vocals_models=["job_bv.ckpt"],
|
||||
other_stems_models=["job_stems.yaml"]
|
||||
)
|
||||
|
||||
with patch('backend.workers.audio_worker.AudioProcessor') as mock_processor:
|
||||
from backend.workers.audio_worker import create_audio_processor
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
# Simulate passing job model fields to create_audio_processor
|
||||
result = create_audio_processor(
|
||||
temp_dir,
|
||||
clean_instrumental_model=mock_job.clean_instrumental_model,
|
||||
backing_vocals_models=mock_job.backing_vocals_models,
|
||||
other_stems_models=mock_job.other_stems_models
|
||||
)
|
||||
|
||||
mock_processor.assert_called_once()
|
||||
call_kwargs = mock_processor.call_args[1]
|
||||
|
||||
# Models from job should be used
|
||||
assert call_kwargs['clean_instrumental_model'] == "job_clean.ckpt"
|
||||
assert call_kwargs['backing_vocals_models'] == ["job_bv.ckpt"]
|
||||
assert call_kwargs['other_stems_models'] == ["job_stems.yaml"]
|
||||
|
||||
def test_none_model_values_use_defaults(self):
|
||||
"""Test that None model values fall back to defaults."""
|
||||
mock_job = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PENDING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test",
|
||||
title="Test",
|
||||
clean_instrumental_model=None, # Not specified
|
||||
backing_vocals_models=None, # Not specified
|
||||
other_stems_models=None # Not specified
|
||||
)
|
||||
|
||||
with patch('backend.workers.audio_worker.AudioProcessor') as mock_processor:
|
||||
from backend.workers.audio_worker import create_audio_processor
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
result = create_audio_processor(
|
||||
temp_dir,
|
||||
clean_instrumental_model=mock_job.clean_instrumental_model,
|
||||
backing_vocals_models=mock_job.backing_vocals_models,
|
||||
other_stems_models=mock_job.other_stems_models
|
||||
)
|
||||
|
||||
mock_processor.assert_called_once()
|
||||
call_kwargs = mock_processor.call_args[1]
|
||||
|
||||
# Should fall back to defaults
|
||||
assert call_kwargs['clean_instrumental_model'] == "model_bs_roformer_ep_317_sdr_12.9755.ckpt"
|
||||
assert call_kwargs['backing_vocals_models'] == ["mel_band_roformer_karaoke_aufr33_viperx_sdr_10.1956.ckpt"]
|
||||
assert call_kwargs['other_stems_models'] == ["htdemucs_6s.yaml"]
|
||||
|
||||
|
||||
class TestDownloadFromUrl:
|
||||
"""Tests for download_from_url function - URL-based audio download."""
|
||||
|
||||
def test_download_from_url_function_exists(self):
|
||||
"""Test that download_from_url function exists in audio_worker."""
|
||||
from backend.workers.audio_worker import download_from_url
|
||||
assert callable(download_from_url)
|
||||
|
||||
def test_download_from_url_signature(self):
|
||||
"""Test that download_from_url has the expected signature."""
|
||||
import inspect
|
||||
from backend.workers.audio_worker import download_from_url
|
||||
|
||||
sig = inspect.signature(download_from_url)
|
||||
params = list(sig.parameters.keys())
|
||||
|
||||
# Should have these parameters
|
||||
assert 'url' in params
|
||||
assert 'temp_dir' in params
|
||||
assert 'artist' in params
|
||||
assert 'title' in params
|
||||
assert 'job_manager' in params
|
||||
assert 'job_id' in params
|
||||
|
||||
def test_download_from_url_is_async(self):
|
||||
"""Test that download_from_url is an async function."""
|
||||
import inspect
|
||||
from backend.workers.audio_worker import download_from_url
|
||||
|
||||
assert inspect.iscoroutinefunction(download_from_url)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_from_url_returns_none_for_invalid_url(self):
|
||||
"""Test that download_from_url handles errors gracefully."""
|
||||
from backend.workers.audio_worker import download_from_url
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
# This should fail gracefully (no yt-dlp in test env or invalid URL)
|
||||
result = await download_from_url(
|
||||
url='not-a-valid-url',
|
||||
temp_dir=temp_dir,
|
||||
artist='Test',
|
||||
title='Test'
|
||||
)
|
||||
|
||||
# Should return None on error (graceful failure)
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestDownloadAudioWithUrl:
|
||||
"""Tests for download_audio function with URL-based jobs."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_job_with_url(self):
|
||||
"""Create a mock job with a URL."""
|
||||
return Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PROCESSING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
url="https://www.youtube.com/watch?v=test123"
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def mock_job_manager(self, mock_job_with_url):
|
||||
"""Create a mock JobManager."""
|
||||
manager = MagicMock()
|
||||
manager.get_job.return_value = mock_job_with_url
|
||||
manager.update_job.return_value = None
|
||||
return manager
|
||||
|
||||
@pytest.fixture
|
||||
def mock_storage(self):
|
||||
"""Create a mock StorageService."""
|
||||
storage = MagicMock()
|
||||
return storage
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_audio_from_url(self, mock_job_with_url, mock_job_manager, mock_storage):
|
||||
"""Test download_audio routes to URL download when job has URL."""
|
||||
with patch('backend.workers.audio_worker.download_from_url', return_value='/tmp/test.wav') as mock_download:
|
||||
from backend.workers.audio_worker import download_audio
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
result = await download_audio(
|
||||
"test123",
|
||||
temp_dir,
|
||||
mock_storage,
|
||||
mock_job_with_url,
|
||||
job_manager_instance=mock_job_manager
|
||||
)
|
||||
|
||||
# Should have called download_from_url
|
||||
mock_download.assert_called_once()
|
||||
# Check positional args
|
||||
args = mock_download.call_args[0]
|
||||
assert args[0] == "https://www.youtube.com/watch?v=test123" # url
|
||||
assert args[1] == temp_dir # temp_dir
|
||||
assert args[2] == "Test Artist" # artist
|
||||
assert args[3] == "Test Song" # title
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_audio_from_gcs_when_no_url(self, mock_job_manager, mock_storage):
|
||||
"""Test download_audio downloads from GCS when job has no URL."""
|
||||
mock_job_no_url = Job(
|
||||
job_id="test123",
|
||||
status=JobStatus.PROCESSING,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
input_media_gcs_path="uploads/test123/song.flac"
|
||||
)
|
||||
|
||||
with patch('backend.workers.audio_worker.download_from_url') as mock_url_download:
|
||||
from backend.workers.audio_worker import download_audio
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
result = await download_audio(
|
||||
"test123",
|
||||
temp_dir,
|
||||
mock_storage,
|
||||
mock_job_no_url,
|
||||
job_manager_instance=mock_job_manager
|
||||
)
|
||||
|
||||
# Should NOT have called download_from_url
|
||||
mock_url_download.assert_not_called()
|
||||
|
||||
# Should have called storage.download_file
|
||||
mock_storage.download_file.assert_called()
|
||||
|
||||
Loaded 100 of 758 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user