diff --git a/backend/api/routes/internal.py b/backend/api/routes/internal.py index e80552c0..418b1658 100644 --- a/backend/api/routes/internal.py +++ b/backend/api/routes/internal.py @@ -704,11 +704,27 @@ async def trigger_gdrive_validation_endpoint( trace_context = extract_trace_context(dict(http_request.headers)) + # Optional brand code (incident-hardening D2): forwarded so the validator can + # check the just-published track's own completeness same-run. expect_cdg says + # whether that track produced a CDG (so the validator requires the CDG folder). + brand_code = None + expect_cdg = False + try: + body = await http_request.json() + if isinstance(body, dict): + brand_code = body.get("brand_code") + expect_cdg = bool(body.get("expect_cdg", False)) + except Exception: # noqa: BLE001 - empty/non-JSON body is the normal case + brand_code = None + expect_cdg = False + logger.info("GDRIVE_VALIDATION_TRIGGER starting (delayed post-job)") add_span_attribute("operation", "gdrive_validation_trigger") + if brand_code: + add_span_attribute("brand_code", str(brand_code)) try: - result = trigger_gdrive_validation() + result = trigger_gdrive_validation(brand_code=brand_code, expect_cdg=expect_cdg) if result is None: return {"status": "skipped", "message": "GDRIVE_VALIDATOR_URL not configured"} diff --git a/backend/services/discord_service.py b/backend/services/discord_service.py index 404ea69a..3a369a71 100644 --- a/backend/services/discord_service.py +++ b/backend/services/discord_service.py @@ -104,7 +104,11 @@ class DiscordNotificationService: return True self.logger.info(f"Posting message to Discord webhook...") - data = {"content": message} + # Disable all mentions: message content can include user-supplied text + # (artist/title in ops alerts), so an embedded @everyone / <@id> must not + # trigger a Discord ping. (CWE-74 — output injection into a downstream + # component.) + data = {"content": message, "allowed_mentions": {"parse": []}} response = requests.post(url, json=data, timeout=30) response.raise_for_status() self.logger.info("Message posted to Discord successfully") diff --git a/backend/services/firestore_service.py b/backend/services/firestore_service.py index bd7e4515..28c3a55b 100644 --- a/backend/services/firestore_service.py +++ b/backend/services/firestore_service.py @@ -113,6 +113,25 @@ class FirestoreService: except Exception as e: logger.error(f"Error updating job status {job_id}: {e}") raise + + # Incident-hardening D1: near-real-time alert on ANY transition to FAILED. + # The spike-based error monitor is blind to a novel low-volume failure + # (rolling_avg==0 → never a spike), so a brand-new error on a couple of + # jobs can go unnoticed. This is the universal net. Best-effort: it must + # never turn a successful status write into a failure, hence the guard. + if status == JobStatus.FAILED: + try: + from backend.services.ops_alerts import notify_job_failed + + notify_job_failed( + db=self.db, + collection=self.collection, + job_id=job_id, + message=message, + additional_fields=additional_fields, + ) + except Exception as alert_exc: # noqa: BLE001 + logger.warning(f"Failure alert for job {job_id} did not send: {alert_exc}") def list_jobs( self, diff --git a/backend/services/gdrive_validator_client.py b/backend/services/gdrive_validator_client.py index 9e0a8d0c..e1a726d2 100644 --- a/backend/services/gdrive_validator_client.py +++ b/backend/services/gdrive_validator_client.py @@ -20,12 +20,21 @@ logger = logging.getLogger(__name__) GDRIVE_VALIDATOR_URL = os.environ.get("GDRIVE_VALIDATOR_URL", "") -def trigger_gdrive_validation() -> dict | None: +def trigger_gdrive_validation( + brand_code: str | None = None, expect_cdg: bool = False +) -> dict | None: """ Invoke the GDrive validator Cloud Function. Uses the backend service account's OIDC token for authentication. + Args: + brand_code: Optional brand code of the just-published track. When set, the + validator additionally verifies this track is present in all expected + folders same-run (incident-hardening D2). + expect_cdg: Whether the just-published track produced a CDG package (so the + validator requires the CDG folder for it). Ignored without brand_code. + Returns: Parsed JSON response from the function, or None on error. """ @@ -41,6 +50,7 @@ def trigger_gdrive_validation() -> dict | None: response = requests.post( GDRIVE_VALIDATOR_URL, headers={"Authorization": f"Bearer {token}"}, + json={"brand_code": brand_code, "expect_cdg": expect_cdg} if brand_code else None, timeout=60, ) response.raise_for_status() diff --git a/backend/services/ops_alerts.py b/backend/services/ops_alerts.py new file mode 100644 index 00000000..aad365b1 --- /dev/null +++ b/backend/services/ops_alerts.py @@ -0,0 +1,304 @@ +"""Near-real-time operational alerting (incident-hardening D1 + G1 channel). + +Why this exists +--------------- +The production error monitor (``backend/services/error_monitor``) is +**spike-based**: a pattern only alerts when its current count exceeds a rolling +average by a multiplier *and* clears a minimum-count floor. A brand-new error +has ``rolling_avg == 0`` → it is mathematically never a spike, so a novel, +low-volume failure is invisible to it. That is exactly how incident NOMAD-1632's +follow-on outage (2 jobs failing with a never-before-seen error string) went +unnoticed until a human spotted failed jobs in the UI. + +This module is the **universal net**: the moment *any* job transitions to +``FAILED`` we emit an immediate Discord alert, independent of the spike detector, +de-duplicated by a normalized error signature so a burst collapses into one +alert. Novel error signatures always alert regardless of count. + +It also exposes :func:`send_ops_alert`, a generic best-effort sender reused by +the publish-completeness shadow invariant (G1). + +Everything here is **best-effort**: any failure to alert is swallowed and +logged. It must never raise into — nor add a failure mode to — the status write +that triggered it. +""" + +from __future__ import annotations + +import logging +import os +from datetime import datetime, timedelta, timezone +from typing import Any, Optional + +logger = logging.getLogger(__name__) + +# Secret Manager name of the ops/alert Discord webhook (same channel the startup +# credential-validation alert uses — see backend/main.py). +_ALERT_WEBHOOK_SECRET = "discord-alert-webhook" + +# Firestore collection used to de-dupe/throttle failure alerts by signature. +_DEDUP_COLLECTION = "ops_failure_alerts" + +# Collapse repeat alerts for the same error signature within this window into a +# single Discord message (the burst is counted, not re-sent). Override with +# FAILURE_ALERT_THROTTLE_MINUTES. +_DEFAULT_THROTTLE_MINUTES = 30 + +# Environments whose failures we do NOT page on (intentional test/dev failures +# would otherwise be noise). Production + unlabeled jobs always alert. Override +# with FAILURE_ALERT_SKIP_ENVIRONMENTS (comma-separated). +_DEFAULT_SKIP_ENVIRONMENTS = {"test", "development"} + + +def _is_enabled() -> bool: + """Only alert from a deployed environment. + + Local runs and the test suite have no ``K_SERVICE`` / non-production + ``ENVIRONMENT`` (see :func:`backend.config.is_production`), so this short + circuits before any Secret Manager or Discord call — keeping unit tests + silent and fast. + """ + if os.environ.get("FAILURE_ALERTS_ENABLED", "").lower() in ("1", "true", "yes"): + return True + try: + from backend.config import is_production + + return is_production() + except Exception: # noqa: BLE001 - never let config import break a status write + return False + + +def _throttle_minutes() -> int: + try: + return int(os.environ.get("FAILURE_ALERT_THROTTLE_MINUTES", _DEFAULT_THROTTLE_MINUTES)) + except (TypeError, ValueError): + return _DEFAULT_THROTTLE_MINUTES + + +def _skip_environments() -> set[str]: + raw = os.environ.get("FAILURE_ALERT_SKIP_ENVIRONMENTS") + if raw is None: + return set(_DEFAULT_SKIP_ENVIRONMENTS) + return {e.strip().lower() for e in raw.split(",") if e.strip()} + + +def _get_webhook_url() -> Optional[str]: + """Resolve the ops-alert Discord webhook (cached via settings.get_secret).""" + try: + from backend.config import settings + + if hasattr(settings, "get_secret"): + return settings.get_secret(_ALERT_WEBHOOK_SECRET) + except Exception as exc: # noqa: BLE001 + logger.debug("ops_alerts: could not resolve webhook: %s", exc) + return None + + +def send_ops_alert(message: str) -> bool: + """Post a plaintext ops alert to the Discord alert channel. Best-effort. + + Returns True if the message was sent, False otherwise (disabled, no webhook, + or a delivery error). Never raises. + """ + try: + if not _is_enabled(): + logger.debug("ops_alerts: disabled (not a deployed environment); skipping") + return False + webhook = _get_webhook_url() + if not webhook: + logger.debug("ops_alerts: no webhook configured; skipping") + return False + from backend.services.discord_service import get_discord_notification_service + + service = get_discord_notification_service(webhook_url=webhook) + return service.post_message(message, webhook_url=webhook) + except Exception as exc: # noqa: BLE001 - alerting must never raise + logger.warning("ops_alerts: failed to send alert: %s", exc) + return False + + +def _signature(error_message: str) -> str: + """Stable fingerprint for an error string (reuses the error-monitor normalizer).""" + try: + from backend.services.error_monitor.normalizer import ( + compute_pattern_hash, + normalize_message, + ) + + return compute_pattern_hash("karaoke-job", normalize_message(error_message or "")) + except Exception: # noqa: BLE001 + # Fallback: hash the raw string so dedup still works if the normalizer + # import fails for any reason. + import hashlib + + return hashlib.sha256((error_message or "").encode("utf-8")).hexdigest() + + +def _should_alert(db: Any, signature: str, now: datetime) -> tuple[bool, bool, int, Any]: + """Decide whether to send, WITHOUT yet acknowledging delivery. + + Returns ``(should_send, is_novel, suppressed_since_last, doc_ref)``: + * ``should_send`` — emit a Discord alert this time. + * ``is_novel`` — we have never seen this signature before. + * ``suppressed_since_last`` — how many failures were collapsed (throttled) + since the last *successful* alert for this signature. + * ``doc_ref`` — the dedup doc, passed to :func:`_mark_alerted` **only after a + successful send** so a failed Discord delivery does NOT advance the + throttle window (which would suppress the next real failure). ``None`` if + Firestore was unreachable. + + This records ``first_seen`` / ``total_count`` (and, when throttled, the + suppressed counter) but deliberately does NOT touch ``last_alerted_at`` — that + is the delivery acknowledgement, set separately once the alert actually sends. + + Best-effort: on any Firestore error we default to *send* (fail open — an extra + alert is far better than a missed outage). NOTE: the ``get()`` + ``update()`` + are not transactional, so two truly-concurrent failures of the same signature + could each send once. That's an acceptable trade (a duplicate alert is + harmless; a missed one is not), so we don't pay for a transaction here. + """ + try: + doc_ref = db.collection(_DEDUP_COLLECTION).document(signature) + snap = doc_ref.get() + now_iso = now.isoformat() + if not snap.exists: + # Create the record but leave last_alerted_at unset — it's stamped by + # _mark_alerted only if the send succeeds. + doc_ref.set( + { + "signature": signature, + "first_seen": now_iso, + "last_alerted_at": None, + "total_count": 1, + "alert_count": 0, + "suppressed_since_last": 0, + } + ) + return True, True, 0, doc_ref + + data = snap.to_dict() or {} + last_alerted_raw = data.get("last_alerted_at") + within_window = False + if last_alerted_raw: + try: + last_alerted = datetime.fromisoformat(last_alerted_raw) + within_window = (now - last_alerted) < timedelta(minutes=_throttle_minutes()) + except (TypeError, ValueError): + within_window = False + + if within_window: + # Collapse: count it but don't re-page. + doc_ref.update( + { + "total_count": (data.get("total_count", 0) or 0) + 1, + "suppressed_since_last": (data.get("suppressed_since_last", 0) or 0) + 1, + } + ) + return False, False, 0, doc_ref + + doc_ref.update({"total_count": (data.get("total_count", 0) or 0) + 1}) + suppressed = data.get("suppressed_since_last", 0) or 0 + return True, False, suppressed, doc_ref + except Exception as exc: # noqa: BLE001 + logger.debug("ops_alerts: dedup lookup failed (%s); defaulting to send", exc) + return True, False, 0, None + + +def _mark_alerted(doc_ref: Any, now: datetime) -> None: + """Acknowledge a successful send: advance the throttle window + reset counters. + + Called ONLY after :func:`send_ops_alert` returns True. Best-effort. + """ + if doc_ref is None: + return + try: + snap = doc_ref.get() + data = snap.to_dict() or {} if snap.exists else {} + doc_ref.update( + { + "last_alerted_at": now.isoformat(), + "alert_count": (data.get("alert_count", 0) or 0) + 1, + "suppressed_since_last": 0, + } + ) + except Exception as exc: # noqa: BLE001 + logger.debug("ops_alerts: could not record alert delivery: %s", exc) + + +def notify_job_failed( + db: Any, + collection: str, + job_id: str, + message: Optional[str] = None, + additional_fields: Optional[dict] = None, +) -> bool: + """Emit a near-real-time alert for a job that just transitioned to FAILED. + + Invoked from the single status-write chokepoint + (:meth:`FirestoreService.update_job_status`). ``db`` is the caller's existing + Firestore client (reused to avoid a second client / circular import); + ``collection`` is the jobs collection so we can enrich the alert with the + job's artist/title/environment. Best-effort; never raises. + + Returns True if an alert was sent. + """ + try: + if not _is_enabled(): + return False + + # Enrich from the job doc (best-effort). We only just wrote it, so this + # is a single extra read on the rare FAILED path. + artist = title = environment = brand_code = None + error_message = (additional_fields or {}).get("error_message") or message or "" + try: + snap = db.collection(collection).document(job_id).get() + if snap.exists: + job = snap.to_dict() or {} + artist = job.get("artist") + title = job.get("title") + meta = job.get("request_metadata") or {} + environment = meta.get("environment") + state = job.get("state_data") or {} + brand_code = state.get("brand_code") + if not error_message: + error_message = job.get("error_message") or "" + except Exception as exc: # noqa: BLE001 + logger.debug("ops_alerts: job enrichment read failed: %s", exc) + + if environment and environment.lower() in _skip_environments(): + logger.debug("ops_alerts: skipping failure alert for %s env job %s", environment, job_id) + return False + + now = datetime.now(timezone.utc) + signature = _signature(error_message) + should_send, is_novel, suppressed, doc_ref = _should_alert(db, signature, now) + if not should_send: + return False + + track = " - ".join(p for p in (artist, title) if p) or "(unknown track)" + lines = [ + "🔴 **Job failed** — near-real-time alert", + f"**Job:** `{job_id}`" + (f" ({brand_code})" if brand_code else ""), + f"**Track:** {track}", + ] + if environment: + lines.append(f"**Env:** {environment}") + if is_novel: + lines.append("🆕 **First time we've ever seen this error signature.**") + elif suppressed: + lines.append(f"_(Collapsed {suppressed} earlier occurrence(s) of this signature.)_") + err = (error_message or "").strip() + if err: + if len(err) > 1500: + err = err[:1500] + " …[truncated]" + lines.append(f"**Error:** ```{err}```") + + sent = send_ops_alert("\n".join(lines)) + # Only advance the throttle window once delivery actually succeeded — a + # failed send must NOT suppress the next occurrence of this signature. + if sent: + _mark_alerted(doc_ref, now) + return sent + except Exception as exc: # noqa: BLE001 - alerting must never raise + logger.warning("ops_alerts: notify_job_failed failed: %s", exc) + return False diff --git a/backend/services/publish_completeness.py b/backend/services/publish_completeness.py new file mode 100644 index 00000000..1abf4e01 --- /dev/null +++ b/backend/services/publish_completeness.py @@ -0,0 +1,72 @@ +"""Publish-boundary completeness invariant (incident-hardening G1), shadow-first. + +Incident NOMAD-1632 published a public track to Google Drive **missing its 720p +variant** while the pipeline reported success: ``GCEEncodingBackend.encode()`` +misclassified ``… (Final Karaoke Lossy 720p).mp4`` (title contained "Portrait") +so ``mp4_720p`` was never set, and ``upload_to_public_share`` *silently skipped* +the ``None`` path. Nothing inline noticed; the daily validator caught it ~24h +later. + +This module computes, at the publish boundary, the set of outputs a public +GDrive release *should* have shipped and compares it against what was actually +distributed (``result.gdrive_files``). The orchestrator logs + alerts on any +shortfall **in shadow mode** (never fails the job) — the right altitude for the +check (publish-aware of what *should* ship) and the rollout discipline the +original fix skipped. + +The compute function is intentionally pure (config + result in, list of missing +outputs out) so it is trivially unit-testable and side-effect free. +""" + +from __future__ import annotations + +from typing import Any + +# gdrive_files keys produced by GDriveService.upload_to_public_share(), mapped to +# a human-readable label for alerts. +_LOSSY_4K_KEY = "mp4" +_720P_KEY = "mp4_720p" +_CDG_KEY = "cdg" + +_LABELS = { + _LOSSY_4K_KEY: "lossy 4K MP4", + _720P_KEY: "720p MP4", + _CDG_KEY: "CDG zip", +} + + +def expected_gdrive_outputs(config: Any, result: Any) -> list[str]: + """The gdrive_files keys a public-share release is expected to contain. + + Always the lossy 4K MP4 and the 720p MP4. The CDG zip is expected only when + the job actually **produced** a CDG package (``result.final_karaoke_cdg_zip`` + is set). Keying CDG off ``config.enable_cdg`` alone false-fires on legitimate + skips: a lyric-less track (``_lrc_has_lyrics_content()`` False) is packaged + without a CDG even when ``enable_cdg`` is True, so the upload omits ``cdg`` by + design. Basing the expectation on the produced artifact still catches the real + fault we care about — a CDG that WAS made but never distributed. + """ + expected = [_LOSSY_4K_KEY, _720P_KEY] + if getattr(result, "final_karaoke_cdg_zip", None): + expected.append(_CDG_KEY) + return expected + + +def compute_publish_shortfall(config: Any, result: Any) -> list[str]: + """Return human-readable labels for expected-but-missing public outputs. + + Empty list == complete (or not a public-share release). Only meaningful for + the public GDrive distribution path (``config.gdrive_folder_id`` set); returns + ``[]`` for any other job so callers can invoke it unconditionally. + """ + if not getattr(config, "gdrive_folder_id", None): + return [] + + distributed = getattr(result, "gdrive_files", None) or {} + missing: list[str] = [] + for key in expected_gdrive_outputs(config, result): + # A key is satisfied only if present AND truthy (an empty id/url means the + # upload didn't actually land). + if not distributed.get(key): + missing.append(_LABELS.get(key, key)) + return missing diff --git a/backend/services/worker_service.py b/backend/services/worker_service.py index 680a7c52..83020c1f 100644 --- a/backend/services/worker_service.py +++ b/backend/services/worker_service.py @@ -807,7 +807,9 @@ class WorkerService: async def schedule_gdrive_validation( self, - delay_seconds: int = GDRIVE_VALIDATION_DELAY_SECONDS + delay_seconds: int = GDRIVE_VALIDATION_DELAY_SECONDS, + brand_code: Optional[str] = None, + expect_cdg: bool = False, ) -> bool: """ Schedule a delayed GDrive validation check. @@ -817,6 +819,9 @@ class WorkerService: Args: delay_seconds: Delay before executing the validation (default: 5 minutes) + brand_code: Optional brand code of the just-published track. When set, + the validator additionally checks this track is present in all + expected folders same-run (incident-hardening D2). Returns: True if task was scheduled successfully, False otherwise @@ -864,13 +869,22 @@ class WorkerService: dispatch_deadline_seconds = WORKER_DISPATCH_DEADLINES.get("gdrive-validation", 120) + # Carry the brand code so the validator can check this track's own + # completeness same-run (incident-hardening D2). Absent → "{}". + # expect_cdg tells the validator whether to require the CDG folder. + body_bytes = b"{}" + if brand_code: + body_bytes = json.dumps( + {"brand_code": brand_code, "expect_cdg": expect_cdg} + ).encode("utf-8") + # Build task payload task = { "http_request": { "http_method": tasks_v2.HttpMethod.POST, "url": f"{self._base_url}/api/internal/trigger-gdrive-validation", "headers": headers, - "body": b"{}", + "body": body_bytes, "oidc_token": { "service_account_email": f"karaoke-backend@{project}.iam.gserviceaccount.com", }, diff --git a/backend/tests/test_discord_service.py b/backend/tests/test_discord_service.py index 74ec4ce0..69f0c70d 100644 --- a/backend/tests/test_discord_service.py +++ b/backend/tests/test_discord_service.py @@ -98,7 +98,7 @@ class TestDiscordNotificationServicePostMessage: assert result is True mock_post.assert_called_once_with( url, - json={"content": "Test message"}, + json={"content": "Test message", "allowed_mentions": {"parse": []}}, timeout=30 ) @@ -117,7 +117,7 @@ class TestDiscordNotificationServicePostMessage: assert result is True mock_post.assert_called_once_with( custom_url, - json={"content": "Test message"}, + json={"content": "Test message", "allowed_mentions": {"parse": []}}, timeout=30 ) diff --git a/backend/tests/test_ops_alerts.py b/backend/tests/test_ops_alerts.py new file mode 100644 index 00000000..a0eb4bfa --- /dev/null +++ b/backend/tests/test_ops_alerts.py @@ -0,0 +1,190 @@ +"""Unit tests for near-real-time failure alerting (incident-hardening D1). + +Covers the universal-net behaviour the spike-based error monitor can't provide: +novel error signatures always alert, repeats within the window collapse, test/dev +failures are skipped, and — critically — nothing here ever raises into the status +write that triggers it. +""" + +from types import SimpleNamespace + +import pytest + +from backend.services import ops_alerts + + +# --------------------------------------------------------------------------- +# In-memory Firestore test double +# --------------------------------------------------------------------------- + +class _FakeSnap: + def __init__(self, data): + self._data = data + self.exists = data is not None + + def to_dict(self): + return dict(self._data) if self._data is not None else None + + +class _FakeDoc: + def __init__(self, store, key): + self._store = store + self._key = key + + def get(self): + return _FakeSnap(self._store.get(self._key)) + + def set(self, data): + self._store[self._key] = dict(data) + + def update(self, data): + self._store.setdefault(self._key, {}).update(data) + + +class _FakeCollection: + def __init__(self, store): + self._store = store + + def document(self, doc_id): + return _FakeDoc(self._store, doc_id) + + +class FakeFirestore: + """Mimics the subset of google.cloud.firestore.Client used by ops_alerts.""" + + def __init__(self, jobs=None): + self._collections = {} + # Seed the jobs collection so enrichment reads succeed. + jobs_store = {} + for job_id, doc in (jobs or {}).items(): + jobs_store[job_id] = doc + self._collections["jobs"] = jobs_store + + def collection(self, name): + return _FakeCollection(self._collections.setdefault(name, {})) + + +@pytest.fixture +def enabled(monkeypatch): + monkeypatch.setenv("FAILURE_ALERTS_ENABLED", "true") + # Deterministic throttle window for tests. + monkeypatch.setenv("FAILURE_ALERT_THROTTLE_MINUTES", "30") + monkeypatch.delenv("FAILURE_ALERT_SKIP_ENVIRONMENTS", raising=False) + + +@pytest.fixture +def captured(monkeypatch): + """Capture send_ops_alert calls instead of hitting Discord.""" + sent = [] + monkeypatch.setattr(ops_alerts, "send_ops_alert", lambda msg: sent.append(msg) or True) + return sent + + +def _job(environment="production", artist="A", title="B", brand="NOMAD-1632", error_message="boom"): + return { + "artist": artist, + "title": title, + "request_metadata": {"environment": environment}, + "state_data": {"brand_code": brand}, + "error_message": error_message, + } + + +# --------------------------------------------------------------------------- +# send_ops_alert gating +# --------------------------------------------------------------------------- + +def test_send_ops_alert_disabled_returns_false(monkeypatch): + monkeypatch.delenv("FAILURE_ALERTS_ENABLED", raising=False) + monkeypatch.setattr(ops_alerts, "_is_enabled", lambda: False) + assert ops_alerts.send_ops_alert("hi") is False + + +def test_send_ops_alert_no_webhook_returns_false(monkeypatch): + monkeypatch.setattr(ops_alerts, "_is_enabled", lambda: True) + monkeypatch.setattr(ops_alerts, "_get_webhook_url", lambda: None) + assert ops_alerts.send_ops_alert("hi") is False + + +# --------------------------------------------------------------------------- +# notify_job_failed +# --------------------------------------------------------------------------- + +def test_novel_signature_alerts(enabled, captured): + db = FakeFirestore(jobs={"job-1": _job(error_message="never seen this")}) + sent = ops_alerts.notify_job_failed(db, "jobs", "job-1", message="never seen this") + assert sent is True + assert len(captured) == 1 + assert "First time" in captured[0] + assert "job-1" in captured[0] + + +def test_repeat_within_window_is_suppressed(enabled, captured): + db = FakeFirestore(jobs={ + "job-1": _job(), + "job-2": _job(), + }) + # Same error string -> same signature. + ops_alerts.notify_job_failed(db, "jobs", "job-1", additional_fields={"error_message": "same err"}) + ops_alerts.notify_job_failed(db, "jobs", "job-2", additional_fields={"error_message": "same err"}) + assert len(captured) == 1 # second collapsed + + +def test_distinct_signatures_each_alert(enabled, captured): + db = FakeFirestore(jobs={"j1": _job(), "j2": _job()}) + ops_alerts.notify_job_failed(db, "jobs", "j1", additional_fields={"error_message": "error one"}) + ops_alerts.notify_job_failed(db, "jobs", "j2", additional_fields={"error_message": "totally different"}) + assert len(captured) == 2 + + +def test_test_environment_skipped(enabled, captured): + db = FakeFirestore(jobs={"job-t": _job(environment="test")}) + sent = ops_alerts.notify_job_failed(db, "jobs", "job-t", message="boom") + assert sent is False + assert captured == [] + + +def test_disabled_environment_no_alert(monkeypatch, captured): + monkeypatch.delenv("FAILURE_ALERTS_ENABLED", raising=False) + monkeypatch.setattr(ops_alerts, "_is_enabled", lambda: False) + db = FakeFirestore(jobs={"job-1": _job()}) + assert ops_alerts.notify_job_failed(db, "jobs", "job-1", message="boom") is False + assert captured == [] + + +def test_best_effort_never_raises_on_broken_db(enabled, captured): + class BrokenDB: + def collection(self, name): + raise RuntimeError("firestore exploded") + + # Must not propagate. Enrichment + dedup both fail, so it fails OPEN and + # still sends (missing a real outage is worse than an unenriched alert). + result = ops_alerts.notify_job_failed(BrokenDB(), "jobs", "job-x", message="boom") + assert result is True + assert len(captured) == 1 + + +def test_fail_open_when_dedup_read_errors(enabled, captured, monkeypatch): + """If the dedup lookup fails we should still alert (missing an outage is worse).""" + db = FakeFirestore(jobs={"job-1": _job()}) + monkeypatch.setattr(ops_alerts, "_should_alert", lambda *a, **k: (True, False, 0, None)) + assert ops_alerts.notify_job_failed(db, "jobs", "job-1", message="boom") is True + assert len(captured) == 1 + + +def test_failed_delivery_does_not_suppress_next(enabled, monkeypatch): + """A failed Discord send must NOT advance the throttle window.""" + db = FakeFirestore(jobs={"job-1": _job(), "job-2": _job()}) + calls = [] + + def flaky_send(msg): + calls.append(msg) + return len(calls) > 1 # first send fails, second succeeds + + monkeypatch.setattr(ops_alerts, "send_ops_alert", flaky_send) + # First failure: send attempted but delivery fails → not acked. + assert ops_alerts.notify_job_failed(db, "jobs", "job-1", additional_fields={"error_message": "same err"}) is False + # Second failure, same signature, within window: because the first never + # acked, this must still be attempted (not throttled away) and now succeed. + assert ops_alerts.notify_job_failed(db, "jobs", "job-2", additional_fields={"error_message": "same err"}) is True + assert len(calls) == 2 diff --git a/backend/tests/test_publish_completeness.py b/backend/tests/test_publish_completeness.py new file mode 100644 index 00000000..fa031854 --- /dev/null +++ b/backend/tests/test_publish_completeness.py @@ -0,0 +1,93 @@ +"""Unit tests for the publish-boundary completeness invariant (incident-hardening G1). + +These pin the exact behaviour that closes the NOMAD-1632 silent-partial class: a +public GDrive release missing its 720p variant (or any expected output) must be +reported as a shortfall, while a complete release — and any non-public job — +reports nothing. +""" + +from types import SimpleNamespace + +from backend.services.publish_completeness import ( + compute_publish_shortfall, + expected_gdrive_outputs, +) + + +def _config(gdrive_folder_id="folder-1", enable_cdg=True): + return SimpleNamespace( + gdrive_folder_id=gdrive_folder_id, + enable_cdg=enable_cdg, + artist="Nat King Cole", + title="Portrait of Jennie", + ) + + +def _result(gdrive_files, cdg_produced=True): + """A result whose CDG was produced by default (final_karaoke_cdg_zip set).""" + return SimpleNamespace( + gdrive_files=gdrive_files, + brand_code="NOMAD-1632", + final_karaoke_cdg_zip=("/tmp/x.zip" if cdg_produced else None), + ) + + +class TestExpectedOutputs: + def test_cdg_produced_expects_all_three(self): + res = _result({}, cdg_produced=True) + assert expected_gdrive_outputs(_config(enable_cdg=True), res) == ["mp4", "mp4_720p", "cdg"] + + def test_cdg_not_produced_expects_two(self): + res = _result({}, cdg_produced=False) + assert expected_gdrive_outputs(_config(enable_cdg=True), res) == ["mp4", "mp4_720p"] + + +class TestComputePublishShortfall: + def test_complete_release_no_shortfall(self): + cfg = _config(enable_cdg=True) + res = _result({"mp4": "id1", "mp4_720p": "id2", "cdg": "id3"}) + assert compute_publish_shortfall(cfg, res) == [] + + def test_missing_720p_is_the_incident(self): + """The NOMAD-1632 case: 4K + CDG shipped, 720p silently skipped.""" + cfg = _config(enable_cdg=True) + res = _result({"mp4": "id1", "cdg": "id3"}) + assert compute_publish_shortfall(cfg, res) == ["720p MP4"] + + def test_missing_cdg_when_produced(self): + """CDG was produced but not distributed → real shortfall.""" + cfg = _config(enable_cdg=True) + res = _result({"mp4": "id1", "mp4_720p": "id2"}, cdg_produced=True) + assert compute_publish_shortfall(cfg, res) == ["CDG zip"] + + def test_lyricless_skip_does_not_require_cdg(self): + """enable_cdg True but no CDG produced (lyric-less track) → no false alarm.""" + cfg = _config(enable_cdg=True) + res = _result({"mp4": "id1", "mp4_720p": "id2"}, cdg_produced=False) + assert compute_publish_shortfall(cfg, res) == [] + + def test_cdg_not_expected_when_disabled(self): + cfg = _config(enable_cdg=False) + res = _result({"mp4": "id1", "mp4_720p": "id2"}, cdg_produced=False) + assert compute_publish_shortfall(cfg, res) == [] + + def test_empty_distribution_lists_all_expected(self): + cfg = _config(enable_cdg=True) + res = _result({}, cdg_produced=True) + assert compute_publish_shortfall(cfg, res) == ["lossy 4K MP4", "720p MP4", "CDG zip"] + + def test_empty_string_id_counts_as_missing(self): + cfg = _config(enable_cdg=False) + res = _result({"mp4": "id1", "mp4_720p": ""}, cdg_produced=False) + assert compute_publish_shortfall(cfg, res) == ["720p MP4"] + + def test_non_public_job_never_flags(self): + """No gdrive_folder_id → not a public-share release → always clean.""" + cfg = _config(gdrive_folder_id=None, enable_cdg=True) + res = _result({}) + assert compute_publish_shortfall(cfg, res) == [] + + def test_none_gdrive_files_is_safe(self): + cfg = _config(enable_cdg=False) + res = _result(None, cdg_produced=False) + assert compute_publish_shortfall(cfg, res) == ["lossy 4K MP4", "720p MP4"] diff --git a/backend/tests/test_services.py b/backend/tests/test_services.py index 7320e0f8..734d2f84 100644 --- a/backend/tests/test_services.py +++ b/backend/tests/test_services.py @@ -825,12 +825,56 @@ class TestFirestoreService: assert job is None + def test_failed_status_triggers_failure_alert(self): + """Incident-hardening D1: a transition to FAILED fires the failure alert.""" + with patch('backend.services.firestore_service.firestore') as mock_firestore: + mock_client = MagicMock() + mock_firestore.Client.return_value = mock_client + mock_collection = MagicMock() + mock_client.collection.return_value = mock_collection + mock_collection.document.return_value = MagicMock() + + firestore_service = FirestoreService() + with patch('backend.services.ops_alerts.notify_job_failed') as mock_notify: + firestore_service.update_job_status("job1", JobStatus.FAILED, message="boom") + mock_notify.assert_called_once() + # Reuses the caller's client + collection (no second Firestore client). + _, kwargs = mock_notify.call_args + assert kwargs["job_id"] == "job1" + assert kwargs["db"] is mock_client + + def test_non_failed_status_does_not_alert(self): + """A normal status transition must NOT fire the failure alert.""" + with patch('backend.services.firestore_service.firestore') as mock_firestore: + mock_client = MagicMock() + mock_firestore.Client.return_value = mock_client + mock_client.collection.return_value.document.return_value = MagicMock() + + firestore_service = FirestoreService() + with patch('backend.services.ops_alerts.notify_job_failed') as mock_notify: + firestore_service.update_job_status("job1", JobStatus.COMPLETE, message="done") + mock_notify.assert_not_called() + + def test_failure_alert_error_does_not_break_status_write(self): + """If the alerter raises, the status write must still succeed (best-effort).""" + with patch('backend.services.firestore_service.firestore') as mock_firestore: + mock_client = MagicMock() + mock_firestore.Client.return_value = mock_client + mock_doc_ref = MagicMock() + mock_client.collection.return_value.document.return_value = mock_doc_ref + + firestore_service = FirestoreService() + with patch('backend.services.ops_alerts.notify_job_failed', side_effect=RuntimeError("x")): + # Should not raise. + firestore_service.update_job_status("job1", JobStatus.FAILED, message="boom") + mock_doc_ref.update.assert_called_once() + def test_update_job(self): """Test updating a job in Firestore.""" with patch('backend.services.firestore_service.firestore') as mock_firestore: mock_client = MagicMock() mock_firestore.Client.return_value = mock_client - + mock_collection = MagicMock() mock_client.collection.return_value = mock_collection diff --git a/backend/workers/video_worker_orchestrator.py b/backend/workers/video_worker_orchestrator.py index befb3509..ab956c73 100644 --- a/backend/workers/video_worker_orchestrator.py +++ b/backend/workers/video_worker_orchestrator.py @@ -720,6 +720,49 @@ class VideoWorkerOrchestrator: }) self.job_log.info("Cleared outputs_deleted_at flag (job was re-processed)") + # Incident-hardening G1 (SHADOW): before a public release is considered + # done, assert the outputs it *should* have shipped (lossy 4K + 720p, plus + # CDG when enabled) actually landed on the public GDrive share. This closes + # the residual silent-partial hole — e.g. a per-file download failure in + # _download_gce_encoded_files clears one result attr, so the upload skips a + # None path and publishes a partial release without the encode() guard ever + # tripping (exactly the NOMAD-1632 class). Shadow-first: log + alert only, + # never fail the job, until a clean shadow window justifies enforcing. + self._check_publish_completeness() + + def _check_publish_completeness(self): + """SHADOW check that a public-share release shipped all expected outputs.""" + try: + from backend.services.publish_completeness import compute_publish_shortfall + + missing = compute_publish_shortfall(self.config, self.result) + if not missing: + return + + missing_str = ", ".join(missing) + self.job_log.error( + f"PUBLISH_COMPLETENESS shortfall (SHADOW) for job {self.config.job_id}: " + f"public release is missing {missing_str}. gdrive_files=" + f"{self.result.gdrive_files}" + ) + try: + from backend.services.ops_alerts import send_ops_alert + + brand = self.result.brand_code or "(no brand)" + track = f"{self.config.artist} - {self.config.title}" + send_ops_alert( + "🟠 **Publish completeness SHADOW** — partial public release detected\n" + f"**Job:** `{self.config.job_id}` ({brand})\n" + f"**Track:** {track}\n" + f"**Missing:** {missing_str}\n" + "_Shadow mode: the job was NOT failed. Investigate the " + "encode→publish path (silent-partial class, NOMAD-1632)._" + ) + except Exception as alert_exc: # noqa: BLE001 + self.job_log.warning(f"Publish-completeness alert did not send: {alert_exc}") + except Exception as e: # noqa: BLE001 - shadow check must never break distribution + self.job_log.warning(f"Publish-completeness check skipped (non-fatal): {e}") + async def _upload_to_youtube(self): """Upload video to YouTube.""" self.job_log.info("Uploading to YouTube") @@ -1047,14 +1090,22 @@ class VideoWorkerOrchestrator: try: from backend.services.worker_service import get_worker_service worker_service = get_worker_service() - success = await worker_service.schedule_gdrive_validation() + # Name the just-published track so the validator can check *its own* + # completeness same-run (incident-hardening D2), not just global gaps. + # Only require CDG when this job actually produced one (video-only / + # lyric-less releases legitimately ship no CDG). + brand_code = self.result.brand_code + expect_cdg = bool(self.result.final_karaoke_cdg_zip) + success = await worker_service.schedule_gdrive_validation( + brand_code=brand_code, expect_cdg=expect_cdg + ) if success: self.job_log.info("Scheduled delayed GDrive validation (5 min)") else: # Fallback: trigger immediately if scheduling failed self.job_log.warning("Cloud Tasks scheduling failed, triggering validation immediately") from backend.services.gdrive_validator_client import trigger_gdrive_validation - trigger_gdrive_validation() + trigger_gdrive_validation(brand_code=brand_code, expect_cdg=expect_cdg) except Exception as e: self.job_log.warning(f"GDrive validation trigger failed (non-fatal): {e}") # Never fail the pipeline for validation diff --git a/docs/GDRIVE-VALIDATOR.md b/docs/GDRIVE-VALIDATOR.md index 86b57a8a..8f379cf9 100644 --- a/docs/GDRIVE-VALIDATOR.md +++ b/docs/GDRIVE-VALIDATOR.md @@ -2,6 +2,15 @@ Automated validation of the Nomad Karaoke public Google Drive share folder. Detects duplicate brand codes, invalid filenames, and sequence gaps. +**Same-run partial-publish check (incident-hardening D2):** when the post-job trigger +names the just-published `brand_code` (passed orchestrator → Cloud Tasks → internal +endpoint → Cloud Function), the validator additionally verifies *that specific track* +is present in all expected folders (MP4, MP4-720p, CDG) — independent of the global-max +gap logic, which only notices a missing latest track once a *later* track raises the +max. This surfaces a silent partial publish (e.g. NOMAD-1632's missing 720p variant) +same-run (~5 min post-job) as a `missing_for_track` issue, instead of up to 24h later. +The `brand_code` is optional; absent → full-scan behaviour as before. + ## Overview The public share folder (`1laRKAyxo0v817SstfM5XkpbWiNKNAMSX`) contains all published karaoke tracks organized into subfolders: diff --git a/docs/LESSONS-LEARNED.md b/docs/LESSONS-LEARNED.md index 0fb6dd3b..d0142871 100644 --- a/docs/LESSONS-LEARNED.md +++ b/docs/LESSONS-LEARNED.md @@ -1853,3 +1853,33 @@ carries the flag; the email now deep-links to item** — verify the deep-link, don't assume "/app shows everything". - Any field a frontend filter branches on must be in `SUMMARY_FIELD_PATHS`, or it arrives `undefined` and the branch silently takes the wrong path. + +--- + +## Spike-based alerting is blind to novel low-volume failures (NOMAD-1632 follow-up, v0.223.0) + +**Context:** The production error monitor only alerts on a *spike* — `current_count` +must exceed a rolling average by a multiplier and clear a min-count floor. A brand-new +error has `rolling_avg == 0`, so it is mathematically never a spike. When a fix's new +guard over-fired and failed exactly 2 real jobs with a never-before-seen error string, +nothing paged; Andrew noticed failed jobs in the UI ~a day's-worth of risk later. + +**Fix (D1):** a **universal net** independent of the spike detector — +`backend/services/ops_alerts.py` hooks the single status-write chokepoint +(`FirestoreService.update_job_status`) on any transition to `FAILED` and sends an +immediate Discord alert, de-duped by a normalized error signature; **novel signatures +always alert** regardless of count. Best-effort (never raises into the status write), +gated to deployed envs, skips test/dev. + +**Principles for next time:** +- Spike/threshold detectors are for *volume regressions*, not *existence* of a new + failure. Always pair them with a per-event net on the terminal failure transition. +- Put the net at the **single chokepoint** every path funnels through (here the + Firestore status write), not at each caller — that's what makes it universal and + future-proof. +- A new *enforcement* guard is itself a change that can fail: ship it **shadow-first** + (log + alert, don't fail — see G1 `publish_completeness.py`), distinguish + "recoverable/empty" from "wrong result", and fail **open** on uncertainty. +- Detect silent *partial success* at the **publish boundary** (what *should* have + shipped vs what did), and make the metadata backstop validate a just-published + track's **own** completeness same-run, not only via global-max gap math (D2). diff --git a/infrastructure/functions/gdrive_validator/main.py b/infrastructure/functions/gdrive_validator/main.py index dcfdb93f..9b4c3357 100644 --- a/infrastructure/functions/gdrive_validator/main.py +++ b/infrastructure/functions/gdrive_validator/main.py @@ -268,12 +268,57 @@ def validate_files(files_by_folder: dict[str, list[str]]) -> dict: return issues +# Folders every public release must appear in. MP4 (lossy 4K) and MP4-720p are +# always produced; CDG is conditional (a lyric-less / video-only release ships no +# CDG package), so it is only required when the caller says the job produced one. +BASE_TRACK_FOLDERS = ('MP4', 'MP4-720p') + + +def check_track_completeness( + files_by_folder: dict[str, list[str]], + brand_code: str, + expect_cdg: bool = False, +) -> list[str]: + """Return the folders a specific brand code is MISSING from. + + Incident-hardening D2: when the post-job trigger names the just-published + ``brand_code`` (e.g. "NOMAD-1632"), verify that track is present in every + expected folder — *independent of the global-max gap logic*, which only + notices a missing latest track once a later track raises the max. This fires + same-run (5 min post-job) instead of up to 24h later. + + MP4 and MP4-720p are always required. CDG is required only when + ``expect_cdg`` is True (the caller passes this when the job actually produced + a CDG package); otherwise a legitimate video-only release would false-fire. + + Matching is by the ``NOMAD-#### `` filename prefix, so it ignores the + (differing) artist/title tail. + """ + # Normalize "1632" / "NOMAD-1632" / "nomad-1632" to the "NOMAD-1632 " prefix. + code = str(brand_code).strip().upper() + if not code.startswith("NOMAD-"): + code = f"NOMAD-{code}" + prefix = f"{code} " + + expected_folders = list(BASE_TRACK_FOLDERS) + if expect_cdg: + expected_folders.append('CDG') + + missing_from = [] + for folder in expected_folders: + files = files_by_folder.get(folder, []) + if not any(f.startswith(prefix) for f in files): + missing_from.append(folder) + return missing_from + + def has_issues(issues: dict) -> bool: """Check if there are any validation issues.""" return ( - bool(issues['duplicates']) or + bool(issues['duplicates']) or bool(issues['invalid_filenames']) or - bool(issues['gaps']) + bool(issues['gaps']) or + bool(issues.get('missing_for_track')) ) @@ -283,7 +328,16 @@ def format_notification(issues: dict) -> tuple[str, str]: lines = [] issue_count = 0 - + + # Missing-for-track (same-run, per just-published brand code) — most urgent, + # list first so a silent partial publish is the headline. + if issues.get('missing_for_track'): + lines.append("🚨 PARTIAL PUBLISH (just-published track missing from folder(s)):") + for code, folders in issues['missing_for_track'].items(): + issue_count += 1 + lines.append(f" • {code}: missing from {', '.join(folders)}") + lines.append("") + # Duplicates if issues['duplicates']: lines.append("DUPLICATES:") @@ -423,24 +477,51 @@ def validate_gdrive(request): Returns JSON with validation results. """ logger.info("Starting Google Drive validation...") - + + # Incident-hardening D2: the post-job trigger may name the just-published + # track so we can verify *its own* completeness same-run. Accept brand_code + # from the JSON body or a query param; absent → full-scan behaviour as before. + brand_code = None + expect_cdg = False + try: + body = request.get_json(silent=True) if hasattr(request, "get_json") else None + if isinstance(body, dict): + brand_code = body.get("brand_code") + expect_cdg = bool(body.get("expect_cdg", False)) + if not brand_code and hasattr(request, "args"): + brand_code = request.args.get("brand_code") + if request.args.get("expect_cdg") is not None: + expect_cdg = request.args.get("expect_cdg") in ("1", "true", "True") + except Exception: # noqa: BLE001 - request parsing must not break validation + brand_code = None + expect_cdg = False + try: # Get Drive service service = get_drive_service() - + # List all files files_by_folder = list_folder_contents(service, GDRIVE_FOLDER_ID) - + if not files_by_folder: logger.warning("No folders found in Google Drive") return json.dumps({ "status": "warning", "message": "No folders found in Google Drive folder" }), 200, {"Content-Type": "application/json"} - + # Validate files issues = validate_files(files_by_folder) - + + # Targeted same-run completeness for the just-published track. + if brand_code: + missing_from = check_track_completeness(files_by_folder, brand_code, expect_cdg=expect_cdg) + if missing_from: + issues['missing_for_track'] = {str(brand_code): missing_from} + logger.warning( + "Track %s missing from folder(s): %s", brand_code, missing_from + ) + # Check if there are any issues if has_issues(issues): logger.warning(f"Validation issues found: {issues}") diff --git a/infrastructure/functions/gdrive_validator/test_main.py b/infrastructure/functions/gdrive_validator/test_main.py index eaf0abbd..63a3d01d 100644 --- a/infrastructure/functions/gdrive_validator/test_main.py +++ b/infrastructure/functions/gdrive_validator/test_main.py @@ -30,6 +30,7 @@ from main import ( # noqa: E402 get_sequence_number, expand_gaps, has_issues, + check_track_completeness, ) @@ -83,6 +84,66 @@ class TestValidateFilenameFormat: assert validate_filename_format(".gitkeep") +class TestCheckTrackCompleteness: + """Incident-hardening D2: same-run per-brand-code completeness.""" + + def _folders(self, present): + """Build files_by_folder where `present` is a set of folder names that + contain NOMAD-1632 (all folders also carry an unrelated track).""" + out = {} + for folder, ext in (("MP4", "mp4"), ("MP4-720p", "mp4"), ("CDG", "zip")): + files = [f"NOMAD-1600 - Other - Song.{ext}"] + if folder in present: + files.append(f"NOMAD-1632 - Artist - Title.{ext}") + out[folder] = files + return out + + def test_present_everywhere_no_missing(self): + files = self._folders({"MP4", "MP4-720p", "CDG"}) + assert check_track_completeness(files, "NOMAD-1632", expect_cdg=True) == [] + + def test_missing_from_720p_is_the_nomad1632_case(self): + files = self._folders({"MP4", "CDG"}) # 720p missing — the real incident + assert check_track_completeness(files, "NOMAD-1632", expect_cdg=True) == ["MP4-720p"] + + def test_missing_from_multiple(self): + files = self._folders({"MP4"}) + assert check_track_completeness(files, "NOMAD-1632", expect_cdg=True) == ["MP4-720p", "CDG"] + + def test_cdg_not_required_by_default(self): + """Video-only release (no CDG produced) must NOT report CDG missing.""" + files = self._folders({"MP4", "MP4-720p"}) # no CDG + assert check_track_completeness(files, "NOMAD-1632") == [] + assert check_track_completeness(files, "NOMAD-1632", expect_cdg=False) == [] + + def test_missing_cdg_only_when_expected(self): + files = self._folders({"MP4", "MP4-720p"}) # no CDG + assert check_track_completeness(files, "NOMAD-1632", expect_cdg=True) == ["CDG"] + + def test_accepts_bare_number(self): + files = self._folders({"MP4", "MP4-720p", "CDG"}) + assert check_track_completeness(files, "1632", expect_cdg=True) == [] + + def test_accepts_lowercase(self): + files = self._folders({"MP4", "CDG"}) + assert check_track_completeness(files, "nomad-1632", expect_cdg=True) == ["MP4-720p"] + + def test_prefix_not_substring(self): + # NOMAD-163 must NOT satisfy a NOMAD-1632 check (prefix includes trailing space) + files = { + "MP4": ["NOMAD-1632 - A - B.mp4"], + "MP4-720p": ["NOMAD-1632 - A - B.mp4"], + "CDG": ["NOMAD-1632 - A - B.zip"], + } + assert check_track_completeness(files, "NOMAD-163", expect_cdg=True) == ["MP4", "MP4-720p", "CDG"] + + def test_has_issues_flags_missing_for_track(self): + assert has_issues({ + 'duplicates': {}, 'invalid_filenames': {}, 'gaps': {}, + 'missing_for_track': {'NOMAD-1632': ['MP4-720p']}, + }) + + class TestGetSequenceNumber: def test_normal(self): assert get_sequence_number("NOMAD-1234 - Artist - Title.mp4") == 1234 diff --git a/pyproject.toml b/pyproject.toml index b6fdbb45..7ffabc3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.223.1" +version = "0.223.2" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"