diff --git a/docs/GDRIVE-VALIDATOR.md b/docs/GDRIVE-VALIDATOR.md index 8f379cf9..282c1cf3 100644 --- a/docs/GDRIVE-VALIDATOR.md +++ b/docs/GDRIVE-VALIDATOR.md @@ -96,9 +96,10 @@ Notifications are sent when: | Variable | Value | Description | |----------|-------|-------------| | `GDRIVE_FOLDER_ID` | `1laRKAyxo0v817SstfM5XkpbWiNKNAMSX` | Root folder of public share | -| `NOTIFY_ON_SUCCESS` | `true` | Send daily "all clear" summary | +| `NOTIFY_ON_SUCCESS` | `false` (prod) | Send an "all clear" email on every clean run; off in prod because the post-job trigger runs after every job | | `EMAIL_TO` | `gen@nomadkaraoke.com` | Email recipient | | `EMAIL_FROM` | `gen@nomadkaraoke.com` | Email sender | +| `IN_FLIGHT_GRACE_MINUTES` | `30` (default) | Tracks with any file created in Drive this recently count as still uploading and are left out of the gap check | ### Secrets (Cloud Function) @@ -202,6 +203,16 @@ The post-job trigger uses a 5-minute Cloud Tasks delay (via `gdrive-validation-q **IMPORTANT: Never add new entries to `KNOWN_GAPS`.** The existing known gaps are historical (pre-generator era, over a year old). Every new gap is a real bug that must be investigated and fixed. +**Tracks still uploading are not gaps.** A job's packaging step uploads MP4, then +MP4-720p, then CDG over ~2 minutes, and another job's post-job trigger can run in that +window. On 2026-11-07 NOMAD-1871's check ran at 12:24:47 UTC while NOMAD-1872 was +uploading (12:23:39–12:25:40) and emailed "CDG/MP4-720p: missing 1872"; the files were +all there a minute later. Since then the validator reads each file's Drive `createdTime` +and leaves out of the gap check any track with a file created in the last +`IN_FLIGHT_GRACE_MINUTES` (30). That track is still checked by its own post-job trigger +(which names its `brand_code`) and by the daily 9 PM run. If you get a gap alert for the +newest track, first check whether that job was in `packaging` at the alert time. + **Investigation steps:** 1. **Identify the missing brand code(s)** from the alert (e.g., "MP4: missing 1295") diff --git a/infrastructure/functions/gdrive_validator/deploy.sh b/infrastructure/functions/gdrive_validator/deploy.sh index 4cbc16a2..225cc2bf 100755 --- a/infrastructure/functions/gdrive_validator/deploy.sh +++ b/infrastructure/functions/gdrive_validator/deploy.sh @@ -53,7 +53,7 @@ gcloud functions deploy "${FUNCTION_NAME}" \ --min-instances=0 \ --max-instances=1 \ --service-account="gdrive-validator@${PROJECT_ID}.iam.gserviceaccount.com" \ - --set-env-vars="GDRIVE_FOLDER_ID=1laRKAyxo0v817SstfM5XkpbWiNKNAMSX,EMAIL_TO=gen@nomadkaraoke.com,EMAIL_FROM=gen@nomadkaraoke.com,NOTIFY_ON_SUCCESS=true" \ + --set-env-vars="GDRIVE_FOLDER_ID=1laRKAyxo0v817SstfM5XkpbWiNKNAMSX,EMAIL_TO=gen@nomadkaraoke.com,EMAIL_FROM=gen@nomadkaraoke.com,NOTIFY_ON_SUCCESS=false,LOG_EXECUTION_ID=true" \ --set-secrets="PUSHBULLET_API_KEY=pushbullet-api-key:latest,POSTMARK_SERVER_TOKEN=postmark-server-token:latest" \ --project="${PROJECT_ID}" diff --git a/infrastructure/functions/gdrive_validator/main.py b/infrastructure/functions/gdrive_validator/main.py index 9b4c3357..3eedddf1 100644 --- a/infrastructure/functions/gdrive_validator/main.py +++ b/infrastructure/functions/gdrive_validator/main.py @@ -16,6 +16,7 @@ import re import json import logging from collections import defaultdict +from datetime import datetime, timedelta, timezone from typing import Union import functions_framework import requests @@ -38,6 +39,21 @@ EMAIL_FROM = os.environ.get("EMAIL_FROM", "gen@nomadkaraoke.com") # Set to "false" to only receive notifications when issues are detected NOTIFY_ON_SUCCESS = os.environ.get("NOTIFY_ON_SUCCESS", "true").lower() == "true" +# A track whose newest file landed in Drive this recently is treated as still +# uploading: a job's packaging step uploads MP4, then MP4-720p, then CDG over a +# couple of minutes, and the post-job trigger for an earlier job can fire right +# in the middle (2026-11-07: NOMAD-1871's check flagged NOMAD-1872 mid-upload). +# Gaps for such tracks are held back; the track's own post-job trigger (which +# names its brand_code) and the daily run still check it once the upload ends. +def _grace_minutes() -> int: + try: + return int(os.environ.get("IN_FLIGHT_GRACE_MINUTES", "30")) + except ValueError: + return 30 + + +IN_FLIGHT_GRACE_MINUTES = _grace_minutes() + # Known gaps configuration - matches the local validate_and_sync.py script # These are HISTORICAL ONLY (pre-generator era, before Dec 2024). Do NOT add new entries. # New gaps are bugs — see docs/GDRIVE-VALIDATOR.md § "Sequence Gap Detected" for fix procedure. @@ -93,7 +109,9 @@ def get_drive_service(): return build('drive', 'v3', credentials=credentials) -def list_folder_contents(service, folder_id: str) -> dict[str, list[str]]: +def list_folder_contents( + service, folder_id: str, created_times: dict[str, str] | None = None +) -> dict[str, list[str]]: """ List all files in a Google Drive folder and its subfolders. @@ -102,6 +120,9 @@ def list_folder_contents(service, folder_id: str) -> dict[str, list[str]]: - CDG/ (contains .zip files) - MP4/ (contains .mp4 files) - MP4-720p/ (contains .mp4 files) + + If ``created_times`` is given, it is filled with ``{filename: createdTime}`` + (RFC 3339) so the caller can tell which tracks are still being uploaded. """ results = {} @@ -134,12 +155,15 @@ def list_folder_contents(service, folder_id: str) -> dict[str, list[str]]: query = f"'{folder_id_inner}' in parents and trashed=false and mimeType!='application/vnd.google-apps.folder'" response = service.files().list( q=query, - fields="nextPageToken, files(name)", + fields="nextPageToken, files(name, createdTime)", pageSize=1000, pageToken=page_token ).execute() - files.extend([f['name'] for f in response.get('files', [])]) + for f in response.get('files', []): + files.append(f['name']) + if created_times is not None and f.get('createdTime'): + created_times[f['name']] = f['createdTime'] page_token = response.get('nextPageToken') if not page_token: @@ -187,10 +211,39 @@ def get_sequence_number(filename: str) -> int | None: return None -def validate_files(files_by_folder: dict[str, list[str]]) -> dict: +def recently_uploaded_sequences( + created_times: dict[str, str], + now: datetime | None = None, + grace_minutes: int = IN_FLIGHT_GRACE_MINUTES, +) -> set[int]: + """Sequence numbers that have any file created within the grace window.""" + now = now or datetime.now(timezone.utc) + cutoff = now - timedelta(minutes=grace_minutes) + recent = set() + for filename, created in created_times.items(): + seq = get_sequence_number(filename) + if seq is None: + continue + try: + ts = datetime.fromisoformat(created.replace("Z", "+00:00")) + except (TypeError, ValueError): + continue + if ts.tzinfo is None: + ts = ts.replace(tzinfo=timezone.utc) + if ts >= cutoff: + recent.add(seq) + return recent + + +def validate_files( + files_by_folder: dict[str, list[str]], in_flight: set[int] | None = None +) -> dict: """ Validate all files and return a dict of issues found. + ``in_flight`` sequence numbers (tracks still uploading) are not reported + as gaps. + Returns: { 'duplicates': {'MP4': {1107: ['file1.mp4', 'file2.mp4']}, ...}, @@ -255,7 +308,7 @@ def validate_files(files_by_folder: dict[str, list[str]]) -> dict: min_num = min(sequence_numbers) expected_range = set(range(min_num, global_max + 1)) excluded = EXCLUDED_NUMBERS.get(folder_name, set()) - missing = sorted(expected_range - sequence_numbers - excluded) + missing = sorted(expected_range - sequence_numbers - excluded - (in_flight or set())) if missing: issues['gaps'][folder_name] = missing @@ -501,7 +554,8 @@ def validate_gdrive(request): service = get_drive_service() # List all files - files_by_folder = list_folder_contents(service, GDRIVE_FOLDER_ID) + created_times: dict[str, str] = {} + files_by_folder = list_folder_contents(service, GDRIVE_FOLDER_ID, created_times) if not files_by_folder: logger.warning("No folders found in Google Drive") @@ -511,7 +565,10 @@ def validate_gdrive(request): }), 200, {"Content-Type": "application/json"} # Validate files - issues = validate_files(files_by_folder) + in_flight = recently_uploaded_sequences(created_times) + if in_flight: + logger.info("Holding back gap checks for tracks still uploading: %s", sorted(in_flight)) + issues = validate_files(files_by_folder, in_flight=in_flight) # Targeted same-run completeness for the just-published track. if brand_code: diff --git a/infrastructure/functions/gdrive_validator/test_main.py b/infrastructure/functions/gdrive_validator/test_main.py index 63a3d01d..9a3883ac 100644 --- a/infrastructure/functions/gdrive_validator/test_main.py +++ b/infrastructure/functions/gdrive_validator/test_main.py @@ -31,6 +31,7 @@ from main import ( # noqa: E402 expand_gaps, has_issues, check_track_completeness, + recently_uploaded_sequences, ) @@ -333,3 +334,35 @@ class TestHasIssues: def test_has_gaps(self): assert has_issues({"duplicates": {}, "invalid_filenames": {}, "gaps": {"MP4": [3]}}) + + +class TestInFlightUploads: + """2026-11-07: NOMAD-1871's post-job check ran while NOMAD-1872 was mid-upload + (MP4 done, 720p + CDG not yet) and emailed a false sequence-gap alert.""" + + NOW = __import__("datetime").datetime(2026, 11, 7, 12, 24, 47, tzinfo=__import__("datetime").timezone.utc) + + def _files(self): + return { + "MP4": [make_mp4(i) for i in range(100, 106)], + "MP4-720p": [make_mp4(i) for i in range(100, 105)], + "CDG": [make_cdg(i) for i in range(100, 105)], + } + + def test_track_still_uploading_is_not_a_gap(self): + created = {make_mp4(105): "2026-11-07T12:24:30.000Z", make_mp4(104): "2026-11-06T09:00:00.000Z"} + in_flight = recently_uploaded_sequences(created, now=self.NOW, grace_minutes=30) + assert in_flight == {105} + issues = validate_files(self._files(), in_flight=in_flight) + assert issues["gaps"] == {} + + def test_old_track_missing_is_still_a_gap(self): + created = {make_mp4(105): "2026-11-07T10:00:00.000Z"} + in_flight = recently_uploaded_sequences(created, now=self.NOW, grace_minutes=30) + assert in_flight == set() + issues = validate_files(self._files(), in_flight=in_flight) + assert issues["gaps"] == {"MP4-720p": [105], "CDG": [105]} + + def test_bad_timestamps_and_non_track_files_ignored(self): + created = {"desktop.ini": "2026-11-07T12:24:00Z", make_mp4(105): "garbage"} + assert recently_uploaded_sequences(created, now=self.NOW) == set()