mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Problem After shipping #741 with structured per-iteration logging for the custom-lyrics repair loop, traces in Cloud Logging showed only \`{job_id, logger, message, timestamp}\` — every diagnostic field (\`worst_lines\`, \`iterations_used\`, \`violations_after\`, \`delta_total_*\`, \`improved\`, \`stop_reason\`, \`settings\`, etc.) was silently dropped. Root cause: \`StructuredFormatter.format()\` in \`backend/services/structured_logging.py\` used an **explicit allowlist** (\`custom_fields = [\"job_id\", \"worker\", ...]\`). Any new diagnostic field added by a future feature owner is dropped on the floor unless they remember to update the allowlist — and they only find out when prod logs are useless. ## Fix Switch to a denylist of standard \`LogRecord\` attributes. Anything else on the record came from \`extra={...}\` and gets forwarded unchanged. Underscore-prefixed and \`None\` values still skipped. Add 5 regression tests covering: arbitrary fields forwarded, complex values (dicts/lists) preserved, None omitted, underscore-prefixed skipped, standard LogRecord attrs not duplicated. ## Verified - Cloud Logging trace for job \`2cb49a45\` (Strict run @ 04:30 UTC) shows \`jsonPayload\` containing only \`job_id, logger, message, timestamp\` despite the service code passing \`extra={\"violations_before\": ..., \"worst_lines\": [...], ...}\` — confirms the allowlist drop. Post-merge, the next custom-lyrics run will populate full diagnostic payload. ## Test plan - [x] 5 new regression tests pass (\`pytest backend/tests/services/test_structured_logging.py\`) - [ ] Post-merge: re-run custom-lyrics in prod, query \`jsonPayload.message=~\"^custom_lyrics_\"\` for the new job_id, verify \`worst_lines\` etc. are present @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
261 lines
9.1 KiB
Python
261 lines
9.1 KiB
Python
"""
|
|
Structured logging with trace correlation for Cloud Logging.
|
|
|
|
This module provides JSON-formatted logging that integrates with Google Cloud Logging
|
|
and correlates logs with OpenTelemetry traces in Cloud Trace.
|
|
|
|
When running in Cloud Run:
|
|
- Logs are output as JSON for Cloud Logging to parse
|
|
- Each log entry includes trace ID and span ID for correlation
|
|
- Custom fields (job_id, worker) are preserved in the log structure
|
|
|
|
Usage:
|
|
# In main.py, before any logging:
|
|
from backend.services.structured_logging import setup_structured_logging
|
|
setup_structured_logging()
|
|
|
|
# Then use standard logging:
|
|
logger = logging.getLogger(__name__)
|
|
logger.info("Processing started", extra={"job_id": "abc123"})
|
|
"""
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
from datetime import datetime
|
|
from typing import Any, Dict, Optional
|
|
|
|
|
|
# Cloud Logging severity mapping (compatible with GCP)
|
|
# https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#LogSeverity
|
|
SEVERITY_MAP = {
|
|
"DEBUG": "DEBUG",
|
|
"INFO": "INFO",
|
|
"WARNING": "WARNING",
|
|
"ERROR": "ERROR",
|
|
"CRITICAL": "CRITICAL",
|
|
}
|
|
|
|
|
|
# Standard LogRecord attributes set by the logging module itself. Anything
|
|
# else on a record came from the caller's `extra=` dict and should be
|
|
# forwarded to Cloud Logging.
|
|
_RESERVED_LOG_RECORD_ATTRS = frozenset({
|
|
"args", "asctime", "created", "exc_info", "exc_text", "filename",
|
|
"funcName", "levelname", "levelno", "lineno", "message", "module",
|
|
"msecs", "msg", "name", "pathname", "process", "processName",
|
|
"relativeCreated", "stack_info", "thread", "threadName", "taskName",
|
|
})
|
|
|
|
|
|
class StructuredFormatter(logging.Formatter):
|
|
"""
|
|
JSON formatter with trace correlation for Google Cloud Logging.
|
|
|
|
Produces log entries in the format expected by Cloud Logging, including:
|
|
- logging.googleapis.com/trace: Links to Cloud Trace
|
|
- logging.googleapis.com/spanId: Current span ID
|
|
- severity: Cloud Logging severity level
|
|
- Custom fields passed via 'extra' dict
|
|
"""
|
|
|
|
def __init__(self, project_id: Optional[str] = None):
|
|
"""
|
|
Initialize the structured formatter.
|
|
|
|
Args:
|
|
project_id: GCP project ID for trace URLs (auto-detected if not provided)
|
|
"""
|
|
super().__init__()
|
|
self.project_id = project_id or os.environ.get("GOOGLE_CLOUD_PROJECT") or os.environ.get("GCP_PROJECT")
|
|
|
|
def format(self, record: logging.LogRecord) -> str:
|
|
"""
|
|
Format a log record as JSON.
|
|
|
|
Args:
|
|
record: The log record to format
|
|
|
|
Returns:
|
|
JSON string for Cloud Logging
|
|
"""
|
|
from backend.services.tracing import get_current_trace_id, get_current_span_id
|
|
|
|
# Build base log entry
|
|
log_entry: Dict[str, Any] = {
|
|
"timestamp": datetime.utcfromtimestamp(record.created).isoformat() + "Z",
|
|
"severity": SEVERITY_MAP.get(record.levelname, record.levelname),
|
|
"message": record.getMessage(),
|
|
"logger": record.name,
|
|
}
|
|
|
|
# Add trace correlation if available
|
|
trace_id = get_current_trace_id()
|
|
span_id = get_current_span_id()
|
|
|
|
if trace_id and self.project_id:
|
|
# Format for Cloud Logging trace correlation
|
|
log_entry["logging.googleapis.com/trace"] = f"projects/{self.project_id}/traces/{trace_id}"
|
|
|
|
if span_id:
|
|
log_entry["logging.googleapis.com/spanId"] = span_id
|
|
|
|
# Forward any custom field passed via `extra=` on the log call.
|
|
# Use a denylist of standard LogRecord attributes rather than an
|
|
# allowlist — otherwise new diagnostic fields silently drop on the
|
|
# floor and feature owners discover this only when debugging prod.
|
|
for key, value in record.__dict__.items():
|
|
if key in _RESERVED_LOG_RECORD_ATTRS or key.startswith("_"):
|
|
continue
|
|
if value is None:
|
|
continue
|
|
log_entry[key] = value
|
|
|
|
# Add source location for debugging
|
|
if record.pathname and record.lineno:
|
|
log_entry["logging.googleapis.com/sourceLocation"] = {
|
|
"file": record.pathname,
|
|
"line": record.lineno,
|
|
"function": record.funcName,
|
|
}
|
|
|
|
# Add exception info if present
|
|
if record.exc_info:
|
|
log_entry["exception"] = self.formatException(record.exc_info)
|
|
|
|
# Remove None values for cleaner output
|
|
return json.dumps({k: v for k, v in log_entry.items() if v is not None})
|
|
|
|
|
|
class HumanReadableFormatter(logging.Formatter):
|
|
"""
|
|
Human-readable formatter for local development.
|
|
|
|
Includes trace context when available but outputs in traditional format.
|
|
"""
|
|
|
|
def format(self, record: logging.LogRecord) -> str:
|
|
"""Format a log record for human readability."""
|
|
from backend.services.tracing import get_current_trace_id
|
|
|
|
# Build base message
|
|
timestamp = datetime.utcfromtimestamp(record.created).strftime("%Y-%m-%d %H:%M:%S")
|
|
|
|
# Add job_id prefix if available
|
|
job_id = getattr(record, "job_id", None)
|
|
job_prefix = f"[job:{job_id[:8]}] " if job_id else ""
|
|
|
|
# Add trace ID suffix if available
|
|
trace_id = get_current_trace_id()
|
|
trace_suffix = f" [trace:{trace_id[:8]}]" if trace_id else ""
|
|
|
|
message = f"{timestamp} - {record.name} - {record.levelname} - {job_prefix}{record.getMessage()}{trace_suffix}"
|
|
|
|
# Add exception info if present
|
|
if record.exc_info:
|
|
message += "\n" + self.formatException(record.exc_info)
|
|
|
|
return message
|
|
|
|
|
|
def is_cloud_run() -> bool:
|
|
"""Check if we're running in Cloud Run."""
|
|
return os.environ.get("K_SERVICE") is not None
|
|
|
|
|
|
def setup_structured_logging(force_json: bool = False, log_level: Optional[str] = None) -> None:
|
|
"""
|
|
Configure structured logging for the application.
|
|
|
|
In Cloud Run: Uses JSON format with trace correlation for Cloud Logging
|
|
In development: Uses human-readable format with optional trace context
|
|
|
|
Args:
|
|
force_json: Force JSON output even in development
|
|
log_level: Override log level (default from settings or INFO)
|
|
"""
|
|
from backend.config import settings
|
|
|
|
# Determine log level
|
|
level = log_level or getattr(settings, "log_level", "INFO")
|
|
level = getattr(logging, level.upper(), logging.INFO)
|
|
|
|
# Choose formatter based on environment
|
|
if is_cloud_run() or force_json:
|
|
formatter = StructuredFormatter()
|
|
else:
|
|
formatter = HumanReadableFormatter()
|
|
|
|
# Create handler
|
|
handler = logging.StreamHandler(sys.stdout)
|
|
handler.setFormatter(formatter)
|
|
handler.setLevel(level)
|
|
|
|
# Configure root logger
|
|
root_logger = logging.getLogger()
|
|
root_logger.setLevel(level)
|
|
|
|
# Remove existing handlers and add our handler
|
|
root_logger.handlers = [handler]
|
|
|
|
# Also configure uvicorn loggers to use our format
|
|
for logger_name in ["uvicorn", "uvicorn.access", "uvicorn.error"]:
|
|
logger = logging.getLogger(logger_name)
|
|
logger.handlers = [handler]
|
|
logger.propagate = False
|
|
|
|
|
|
class JobLogAdapter(logging.LoggerAdapter):
|
|
"""
|
|
Logger adapter that automatically adds job context to log records.
|
|
|
|
Usage:
|
|
logger = logging.getLogger(__name__)
|
|
job_logger = JobLogAdapter(logger, job_id="abc123", worker="audio")
|
|
job_logger.info("Processing started") # Automatically includes job_id and worker
|
|
"""
|
|
|
|
def __init__(self, logger: logging.Logger, job_id: str, worker: Optional[str] = None, **extra):
|
|
"""
|
|
Initialize the adapter.
|
|
|
|
Args:
|
|
logger: Base logger instance
|
|
job_id: Job ID to include in all log records
|
|
worker: Optional worker name
|
|
**extra: Additional fields to include in all log records
|
|
"""
|
|
super().__init__(logger, {"job_id": job_id, "worker": worker, **extra})
|
|
|
|
def process(self, msg: str, kwargs: Dict[str, Any]) -> tuple:
|
|
"""Process a logging call to add extra context."""
|
|
# Merge extra dict from adapter with any extra passed to the log call
|
|
extra = kwargs.get("extra", {})
|
|
extra.update(self.extra)
|
|
kwargs["extra"] = extra
|
|
return msg, kwargs
|
|
|
|
|
|
def get_job_logger(job_id: str, worker: Optional[str] = None, name: Optional[str] = None) -> JobLogAdapter:
|
|
"""
|
|
Get a logger adapter configured for a specific job.
|
|
|
|
This is a convenience function for creating job-specific loggers.
|
|
|
|
Args:
|
|
job_id: Job ID to include in all log records
|
|
worker: Optional worker name
|
|
name: Logger name (defaults to "job")
|
|
|
|
Returns:
|
|
JobLogAdapter configured for the job
|
|
|
|
Usage:
|
|
logger = get_job_logger("abc123", "audio")
|
|
logger.info("Starting audio separation")
|
|
logger.error("Failed to process", extra={"error": str(e)})
|
|
"""
|
|
base_logger = logging.getLogger(name or "job")
|
|
return JobLogAdapter(base_logger, job_id=job_id, worker=worker)
|
|
|