# incident_response/cli.py — how you acknowledge
import fire  # Google's CLI library that turns functions into CLI commands automatically
import requests  # HTTP library for sending API requests to incident management services
from datetime import datetime  # For timestamping the incident with the exact time of acknowledgment

def acknowledge_incident(severity: str = "SEV1"):
    """Acknowledge the incident in your monitoring system."""
    payload = {
        "title": f"AI Platform error rate spike — {datetime.now().isoformat()}",  # Human-readable title with current timestamp
        "severity": severity,  # Severity level (SEV1 = critical, SEV2 = high, etc.)
        "current_error_rate": "34%",  # The observed error rate at the time of acknowledgment
        "baseline": "0.1%",  # The normal baseline error rate for comparison
    }
    # POST to your incident management API (PagerDuty, Opsgenie, etc.)
    resp = requests.post(
        "https://api.pagerduty.com/incidents",  # PagerDuty's incident creation endpoint
        json=payload,  # Send the payload as JSON in the request body
        headers={"Authorization": "Token YOUR_API_KEY"},  # Authentication header with your PagerDuty API token
    )
    print(f"Incident acknowledged: {resp.status_code}")  # Print the HTTP status code to confirm acknowledgment

if __name__ == "__main__":
    fire.Fire(acknowledge_incident)  # Expose the function as a CLI command (e.g., `python cli.py --severity=SEV1`)

# If you identify a bad tenant, rate-limit at the API gateway
from fastapi import FastAPI, Request, HTTPException  # FastAPI web framework components for building the API and returning HTTP errors
from collections import defaultdict  # Dict subclass that provides default values for missing keys
import time  # For tracking rate-limit reset windows

app = FastAPI()  # Initialize the FastAPI application instance

rate_limits = defaultdict(lambda: {"count": 0, "reset": time.time() + 60})  # Per-tenant rate limit state: count and a 60-second reset window

@app.middleware("http")
async def tenant_rate_limit(request: Request, call_next):
    tenant = request.headers.get("X-Tenant-ID", "unknown")  # Extract the tenant identifier from the request header, defaulting to "unknown"

    # If this tenant is causing errors, block them
    if tenant in BLOCKED_TENANTS:  # BLOCKED_TENANTS is a set populated during the incident investigation
        raise HTTPException(429, "Rate limited — suspected error source")  # Return HTTP 429 Too Many Requests, stopping the request here

    response = await call_next(request)  # Pass the request to the next middleware or route handler
    return response  # Return the response back up the middleware chain to the client

# inference_service/health.py
from fastapi import FastAPI, HTTPException  # FastAPI framework and HTTP exception for failing health checks
import torch  # PyTorch — used here to check CUDA GPU status and memory

app = FastAPI()  # Create the FastAPI application that serves the health endpoint

@app.get("/health")
async def health():
    checks = {
        "model_loaded": model is not None,  # Boolean: is the ML model loaded into memory?
        "gpu_available": torch.cuda.is_available(),  # Boolean: does the server have access to a CUDA-capable GPU?
        "gpu_memory_free": torch.cuda.mem_get_info()[0] if torch.cuda.is_available() else 0,  # Free GPU memory in bytes; 0 if no GPU
        "avg_latency_1m": get_recent_avg_latency(),  # Average inference latency over the last minute (defined elsewhere)
    }

    # Fail health check if GPU memory is critically low
    if checks["gpu_available"] and checks["gpu_memory_free"] < 1_000_000_000:  # Less than ~1GB free means the GPU is nearly full
        raise HTTPException(503, detail="GPU memory exhausted")  # Return HTTP 503 Service Unavailable — orchestrator will restart the pod

    return checks  # Return the full health check dictionary as JSON to the caller (probe, load balancer, etc.)

# upstream_check.py
import httpx  # Modern async HTTP client for Python, used here to probe external model APIs
import asyncio  # Async I/O library — lets us check multiple upstream providers concurrently
from datetime import datetime  # For measuring request latency with timestamps

async def check_upstream(provider: str, api_key: str) -> dict:
    start = datetime.now()  # Record the start time for latency measurement
    try:
        async with httpx.AsyncClient(timeout=10) as client:  # Create an async HTTP client with a 10-second timeout per request
            if provider == "openai":
                resp = await client.post(
                    "https://api.openai.com/v1/chat/completions",  # OpenAI's chat completions endpoint
                    headers={"Authorization": f"Bearer {api_key}"},  # Bearer token auth using the provided API key
                    json={
                        "model": "gpt-4o-mini",  # Use a cheap, fast model for the health probe
                        "messages": [{"role": "user", "content": "ping"}],  # Minimal message to test the API
                        "max_tokens": 1,  # Request only 1 token to minimize cost and latency
                    },
                )
            elif provider == "anthropic":
                resp = await client.post(
                    "https://api.anthropic.com/v1/messages",  # Anthropic's messages API endpoint
                    headers={"x-api-key": api_key, "anthropic-version": "2023-06-01"},  # API key auth + required version header
                    json={
                        "model": "claude-3-haiku",  # Anthropic's fastest/cheapest model for the probe
                        "messages": [{"role": "user", "content": "ping"}],  # Minimal probe message
                        "max_tokens": 1,  # Single-token response is enough to confirm the API works
                    },
                )
            elapsed = (datetime.now() - start).total_seconds()  # Calculate total round-trip time in seconds
            return {
                "provider": provider,  # Which provider was probed
                "status_code": resp.status_code,  # HTTP status code (200 = OK, 5xx = degraded)
                "latency": elapsed,  # How long the request took
                "healthy": resp.status_code == 200 and elapsed < 5,  # Healthy only if status is 200 AND latency under 5 seconds
            }
    except Exception as e:  # Catches timeouts, connection errors, DNS failures, etc.
        return {"provider": provider, "error": str(e), "healthy": False}  # Report the error and mark as unhealthy

async def check_all_upstreams():
    results = await asyncio.gather(  # Run both health checks concurrently
        check_upstream("openai", os.getenv("OPENAI_API_KEY")),  # Check OpenAI with key from environment variable
        check_upstream("anthropic", os.getenv("ANTHROPIC_API_KEY")),  # Check Anthropic with key from environment variable
    )
    unhealthy = [r for r in results if not r["healthy"]]  # Filter to only failed checks
    if unhealthy:
        # Upstream is partially down — this is likely the root cause
        log_incident("Upstream API degraded", unhealthy)  # Log the upstream failure as part of the incident timeline

# inference_service/router.py
from fastapi import FastAPI, HTTPException  # FastAPI framework plus HTTP exception for the fallback error
import httpx  # Async HTTP client for calling upstream model APIs
import random  # Used to shuffle provider order for basic load distribution

app = FastAPI()  # Initialize the FastAPI application

PROVIDERS = [
    {"name": "openai", "url": "https://api.openai.com/v1/chat/completions", "key": os.getenv("OPENAI_API_KEY")},  # OpenAI provider config with key from env
    {"name": "anthropic", "url": "https://api.anthropic.com/v1/messages", "key": os.getenv("ANTHROPIC_API_KEY")},  # Anthropic provider config with key from env
]

@app.post("/v1/completions")
async def completions(prompt: str):
    # Shuffle providers for load balancing and failover
    random.shuffle(PROVIDERS)  # Randomize provider order so traffic isn't always sent to the same provider first

    for provider in PROVIDERS:  # Try each provider in shuffled order until one succeeds
        try:
            async with httpx.AsyncClient(timeout=30) as client:  # 30-second timeout per provider attempt
                if provider["name"] == "openai":
                    resp = await client.post(
                        provider["url"],  # Use the provider's URL from the config list
                        headers={"Authorization": f"Bearer {provider['key']}"},  # Bearer token authentication
                        json={
                            "model": "gpt-4o-mini",  # Fast, cost-effective model for inference
                            "messages": [{"role": "user", "content": prompt}],  # Wrap the user's prompt in the chat format
                        },
                    )
                elif provider["name"] == "anthropic":
                    resp = await client.post(
                        provider["url"],  # Use the provider's URL from the config list
                        headers={"x-api-key": provider["key"], "anthropic-version": "2023-06-01"},  # Anthropic uses x-api-key header + version header
                        json={
                            "model": "claude-3-haiku",  # Anthropic's fastest model for inference
                            "messages": [{"role": "user", "content": prompt}],  # Wrap the user's prompt in the chat format
                        },
                    )
                resp.raise_for_status()  # Raise an exception for 4xx/5xx status codes — same as catching an HTTP error
                return resp.json()  # Return the successful provider's JSON response to the client
        except Exception as e:
            logger.warning(f"Provider {provider['name']} failed: {e}")  # Log the failure for observability
            continue  # Try the next provider in the list

    raise HTTPException(503, "All upstream providers failed")  # All providers failed — return 503 Service Unavailable

# monitoring/drift_detector.py
import numpy as np  # NumPy — used here for numerical operations like z-score calculation
from fastapi import FastAPI, Request  # FastAPI framework components for route handling and raw request access
from pydantic import BaseModel  # Pydantic for request/response schema validation and serialization
import time  # Time utilities for potential rate calculations (available but not used in this snippet)

app = FastAPI()  # Initialize the FastAPI application

# Reference distribution from training data
REFERENCE = {
    "text_length_mean": 120,  # Average token/word count in the training dataset
    "text_length_std": 80,  # Standard deviation of token/word counts in the training dataset
}

class PredictionResponse(BaseModel):
    prediction: int  # The model's predicted class label (e.g., 0 = negative, 1 = positive)
    confidence: float  # The model's confidence score for this prediction (0.0 to 1.0)
    drift_warning: str | None = None  # Optional warning if the input distribution has drifted from training

@app.post("/predict", response_model=PredictionResponse)
async def predict(request: Request):
    data = await request.json()  # Parse the raw JSON body from the incoming request
    text = data["text"]  # Extract the text field from the request payload
    text_len = len(text.split())  # Count the number of whitespace-separated words in the input

    # Detect drift in input distribution
    z_score = abs(text_len - REFERENCE["text_length_mean"]) / REFERENCE["text_length_std"]  # Calculate how many standard deviations from the training mean
    drift_warning = None  # Initialize drift warning as None — will be set only if drift is detected
    if z_score > 3:  # A z-score > 3 means the input is more than 3 standard deviations from the mean (statistically significant anomaly)
        drift_warning = f"Input length ({text_len}) is {z_score:.1f} std from training mean"  # Human-readable drift alert

    # Run inference
    result = model.predict([text])[0]  # Run the ML model's prediction and take the first (only) result
    confidence = max(model.predict_proba([text])[0])  # Get the highest probability across all classes as the confidence score

    return PredictionResponse(
        prediction=result,  # The model's predicted class
        confidence=float(confidence),  # Confidence score cast to Python float for JSON serialization
        drift_warning=drift_warning,  # Drift warning (None if input was within the expected distribution)
    )

# monitoring/resource_monitor.py
import psutil  # Cross-platform system/process monitoring library (CPU, memory, disk, network)
import torch  # PyTorch — used here to query CUDA GPU memory stats
from fastapi import FastAPI  # FastAPI framework for the debug endpoint

app = FastAPI()  # Initialize the FastAPI application

@app.get("/debug/resource-check")
async def resource_check():
    gpu_ok = True  # Default to healthy — will be set to False if GPU memory is nearly full
    gpu_detail = "no GPU"  # Default message for systems without a CUDA-capable GPU

    if torch.cuda.is_available():  # Check if a CUDA GPU is present
        free, total = torch.cuda.mem_get_info()  # Get free and total GPU memory in bytes
        used_percent = (total - free) / total * 100  # Calculate GPU memory utilization as a percentage
        gpu_ok = used_percent < 95  # Mark as unhealthy if GPU memory usage exceeds 95%
        gpu_detail = f"{used_percent:.1f}% used ({free/1e9:.1f}GB free / {total/1e9:.1f}GB total)"  # Human-readable GPU memory report

    cpu_percent = psutil.cpu_percent(interval=1)  # CPU utilization over a 1-second sampling interval
    memory = psutil.virtual_memory()  # System RAM stats (total, available, percent used, etc.)
    disk = psutil.disk_usage("/")  # Disk usage for the root filesystem (total, used, free, percent)

    return {
        "gpu": gpu_detail,  # GPU memory details or "no GPU"
        "gpu_ok": gpu_ok,  # Boolean: is GPU memory utilization below the 95% threshold?
        "cpu_percent": cpu_percent,  # CPU utilization percentage
        "memory_percent": memory.percent,  # System RAM utilization percentage
        "disk_percent": disk.percent,  # Disk utilization percentage
        "connections": len(psutil.net_connections()),  # Number of active network connections (indicator of connection leaks)
    }

# inference_service/app.py — GPU memory guard
from contextlib import asynccontextmanager  # Decorator for creating async context managers (used for app lifespan)

@asynccontextmanager
async def lifespan(app: FastAPI):
    # Startup: register a metric for GPU memory
    yield  # The application runs here — startup code before yield, shutdown code after
    # Shutdown: clean up CUDA cache
    if torch.cuda.is_available():  # Only attempt GPU cleanup if a CUDA device exists
        torch.cuda.empty_cache()  # Free all cached CUDA memory that PyTorch is holding onto

app = FastAPI(lifespan=lifespan)  # Create the FastAPI app with the lifespan handler for startup/shutdown hooks

@app.middleware("http")
async def check_gpu_memory(request: Request, call_next):
    if torch.cuda.is_available():  # Only check GPU if a CUDA device is present
        free, total = torch.cuda.mem_get_info()  # Query current free and total GPU memory
        if free < 500_000_000:  # Less than 500MB free — critical threshold for OOM risk
            torch.cuda.empty_cache()  # Force-release cached GPU memory allocations
            logger.warning("GPU memory critically low — cache emptied")  # Log the event for incident timeline

    response = await call_next(request)  # Pass the request to the next middleware or route handler
    return response  # Return the response back up the chain

# inference_service/inference.py
import torch  # PyTorch — used for model inference and GPU tensor management
import gc  # Python garbage collector — ensures Python-side objects are cleaned up after inference

def run_inference(text: str) -> dict:
    try:
        inputs = tokenizer(text, return_tensors="pt", truncation=True).to("cuda")  # Tokenize input, return PyTorch tensors, truncate to max length, move to GPU
        with torch.no_grad():  # Disable gradient computation — not needed for inference, saves memory and speeds up execution
            outputs = model(**inputs)  # Run the model forward pass with the tokenized inputs

        result = {
            "prediction": outputs.logits.argmax().item(),  # Get the class with the highest logit value (the model's predicted class)
            "confidence": outputs.logits.softmax(-1).max().item(),  # Apply softmax to convert logits to probabilities, then take the max as confidence
        }
        return result  # Return the prediction result to the caller
    finally:
        # Always clean up GPU tensors, even on error
        del inputs, outputs  # Delete tensor references explicitly so they can be garbage collected
        torch.cuda.empty_cache()  # Clear PyTorch's CUDA memory cache to free GPU memory
        gc.collect()  # Force Python garbage collection to release any remaining references

# analytics/correlation_check.py
from datetime import datetime, timedelta  # For computing the time window around the incident
import httpx  # Async HTTP client for querying the Prometheus API

# Query Prometheus for correlated metrics
async def query_correlated_metrics(error_timestamp: datetime, window_minutes: int = 5):
    start = error_timestamp - timedelta(minutes=window_minutes)  # Start of the query window: 5 minutes before the error spike
    end = error_timestamp + timedelta(minutes=window_minutes)  # End of the query window: 5 minutes after the error spike

    queries = [
        'rate(http_requests_total{status=~"5.."}[1m])',  # Rate of HTTP 5xx errors per second (1m rate)
        'rate(node_cpu_seconds_total[1m])',  # CPU utilization rate — spiking CPU could indicate a thundering herd
        'node_memory_MemAvailable_bytes',  # Available system memory in bytes — low memory can cause OOM kills
        'rate(nvidia_smi_memory_used_bytes[1m])',  # GPU memory usage rate — GPU OOM is the most common AI incident cause
        'rate(container_oom_events_total[1m])',  # Container out-of-memory kill events — OOMKilled pods restart
    ]

    async with httpx.AsyncClient() as client:  # Create an async HTTP client for Prometheus queries
        results = {}
        for query in queries:  # Query each metric independently
            resp = await client.get(
                "http://prometheus:9090/api/v1/query_range",  # Prometheus range query endpoint
                params={
                    "query": query,  # The PromQL query string
                    "start": start.timestamp(),  # Start of the range as a Unix timestamp
                    "end": end.timestamp(),  # End of the range as a Unix timestamp
                    "step": "15",  # 15-second resolution for the time series data
                },
            )
            results[query] = resp.json()  # Parse the JSON response and store by query name
        return results  # Return all queried metrics for correlation analysis

# analytics/error_analysis.py
from fastapi import FastAPI, Request  # FastAPI framework components for the middleware
import logging  # Python's built-in logging module for structured error logging

app = FastAPI()  # Initialize the FastAPI application
error_logger = logging.getLogger("error_analysis")  # Create a named logger for error analysis — can be routed to a separate file or sink

@app.middleware("http")
async def capture_errors(request: Request, call_next):
    response = await call_next(request)  # Let the request proceed through all handlers and collect the response
    if response.status_code >= 500:  # Only log server-side errors (5xx), not client errors (4xx)
        error_logger.error("server_error", extra={  # Log at ERROR level with structured context in the "extra" dict
            "path": request.url.path,  # Which API endpoint produced the error (e.g., /v1/completions)
            "tenant": request.headers.get("X-Tenant-ID", "unknown"),  # Which tenant's request failed — helps identify noisy neighbors
            "user_agent": request.headers.get("User-Agent", "unknown"),  # Client identifier — could reveal a scraper or misconfigured client
            "method": request.method,  # HTTP method (GET, POST, etc.)
            "query_params": str(request.query_params),  # Full query string — useful for reproducing the failing request
            "status_code": response.status_code,  # The exact 5xx status code (500, 502, 503, etc.)
        })
    return response  # Return the response to the client unchanged

# incident_response/investigate.py
from datetime import datetime, timedelta  # For time arithmetic around the incident window
import httpx  # HTTP client for querying external APIs (GitHub, MLflow, etc.)
import json  # For serializing findings to a JSON string for output

class IncidentInvestigator:
    def __init__(self, incident_time: datetime):
        self.incident_time = incident_time  # Store the exact time the incident started (e.g., 14:47)
        self.findings = []  # Accumulate evidence as the investigation progresses

    def add_finding(self, source: str, evidence: str, confidence: float):
        self.findings.append({
            "source": source,  # Where the evidence came from (e.g., "GitHub API", "MLflow")
            "evidence": evidence,  # Human-readable description of what was found
            "confidence": confidence,  # How confident we are this is related to the incident (0.0 to 1.0)
            "timestamp": datetime.now().isoformat(),  # When this finding was recorded
        })

    def check_recent_deployments(self):
        """Check if there were any deployments around the incident time."""
        resp = httpx.get(
            "https://api.github.com/repos/org/platform/deployments",  # GitHub API endpoint for deployment history
            params={"since": (self.incident_time - timedelta(hours=2)).isoformat()},  # Look back 2 hours from the incident
        )
        deployments = resp.json()  # Parse the JSON response into a list of deployments
        if deployments:  # If there are any deployments in the window
            self.add_finding(
                "GitHub API",
                f"Found deployment at {deployments[0]['created_at']}",  # Record the most recent deployment time
                0.3,  # Low confidence — deployments don't always cause incidents
            )
        return len(deployments) == 0  # Return True if no deployments were found (eliminating this as a cause)

    def check_database_migrations(self):
        """Check if any migrations ran around the incident time."""
        # Query your migration tracking table
        pass  # Placeholder — implement by querying your migration history database

    def check_model_registry(self):
        """Check if model version was updated."""
        resp = httpx.get("http://mlflow:5000/api/2.0/mlflow/registered-models/alias")  # MLflow API to get the current model version for the production alias
        model = resp.json()  # Parse the response to get model metadata
        self.add_finding(
            "MLflow",
            f"Current model version: {model.get('version', 'unknown')}",  # Record which model version is currently deployed
            0.5,  # Medium confidence — a model change is a plausible root cause
        )
        return model  # Return the full model metadata for further analysis

    def summarize(self) -> str:
        return json.dumps(self.findings, indent=2)  # Pretty-print all findings as a JSON string for the incident log

# Usage
investigator = IncidentInvestigator(datetime(2026, 7, 22, 14, 47))  # Create investigator for the specific incident timestamp
investigator.check_recent_deployments()  # Check GitHub for recent deployments — returns False if any found
investigator.check_model_registry()  # Check MLflow for model version changes
print(investigator.summarize())  # Print all findings as a formatted JSON report

# inference_service/failover.py
class UpstreamRouter:
    def __init__(self):
        self.providers = [
            {"name": "primary", "url": "...", "priority": 1, "errors": 0},  # Primary provider: highest priority (1), zero errors initially
            {"name": "secondary", "url": "...", "priority": 2, "errors": 0},  # Secondary provider: lower priority (2), fallback
        ]
        self.error_threshold = 5  # Consecutive errors before a provider is deprioritized and considered degraded

    async def route(self, prompt: str) -> dict:
        for provider in sorted(self.providers, key=lambda p: p["priority"]):  # Try providers in priority order (lowest number first)
            try:
                result = await self._call(provider, prompt)  # Attempt to call the provider's API with the user's prompt
                provider["errors"] = 0  # Reset error count on success — provider is healthy again
                return result  # Return the successful result immediately
            except Exception:  # Any exception (timeout, connection error, 5xx response)
                provider["errors"] += 1  # Increment the consecutive error counter for this provider
                if provider["errors"] >= self.error_threshold:  # If error count exceeds the threshold (5)
                    provider["priority"] = 99  # Set priority to 99 (very low) — effectively demote this provider to last resort

        raise RuntimeError("All providers failed")  # Every provider has been exhausted — raise a fatal error

# inference_service/input_validator.py
from pydantic import BaseModel, Field, field_validator  # Pydantic for request schema, field constraints, and custom validators
import re  # Regular expressions (imported for potential regex validation, though not used in this snippet)

class CompletionRequest(BaseModel):
    text: str = Field(..., min_length=1, max_length=8192)  # Required field: text must be between 1 and 8192 characters

    @field_validator("text")
    @classmethod
    def validate_text(cls, v: str):
        # Reject binary garbage
        if not v.isprintable():  # Check if every character in the string is printable (no null bytes, control chars, etc.)
            raise ValueError("Text contains non-printable characters")  # Reject with a clear validation error message

        # Reject extremely long words (likely garbage)
        for word in v.split():  # Split the text into whitespace-separated words
            if len(word) > 200:  # If any single word exceeds 200 characters, it's probably garbage data
                raise ValueError(f"Word too long: {len(word)} chars")  # Reject with the offending word length

        # Reject if mostly numbers (likely a scraper)
        digit_ratio = sum(c.isdigit() for c in v) / len(v)  # Calculate the proportion of digit characters in the text
        if digit_ratio > 0.8:  # If more than 80% of characters are digits, it's likely a scraper or automated tool
            raise ValueError("Text is mostly digits — possible scraper")  # Reject with a descriptive error

        return v  # Return the validated text (Pydantic will store it as-is after validation)

@app.post("/v1/completions")
async def completions(request: CompletionRequest):
    # If the request passes validation, run inference
    # If it fails, the client gets a 422 with a clear error
    result = run_inference(request.text)  # Pass the validated text to the inference function
    return result  # Return the inference result as the HTTP response

# postmortem/template.py
from datetime import datetime  # For timestamping the postmortem document with the current date
import json  # For serializing the postmortem sections into a structured JSON output

class Postmortem:
    def __init__(self, incident_id: str):
        self.incident_id = incident_id  # Unique incident identifier (e.g., "INC-2026-07-22")
        self.sections = {
            "title": f"AI Platform Error Spike — {incident_id}",  # Document title with incident ID for traceability
            "date": datetime.now().isoformat(),  # ISO-formatted timestamp of when the postmortem was created
            "duration_minutes": None,  # Total incident duration — populated after the incident is resolved
            "impact": None,  # Description of user/business impact — populated during analysis
            "root_cause": None,  # The identified root cause — filled in after investigation concludes
            "timeline": [],  # Chronological list of key events during the incident
            "action_items": [],  # Remediation tasks assigned to teams after the incident
        }

    def add_timeline_entry(self, time: str, event: str, action: str = ""):
        self.sections["timeline"].append({
            "time": time,  # Timestamp of the event (e.g., "14:47")
            "event": event,  # Description of what happened
            "action": action,  # What action was taken in response
        })

    def add_action_item(self, description: str, owner: str, priority: str):
        self.sections["action_items"].append({
            "description": description,  # What needs to be done to prevent recurrence
            "owner": owner,  # Team responsible for implementing the fix
            "priority": priority,  # Priority level: P0 = critical, P1 = high, P2 = medium, etc.
        })

    def export(self) -> str:
        return json.dumps(self.sections, indent=2)  # Pretty-print the full postmortem as a JSON string

postmortem = Postmortem("INC-2026-07-22")  # Create a new postmortem for this specific incident
postmortem.add_timeline_entry("14:47", "Error rate spikes to 34%", "Page sent")  # Alert fired, on-call paged
postmortem.add_timeline_entry("14:49", "GPU memory exhaustion identified", "Restarted pod")  # Root cause found, immediate mitigation applied
postmortem.add_timeline_entry("14:52", "Error rate returns to 0.1%", "Monitoring")  # Error rate normalized after pod restart

postmortem.add_action_item(
    "Add GPU memory alert at 80% utilization",  # Proactive alerting to catch GPU pressure before it becomes critical
    "infra-team",  # Infrastructure team owns monitoring and alerting configuration
    "P0",  # P0 = must fix immediately to prevent the same incident
)
postmortem.add_action_item(
    "Add automatic GPU cache clearing middleware",  # Software-level defense against GPU memory leaks
    "ml-team",  # ML team owns the inference service code
    "P1",  # P1 = high priority but can be scheduled after the immediate fix
)
postmortem.add_action_item(
    "Update max_batch_size to prevent OOM",  # Configuration change to limit concurrent GPU memory usage
    "ml-team",  # ML team owns the model serving configuration
    "P1",  # P1 = important but not blocking production
)

print(postmortem.export())  # Print the full postmortem document as JSON for sharing and archiving