#!/bin/bash
set -euo pipefail

cd /app
mkdir -p /app/outputs

python3 - <<'PY'
import json
import pathlib
import re
import textwrap
from datetime import datetime, timedelta

base = pathlib.Path("/app")

before_metrics = json.loads((base / "data/metrics/before.json").read_text())
after_metrics = json.loads((base / "data/metrics/after.json").read_text())
payments_before = before_metrics["payments-api"]
payments_after = after_metrics["payments-api"]

traces_data = json.loads((base / "data/traces/trace_sample.json").read_text())

payments_canary_errors = []
for t in traces_data.get("traces", []):
    if t.get("root_service") != "payments-api" or t.get("outcome") != "error":
        continue
    for span in t.get("spans", []):
        tags = span.get("tags", {})
        if tags.get("deployment") == "canary" and tags.get("db.error"):
            payments_canary_errors.append(t)
            break

payments_canary_errors.sort(key=lambda x: x.get("timestamp", ""))
primary_trace = payments_canary_errors[0] if payments_canary_errors else None
primary_trace_id = primary_trace["trace_id"] if primary_trace else ""

canary_pod = None
db_host = None
if primary_trace:
    for span in primary_trace.get("spans", []):
        if span.get("tags", {}).get("pod"):
            canary_pod = span["tags"]["pod"]
        if span.get("tags", {}).get("host"):
            db_host = span["tags"]["host"]

logs = (base / "data/logs/app.log").read_text()

deployments_path = base / "data/deployments/canary_manifest.json"
deployments = json.loads(deployments_path.read_text())
rollback_token = None
canary_version = None
previous_status = None
for dep in deployments.get("deployments", []):
    if dep.get("service") == "payments-api" and dep.get("status") == "active":
        rollback_token = dep.get("rollback_token")
        canary_version = dep.get("version")
        previous_status = dep.get("status")
        dep["status"] = "rolled_back"
        break

deployments_path.write_text(json.dumps(deployments, indent=2))

affected_txns = set()
error_timestamps = []
correlation_id = None
for line in logs.split("\n"):
    if "payments-api" in line and "ERROR" in line and "[txn:" in line:
        if "canary pod" in line and "db connection refused" in line:
            match = re.search(r'\[txn:(pay-\d+)\]', line)
            if match:
                affected_txns.add(match.group(1))
            timestamp = line.split()[0]
            error_timestamps.append(timestamp)
            corr_match = re.search(r'\[corr:(INC-\d{8}-\d{4})\]', line)
            if corr_match and correlation_id is None:
                correlation_id = corr_match.group(1)

affected_txns = sorted(list(affected_txns))
affected_txn_count = len(affected_txns)

incident_window = {
    "start": min(error_timestamps) if error_timestamps else "",
    "end": max(error_timestamps) if error_timestamps else ""
}

start_time = datetime.fromisoformat(incident_window["start"].replace("Z", "+00:00"))
end_time = datetime.fromisoformat(incident_window["end"].replace("Z", "+00:00"))
mttr_seconds = int((end_time - start_time).total_seconds())

if mttr_seconds > 0:
    error_velocity = round(affected_txn_count / (mttr_seconds / 60), 2)
else:
    error_velocity = float(affected_txn_count)

total_requests = payments_before["requests"]
impact_pct = round((affected_txn_count / total_requests) * 100, 2)

error_rate_delta = payments_after["error_rate_pct"] - payments_before["error_rate_pct"]
p99_delta = payments_after["p99_latency_ms"] - payments_before["p99_latency_ms"]

state_path = base / "config/service_state.json"
state = json.loads(state_path.read_text())
payments_owner = state.get("payments-api", {}).get("owner", "payments-team")
payments_state = state.get("payments-api", {})
payments_state["canary_enabled"] = False
payments_state["traffic_split"] = {"stable": 100, "canary": 0}
state["payments-api"] = payments_state
state_path.write_text(json.dumps(state, indent=2))

incident_time = datetime.fromisoformat(primary_trace["timestamp"].replace("Z", "+00:00"))
expires_time = incident_time + timedelta(minutes=30)
expires_at = expires_time.strftime("%Y-%m-%dT%H:%M:%SZ")

alert_suppression = {
    "service": "payments-api",
    "suppression_window_minutes": 30,
    "reason": "Canary rollback due to DB connection failures on canary pod",
    "created_by": "incident-commander",
    "expires_at": expires_at
}
(base / "config/alert_suppression.json").write_text(json.dumps(alert_suppression, indent=2))

priority = "P1" if impact_pct > 1 else "P2"
channel = "pagerduty" if priority == "P1" else "slack"
escalation_level = "L2" if impact_pct > 5 else "L1"

actions = [
    {
        "action": "disable_canary",
        "target": "payments-api",
        "status": "applied",
        "details": "Set traffic_split to stable=100 and canary=0 per runbook",
        "evidence": f"/app/data/logs/app.log; trace_id={primary_trace_id}",
        "rollback_token": rollback_token,
        "correlation_id": correlation_id,
    },
    {
        "action": "create_alert_suppression",
        "target": "payments-api",
        "status": "applied",
        "details": "Created alert suppression for 30 minutes to prevent false positives during recovery",
    },
    {
        "action": "create_followup_ticket",
        "target": "payments-api",
        "status": "applied",
        "details": f"Post-incident review ticket for canary DB connection failures; impact: {impact_pct}% of requests",
        "priority": priority,
        "assigned_team": payments_owner,
    },
    {
        "action": "notify_stakeholders",
        "target": "payments-api",
        "status": "applied",
        "details": f"Notified stakeholders about payments-api canary rollback via {channel}",
        "channel": channel,
        "escalation_level": escalation_level,
    },
    {
        "action": "update_deployment_status",
        "target": "payments-api",
        "status": "applied",
        "details": f"Updated deployment status from {previous_status} to rolled_back",
        "previous_status": previous_status,
        "new_status": "rolled_back",
        "canary_version": canary_version,
    }
]
(base / "outputs/actions.json").write_text(json.dumps(actions, indent=2))

validation_passed = payments_after["error_rate_pct"] < 5 and payments_after["p99_latency_ms"] < 400
postcheck = {
    "error_rate_pct": payments_after["error_rate_pct"],
    "p99_latency_ms": payments_after["p99_latency_ms"],
    "validation_passed": validation_passed,
    "notes": "Values from /app/data/metrics/after.json for payments-api; thresholds: error_rate_pct < 5 and p99_latency_ms < 400",
    "affected_txn_count": affected_txn_count,
    "impact_pct": impact_pct,
    "incident_window": incident_window,
    "slo_budget": {
        "monthly_budget_pct": 0.1,
        "consumed_pct": payments_before["error_rate_pct"],
        "remaining_pct": round(0.1 - payments_before["error_rate_pct"], 2),
    },
    "mttr_seconds": mttr_seconds,
    "error_velocity": error_velocity,
}
(base / "outputs/postcheck.json").write_text(json.dumps(postcheck, indent=2))

summary = textwrap.dedent(
    f"""\
# Incident Summary

## Overview
Spike in payments-api 5xx errors after canary rollout; mitigated by rollback to stable.
Rollback token: {rollback_token}
Correlation ID: {correlation_id}

Incident window: {incident_window["start"]} to {incident_window["end"]}
MTTR: {mttr_seconds} seconds
Error velocity: {error_velocity} errors/minute

## Root Cause
Payments-api canary pod `{canary_pod}` experienced DB connection failures to `{db_host}`.
The canary build could not establish connections, triggering circuit breaker after {affected_txn_count} consecutive failures.

Before incident: error_rate_pct={payments_before["error_rate_pct"]}%, p99_latency_ms={payments_before["p99_latency_ms"]}
After mitigation: error_rate_pct={payments_after["error_rate_pct"]}%, p99_latency_ms={payments_after["p99_latency_ms"]}
Delta: error_rate_pct={error_rate_delta}% (Δ{error_rate_delta}%), p99_latency_ms={p99_delta}ms (Δ{p99_delta}ms)

Primary trace ID: {primary_trace_id}

## Blast Radius
Impact limited to payments-api requests routed to canary (30% traffic).
Affected transactions: {affected_txn_count} ({", ".join(affected_txns)})
Impact percentage: {impact_pct}% of total requests

Unaffected services:
- checkout: No correlated errors detected during incident window
- inventory-api: Normal operation, lock timeout was unrelated to payments incident
- notification-api: No alerts triggered
- user-api: Canary operating normally, validation error was user input issue (unrelated)
- audit-api: No data integrity issues detected

## Evidence
- Logs: /app/data/logs/app.log
- Metrics (before): /app/data/metrics/before.json
- Metrics (after): /app/data/metrics/after.json
- Trace sample: /app/data/traces/trace_sample.json
- Runbook: /app/runbooks/payments-canary-rollback.md
- Deployment manifest: /app/data/deployments/canary_manifest.json

## Actions Taken
- Executed rollback: disable_canary on payments-api
- traffic_split set to stable=100, canary=0
- Created alert suppression for payments-api (30 minute window)
- Created follow-up ticket ({priority}) assigned to {payments_owner}
- Notified stakeholders via {channel} (escalation level: {escalation_level})
- Updated deployment status to rolled_back (was: {previous_status})
- Rollback token: {rollback_token}

## Validation
Post-mitigation metrics:
- error_rate_pct: {payments_after["error_rate_pct"]}% (threshold < 5%) ✓
- p99_latency_ms: {payments_after["p99_latency_ms"]}ms (threshold < 400ms) ✓
- validation_passed: {validation_passed}

Metrics show error_rate_pct: {payments_before["error_rate_pct"]}% → {payments_after["error_rate_pct"]}% (Δ{error_rate_delta}%)
Metrics show p99_latency_ms: {payments_before["p99_latency_ms"]}ms → {payments_after["p99_latency_ms"]}ms (Δ{p99_delta}ms)

SLO Budget Analysis:
- Monthly error budget: 0.1% (99.9% availability target)
- Consumed during incident: {payments_before["error_rate_pct"]}%
- Remaining budget: {round(0.1 - payments_before["error_rate_pct"], 2)}%

## Next Steps
- Keep canary disabled until a fixed build is available
- Investigate DB connection pool settings on canary pods
- Add connection retry/backoff tuning
- Monitor error budget for payments-api for 24h
- Post-incident review scheduled ({priority} ticket created)
"""
)
(base / "outputs/incident_summary.md").write_text(summary)
PY