#!/usr/bin/env python3 """Validate the checked-in Prometheus discovery and alert resources.""" from __future__ import annotations import argparse from pathlib import Path import re import sys ROOT = Path(__file__).resolve().parents[1] DEFAULT_DIRECTORY = ROOT / "deploy" / "observability" def verify(directory: Path, service_path: Path) -> None: monitor = (directory / "prometheus-service-monitor.yaml").read_text() rules = (directory / "prometheus-rules.yaml").read_text() service = service_path.read_text() if "kind: ServiceMonitor" not in monitor: raise ValueError("ServiceMonitor resource is missing") if "apiVersion: monitoring.coreos.com/v1" not in monitor: raise ValueError("ServiceMonitor API version is not pinned") if "namespace: cosmic-clash" not in monitor or ' - cosmic-clash' not in monitor: raise ValueError("ServiceMonitor namespace is not restricted to cosmic-clash") if "app.kubernetes.io/name: control-plane" not in monitor: raise ValueError("ServiceMonitor does not select the control-plane") if not re.search(r"(?m)^ - port: http$", monitor): raise ValueError("ServiceMonitor does not use the named http port") if not re.search(r"(?m)^ path: /metrics$", monitor): raise ValueError("ServiceMonitor path is not /metrics") if "interval: 15s" not in monitor or "scrapeTimeout: 5s" not in monitor: raise ValueError("ServiceMonitor interval/timeout contract changed") if "kind: Service" not in service or "name: control-plane" not in service: raise ValueError("control-plane Service is missing") if not re.search(r"(?m)^ - name: http$", service): raise ValueError("control-plane Service has no named http port") if "kind: PrometheusRule" not in rules: raise ValueError("PrometheusRule resource is missing") for alert in ("CosmicClashControlPlaneAPIP95High", "CosmicClashControlPlaneAPI5xxHigh"): if f"alert: {alert}" not in rules: raise ValueError(f"required alert is missing: {alert}") if "histogram_quantile" not in rules or "cosmic_clash_api_latency_seconds_bucket" not in rules: raise ValueError("API p95 alert is not based on the exported histogram") if 'status="5xx"' not in rules or "cosmic_clash_api_requests_total" not in rules: raise ValueError("API error alert is not based on the exported counter") if "severity: page" not in rules or "owner: api" not in rules: raise ValueError("alerts must have bounded routing labels") routing = rules.split("labels:", 1)[-1].split("annotations:", 1)[0] if "{{ $labels." in routing: raise ValueError("dynamic labels were added to alert routing") def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--directory", type=Path, default=DEFAULT_DIRECTORY) parser.add_argument("--service", type=Path, default=ROOT / "deploy/k8s/base/control-plane-service.yaml") args = parser.parse_args() try: verify(args.directory, args.service) except (OSError, ValueError) as error: print(f"observability manifest verification failed: {error}", file=sys.stderr) return 1 print("observability manifest verification passed") return 0 if __name__ == "__main__": raise SystemExit(main())