mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
Josh Creek
97 lines
4.9 KiB
Python
97 lines
4.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Validate the checked-in Prometheus discovery and alert resources."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from pathlib import Path
|
|
import re
|
|
import sys
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DEFAULT_DIRECTORY = ROOT / "deploy" / "observability"
|
|
|
|
|
|
def verify(directory: Path, service_path: Path) -> None:
|
|
kustomization = (directory / "kustomization.yaml").read_text()
|
|
monitor = (directory / "prometheus-service-monitor.yaml").read_text()
|
|
rules = (directory / "prometheus-rules.yaml").read_text()
|
|
service = service_path.read_text()
|
|
allocator_monitor = (directory / "prometheus-allocator-service-monitor.yaml").read_text()
|
|
allocator_service = (ROOT / "deploy/k8s/base/allocator-service.yaml").read_text()
|
|
|
|
for resource in (
|
|
"prometheus-rules.yaml",
|
|
"prometheus-service-monitor.yaml",
|
|
"prometheus-allocator-service-monitor.yaml",
|
|
):
|
|
if resource not in kustomization:
|
|
raise ValueError(f"observability Kustomization omits {resource}")
|
|
|
|
if "kind: ServiceMonitor" not in monitor:
|
|
raise ValueError("ServiceMonitor resource is missing")
|
|
if "apiVersion: monitoring.coreos.com/v1" not in monitor:
|
|
raise ValueError("ServiceMonitor API version is not pinned")
|
|
if "namespace: cosmic-clash" not in monitor or ' - cosmic-clash' not in monitor:
|
|
raise ValueError("ServiceMonitor namespace is not restricted to cosmic-clash")
|
|
if "app.kubernetes.io/name: control-plane" not in monitor:
|
|
raise ValueError("ServiceMonitor does not select the control-plane")
|
|
if not re.search(r"(?m)^ - port: http$", monitor):
|
|
raise ValueError("ServiceMonitor does not use the named http port")
|
|
if not re.search(r"(?m)^ path: /metrics$", monitor):
|
|
raise ValueError("ServiceMonitor path is not /metrics")
|
|
if "interval: 15s" not in monitor or "scrapeTimeout: 5s" not in monitor:
|
|
raise ValueError("ServiceMonitor interval/timeout contract changed")
|
|
|
|
if "kind: ServiceMonitor" not in allocator_monitor or "name: allocator" not in allocator_monitor:
|
|
raise ValueError("allocator ServiceMonitor is missing")
|
|
if not re.search(r"(?m)^ - port: metrics$", allocator_monitor) or not re.search(r"(?m)^ path: /metrics$", allocator_monitor):
|
|
raise ValueError("allocator ServiceMonitor endpoint is invalid")
|
|
if "namespace: cosmic-clash" not in allocator_monitor or " - cosmic-clash" not in allocator_monitor:
|
|
raise ValueError("allocator ServiceMonitor namespace is not restricted")
|
|
if "kind: Service" not in allocator_service or "name: allocator" not in allocator_service:
|
|
raise ValueError("allocator metrics Service is missing")
|
|
if "name: metrics" not in allocator_service or "port: 9091" not in allocator_service:
|
|
raise ValueError("allocator metrics Service port is missing")
|
|
|
|
if "kind: Service" not in service or "name: control-plane" not in service:
|
|
raise ValueError("control-plane Service is missing")
|
|
if not re.search(r"(?m)^ - name: http$", service):
|
|
raise ValueError("control-plane Service has no named http port")
|
|
|
|
if "kind: PrometheusRule" not in rules:
|
|
raise ValueError("PrometheusRule resource is missing")
|
|
for alert in ("CosmicClashControlPlaneAPIP95High", "CosmicClashControlPlaneAPI5xxHigh", "CosmicClashAllocatorQuotaDenials"):
|
|
if f"alert: {alert}" not in rules:
|
|
raise ValueError(f"required alert is missing: {alert}")
|
|
if "histogram_quantile" not in rules or "cosmic_clash_api_latency_seconds_bucket" not in rules:
|
|
raise ValueError("API p95 alert is not based on the exported histogram")
|
|
if 'status="5xx"' not in rules or "cosmic_clash_api_requests_total" not in rules:
|
|
raise ValueError("API error alert is not based on the exported counter")
|
|
if "severity: page" not in rules or "owner: api" not in rules:
|
|
raise ValueError("alerts must have bounded routing labels")
|
|
if "cosmic_clash_allocator_quota_denials_total" not in rules or "owner: allocator" not in rules:
|
|
raise ValueError("allocator quota alert is not based on bounded allocator metrics")
|
|
routing = rules.split("labels:", 1)[-1].split("annotations:", 1)[0]
|
|
if "{{ $labels." in routing:
|
|
raise ValueError("dynamic labels were added to alert routing")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--directory", type=Path, default=DEFAULT_DIRECTORY)
|
|
parser.add_argument("--service", type=Path, default=ROOT / "deploy/k8s/base/control-plane-service.yaml")
|
|
args = parser.parse_args()
|
|
try:
|
|
verify(args.directory, args.service)
|
|
except (OSError, ValueError) as error:
|
|
print(f"observability manifest verification failed: {error}", file=sys.stderr)
|
|
return 1
|
|
print("observability manifest verification passed")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|