CosmicClash/deploy/observability/prometheus-rules.yaml

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: cosmic-clash-control-plane
  namespace: cosmic-clash
  labels:
    app.kubernetes.io/name: cosmic-clash
    app.kubernetes.io/component: observability
spec:
  groups:
    - name: cosmic-clash.control-plane
      rules:
        - alert: CosmicClashControlPlaneAPIP95High
          expr: |
            histogram_quantile(
              0.95,
              sum by (le, operation) (
                rate(cosmic_clash_api_latency_seconds_bucket[5m])
              )
            ) > 0.25
          for: 5m
          labels:
            severity: page
            owner: api
          annotations:
            summary: Cosmic Clash control-plane API p95 latency is high
            description: >-
              The 5-minute p95 latency for operation {{ $labels.operation }}
              has exceeded the 250 ms API SLO for 5 minutes.
            runbook_url: https://example.invalid/cosmic-clash/runbooks/control-plane-api
        - alert: CosmicClashControlPlaneAPI5xxHigh
          expr: |
            (
              sum by (operation) (
                rate(cosmic_clash_api_requests_total{status="5xx"}[5m])
              )
              /
              clamp_min(
                sum by (operation) (
                  rate(cosmic_clash_api_requests_total[5m])
                ),
                0.001
              )
            ) > 0.01
          for: 5m
          labels:
            severity: page
            owner: api
          annotations:
            summary: Cosmic Clash control-plane API 5xx rate is high
            description: >-
              The 5-minute 5xx ratio for operation {{ $labels.operation }}
              has exceeded 1 percent for 5 minutes.
            runbook_url: https://example.invalid/cosmic-clash/runbooks/control-plane-api
    - name: cosmic-clash.allocator
      rules:
        - alert: CosmicClashAllocatorQuotaDenials
          expr: |
            sum by (region) (
              increase(cosmic_clash_allocator_quota_denials_total[15m])
            ) > 0
          for: 5m
          labels:
            severity: warning
            owner: allocator
          annotations:
            summary: Cosmic Clash allocator quota is denying allocation attempts
            description: >-
              The {{ $labels.region }} allocator has denied at least one
              allocation attempt in the last 15 minutes; verify quota capacity,
              provider health, and denial-of-wallet activity.
            runbook_url: https://example.invalid/cosmic-clash/runbooks/allocator-quota