diff --git a/deploy/observability/prometheus-rules.yaml b/deploy/observability/prometheus-rules.yaml index 973a349d..9b0b509f 100644 --- a/deploy/observability/prometheus-rules.yaml +++ b/deploy/observability/prometheus-rules.yaml @@ -52,6 +52,27 @@ spec: The 5-minute 5xx ratio for operation {{ $labels.operation }} has exceeded 1 percent for 5 minutes. runbook_url: https://example.invalid/cosmic-clash/runbooks/control-plane-api + - alert: CosmicClashControlPlaneServerConflicts + expr: | + sum by (kind) ( + increase(cosmic_clash_api_server_conflicts_total[15m]) + ) > 3 + for: 5m + labels: + severity: warning + owner: api + annotations: + summary: Cosmic Clash workload-authenticated server mutations are conflicting + description: >- + More than 3 workload-authenticated {{ $labels.kind }} requests + (register/connect/disconnect/shutdown/result) have been rejected + as durable conflicts in the last 15 minutes; this is a distinct, + tighter-scoped signal than the generic 4xx ratio above and can + indicate a raced/duplicate GameServer registration, a replayed + result, or a reconnect fencing bug rather than ordinary client + noise. Correlate with server_{{ $labels.kind }} "conflict"-stage + log events for the affected match/server IDs. + runbook_url: https://example.invalid/cosmic-clash/runbooks/control-plane-api - name: cosmic-clash.allocator rules: - alert: CosmicClashAllocatorQuotaDenials diff --git a/multiplayer-next.md b/multiplayer-next.md index b30f0e77..36bd9117 100644 --- a/multiplayer-next.md +++ b/multiplayer-next.md @@ -1193,7 +1193,7 @@ production fallback. | 8.7 `[D:7.6,8.3]` | **IN PROGRESS.** Pure Go ticket policy binds the expected App ID and verified identity, rejects expiry/replay/wrong app/malformed tickets, and consumes each ticket once. `cmd/control-plane` now wires `SessionIssuer: store.PostgresSessions{DB: db}` (same discovery/fix pattern as §8.10's `ResultSubmitter`: the adapter already correctly implemented `Issue`, just wasn't wired, so `/v1/session/steam` 503'd even before considering whether `SteamLogin` — the real, still-correctly-unwired Steam blocker — was available) | `server/domain/auth.go` covers single-use and binding invariants; real `AuthenticateUserTicket` backend adapter, bans, publisher secret store and Steam verification remain. A separate test-only `server/cmd/testkit-api` binary (never referenced by any Dockerfile/K8s manifest) substitutes a fake Steam login accepting any non-empty ticket, enabling §8.40's real Go+Postgres+Godot integration test | | 8.8 `[D:8.7]` | **IN PROGRESS.** Pure Go session policy issues opaque short-lived tokens, stores only digests, authenticates by verified player identity and supports revocation; the production control-plane uses bounded atomic account+IP request limits | `server/domain/auth.go` covers wrong-token/expiry/revocation behavior; `server/store/session_sql.go` provides durable digest/revocation persistence and `server/api/rate_limit.go` plus `cmd/control-plane` provide per-replica request limiting; distributed revocation coordination and live Steam/session integration remain | | 8.9 `[D:8.4,8.7]` | **IN PROGRESS.** Pure Go join policy binds SteamID/player/match/server/team/slot/protocol/expiry, rejects duplicate roster slots, permits same-identity reclaim through backend loss and fences prior server-owned generations. A workload-authenticated durable lease API atomically claims generations and records exact-generation disconnects against the allocation/match/server/participant roster. Allocated Godot startup now fails closed without valid control-plane lease configuration, and admission awaits one bounded durable claim before publishing the roster entry; definitive conflicts fail closed, while a known nonzero same-process generation may reconnect during an outage and queues its connect/disconnect sequence for ordered reconciliation. A fresh process never guesses generation one offline and can adopt a later backend generation only from a durably disconnected lease | `server/domain/reconnect.go`, `server/store/server_connection_sql.go`, migration 0011, `/servers/{serverId}/{connect|disconnect}`, `connection_lease_client.gd`, and adversarial tests cover missing workload configuration, active duplicate claims, stale disconnect fencing, exact 60-second reclaim, process recovery, wrong binding, initial assignment expiry, malformed/skipped responses, ordered outage rules, retry-safe active receipts, and migration backfill. Admission rechecks drain, token expiry, and peer presence after the awaited claim and releases a claim that became unusable. Live PostgreSQL/Godot process-restart and outage recovery verification remains | -| 8.10 `[D:8.5,8.31]` | **IN PROGRESS.** Dependency-free workload credential policy validates an adapter-verified signature, issuer/audience/time bounds, namespace/service account, pod UID, GameServer UID, allocation ID, match ID and server ID before result submission; `server/workload` now parses projected compact JWTs, rejects unsigned/malformed/ambiguous tokens, verifies signing input through an injected trust callback, and delegates exact binding to the domain policy; the API exposes the workload-authenticated server result route. **`WorkloadVerify` is now wired for real** -- the blocker this row previously named (`cmd/control-plane/main.go` never wiring it, so `/register` and `/result` 503'd on every real request) is closed by taking a different, equally-valid approach instead of the Kubernetes-JWT one: `serverMutation` only ever compares `WorkloadBinding.ServerID`/`.MatchID` (and `AdvanceServerRegistration` only additionally needs `.AllocationID`) -- nothing downstream requires Namespace/ServiceAcct/PodUID/GameServerUID, so a Kubernetes-issued token was never actually required, just the assumption that it was. `server/workload/signed_token.go` mints and verifies a short-lived HMAC-signed token with a secret only the control plane holds (the same trust model `domain.SessionStore` already uses for player sessions), needing no cluster JWKS/TokenReview to validate -- signature and expiry are fully self-contained. The token binds only `allocation_id` (deliberately, not `match_id`/`server_id` too: it must be requestable in the same Agones request that asks for a server, before Agones has picked one) -- `store.AllocationBindingByAllocationID` resolves `match_id`/`server_id` durably from the `allocations` table at verify time, so a token can never claim a pairing that wasn't actually, durably allocated. `api.WorkloadVerifierFromSignedToken` combines both and is wired into `cmd/control-plane` (`--workload-secret`/`COSMIC_CLASH_WORKLOAD_SECRET`; a startup warning fires and the route stays 503 if it's left unset) and `cmd/testkit-api` (fixed test secret). **The delivery channel is now wired too**: `agones.Client` gains `WorkloadSecret`/`WorkloadTokenTTL` -- when set, `Allocate` mints a token for the allocation and requests it as a third `cosmic-clash.io/workload-token` annotation alongside match-id/allocation-id, wired from `cmd/allocator`'s own `--workload-secret`/`COSMIC_CLASH_WORKLOAD_SECRET` (must match the control plane's); `supervisor.Supervisor.workloadToken()` resolves the bearer credential with an explicit `--workload-token-path` (kept for a possible future Kubernetes-JWT path) always winning, otherwise falling back to that same annotation -- the same fallback pattern `matchID()` already used for `cosmic-clash.io/match-id`. `WorkloadTokenPath` is accordingly no longer required at construction time | `server/domain/workload.go`, `server/workload/jwt.go`, `server/api/service.go` and adversarial tests reject every binding mutation, missing/unverified signature, `none`/malformed JWT, ambiguous audience, server/match mismatch and time boundary; `server/testkit/pipeline_test.go` carries allocation identity through the offline result path; `cmd/control-plane` now wires `ResultSubmitter: store.PostgresResults{DB: db}` (a ready-made adapter that had been referenced from nowhere at all, not even a test); `server/workload/signed_token_test.go` covers round-trip, tampered payload, wrong secret, malformed input and the exact expiry boundary; `server/api/workload_verifier_integration_test.go` (opt-in, real PostgreSQL) covers acceptance against a real allocation row, rejection of an unrecorded allocation, that two distinct real allocations each resolve to their own and only their own match/server pairing, and the previously-503 `Service.WorkloadVerify` field itself now succeeding -- all verified clean with `-race` across multiple runs; `cmd/control-plane/main_test.go`'s `TestServerRoutesRequireWorkloadVerifyToBeWired` now documents and pins the misconfigured-secret case specifically, not the "always unwired" case; `server/agones/allocation_test.go`'s `TestAllocateRequestsAWorkloadTokenAnnotationWhenConfigured` and `server/supervisor/supervisor_test.go`'s new annotation-fallback/fails-closed pair cover the delivery channel end to end (mint -> annotation -> supervisor read -> Authorization header) short of a live cluster. **What's still missing**: this has never run against a real Agones cluster (only HTTP-level fakes), so the exact `object_meta` JSON casing Agones actually returns remains unverified from this sandbox (see `GameServer.ObjectMeta`'s existing caveat); live duplicate/conflict alerting also remains | +| 8.10 `[D:8.5,8.31]` | **IN PROGRESS.** Dependency-free workload credential policy validates an adapter-verified signature, issuer/audience/time bounds, namespace/service account, pod UID, GameServer UID, allocation ID, match ID and server ID before result submission; `server/workload` now parses projected compact JWTs, rejects unsigned/malformed/ambiguous tokens, verifies signing input through an injected trust callback, and delegates exact binding to the domain policy; the API exposes the workload-authenticated server result route. **`WorkloadVerify` is now wired for real** -- the blocker this row previously named (`cmd/control-plane/main.go` never wiring it, so `/register` and `/result` 503'd on every real request) is closed by taking a different, equally-valid approach instead of the Kubernetes-JWT one: `serverMutation` only ever compares `WorkloadBinding.ServerID`/`.MatchID` (and `AdvanceServerRegistration` only additionally needs `.AllocationID`) -- nothing downstream requires Namespace/ServiceAcct/PodUID/GameServerUID, so a Kubernetes-issued token was never actually required, just the assumption that it was. `server/workload/signed_token.go` mints and verifies a short-lived HMAC-signed token with a secret only the control plane holds (the same trust model `domain.SessionStore` already uses for player sessions), needing no cluster JWKS/TokenReview to validate -- signature and expiry are fully self-contained. The token binds only `allocation_id` (deliberately, not `match_id`/`server_id` too: it must be requestable in the same Agones request that asks for a server, before Agones has picked one) -- `store.AllocationBindingByAllocationID` resolves `match_id`/`server_id` durably from the `allocations` table at verify time, so a token can never claim a pairing that wasn't actually, durably allocated. `api.WorkloadVerifierFromSignedToken` combines both and is wired into `cmd/control-plane` (`--workload-secret`/`COSMIC_CLASH_WORKLOAD_SECRET`; a startup warning fires and the route stays 503 if it's left unset) and `cmd/testkit-api` (fixed test secret). **The delivery channel is now wired too**: `agones.Client` gains `WorkloadSecret`/`WorkloadTokenTTL` -- when set, `Allocate` mints a token for the allocation and requests it as a third `cosmic-clash.io/workload-token` annotation alongside match-id/allocation-id, wired from `cmd/allocator`'s own `--workload-secret`/`COSMIC_CLASH_WORKLOAD_SECRET` (must match the control plane's); `supervisor.Supervisor.workloadToken()` resolves the bearer credential with an explicit `--workload-token-path` (kept for a possible future Kubernetes-JWT path) always winning, otherwise falling back to that same annotation -- the same fallback pattern `matchID()` already used for `cosmic-clash.io/match-id`. `WorkloadTokenPath` is accordingly no longer required at construction time | `server/domain/workload.go`, `server/workload/jwt.go`, `server/api/service.go` and adversarial tests reject every binding mutation, missing/unverified signature, `none`/malformed JWT, ambiguous audience, server/match mismatch and time boundary; `server/testkit/pipeline_test.go` carries allocation identity through the offline result path; `cmd/control-plane` now wires `ResultSubmitter: store.PostgresResults{DB: db}` (a ready-made adapter that had been referenced from nowhere at all, not even a test); `server/workload/signed_token_test.go` covers round-trip, tampered payload, wrong secret, malformed input and the exact expiry boundary; `server/api/workload_verifier_integration_test.go` (opt-in, real PostgreSQL) covers acceptance against a real allocation row, rejection of an unrecorded allocation, that two distinct real allocations each resolve to their own and only their own match/server pairing, and the previously-503 `Service.WorkloadVerify` field itself now succeeding -- all verified clean with `-race` across multiple runs; `cmd/control-plane/main_test.go`'s `TestServerRoutesRequireWorkloadVerifyToBeWired` now documents and pins the misconfigured-secret case specifically, not the "always unwired" case; `server/agones/allocation_test.go`'s `TestAllocateRequestsAWorkloadTokenAnnotationWhenConfigured` and `server/supervisor/supervisor_test.go`'s new annotation-fallback/fails-closed pair cover the delivery channel end to end (mint -> annotation -> supervisor read -> Authorization header) short of a live cluster. **Duplicate/conflict alerting is now wired**: `observability.Metrics.ObserveServerConflict(kind)` adds a dedicated `cosmic_clash_api_server_conflicts_total{kind}` counter (bounded to `register`/`connect`/`disconnect`/`shutdown`/`result`, matching `serverMutation`'s own routes), incremented at every `domain.ErrConflict`/`ErrResultConflict` branch in `serverMutation` -- deliberately separate from `ObserveAPI`'s generic 4xx-class bucket, which also catches ordinary client noise (malformed bodies, expired tokens) that isn't a duplicate/conflict signal at all. `deploy/observability/prometheus-rules.yaml` adds `CosmicClashControlPlaneServerConflicts`, alongside the existing p95/5xx rules, firing on >3 conflicts of one kind in 15 minutes. **What's still missing**: this has never run against a real Agones cluster (only HTTP-level fakes), so the exact `object_meta` JSON casing Agones actually returns remains unverified from this sandbox (see `GameServer.ObjectMeta`'s existing caveat); the alert itself has only been validated statically (`scripts/verify_observability_manifests.py`), never against a live Prometheus/Alertmanager firing on real traffic | | 8.11 `[D:8.1]` | **DONE.** Write the threat model: forged clients/replay/queues/results, floods/bots, pod/insider compromise, gameplay and API DDoS, SDR signing-key theft, dependencies and denial-of-wallet | [`docs/THREAT-MODEL.md`](docs/THREAT-MODEL.md) records prevention, detection/response, owner and residual risk for every threat; it separates offline CA/online signer, client/game-server/PostgreSQL/Redis trust boundaries and recovery behavior | | 8.12 `[D:8.11]` | **IN PROGRESS.** Provider-neutral Kubernetes baseline enforces restricted namespace admission, non-root/read-only/no-capability workloads, separate service accounts, default-deny networking and explicit edge/data/DNS/Agones/metrics flows; application manifests consume externally populated Secret objects; the Go API and allocator now have bounded per-replica account+IP rate/quota boundaries and metrics endpoints, plus an atomic operator-controlled degraded-mode gate that sheds new login/queue/proposal work while preserving live-match paths | `deploy/k8s/base/` includes hardened control-plane, allocator, and game-server resources; the control-plane now has health probes, a disruption budget, explicit zero-unavailable/one-surge rolling updates with graceful termination, failure-domain spreading/anti-affinity, and Secret-backed `COSMIC_CLASH_POSTGRES_DSN`/`COSMIC_CLASH_WORKLOAD_SECRET` runtime wiring; the authenticated WebSocket now requires RFC 6455 version 13, enforces a bounded 64 KiB frame size, two-minute idle deadline, 120-message/minute inbound budget, and bounded per-player fan-out; `server/security/test_kubernetes_policies.py`, `server/api/rate_limit.go`, `server/api/admission.go`, allocator metrics tests and adversarial checks cover static hardening, secret-reference invariants, fixed-window limits, bounded labels/key memory, atomic account+IP charging, route classification, concurrent toggling and degraded responses; production `cmd/control-plane` now wires the bounded limiter by default with `--rate-limit`, `--rate-limit-window`, and `--rate-limit-max-keys`, and the base Deployment declares those defaults explicitly; start with control-plane `--degraded`, enable with SIGUSR1, disable with SIGUSR2; private-store provisioning, distributed/global quotas, edge DDoS/WAF/origin shielding, encrypted backups and live policy/load tests remain | | 8.13 `[D:8.12]` | **IN PROGRESS.** Docker/Kubernetes references are digest-pinned, a dependency-free checker rejects mutable tags/plaintext credentials and concrete release overlays can reject template digests; release documentation defines SBOM, dependency/image scanning, signing, admission verification and a 24-hour critical-fix SLA | `scripts/verify_supply_chain.py`, `server/security/test_supply_chain.py`, `docs/SUPPLY-CHAIN.md` and `.github/workflows/supply-chain.yml` cover repository policy and provenance requirements; registry SBOM/scan/sign/admission execution and a concrete production overlay remain | diff --git a/server/api/service.go b/server/api/service.go index 6f0efe9c..c5b6acaf 100644 --- a/server/api/service.go +++ b/server/api/service.go @@ -652,6 +652,7 @@ func (s *Service) serverMutation(w http.ResponseWriter, r *http.Request) { stage := "invalid" if errors.Is(err, domain.ErrConflict) { stage = "conflict" + s.Metrics.ObserveServerConflict("register") writeError(w, http.StatusConflict, "conflict") } else { writeError(w, http.StatusUnprocessableEntity, "invalid_request") @@ -702,6 +703,7 @@ func (s *Service) serverMutation(w http.ResponseWriter, r *http.Request) { } if err != nil { if errors.Is(err, domain.ErrConflict) { + s.Metrics.ObserveServerConflict(parts[1]) writeError(w, http.StatusConflict, "conflict") } else { // The request has already passed schema and workload checks. An @@ -747,6 +749,7 @@ func (s *Service) serverMutation(w http.ResponseWriter, r *http.Request) { stage := "invalid" if errors.Is(err, domain.ErrConflict) { stage = "conflict" + s.Metrics.ObserveServerConflict("shutdown") writeError(w, http.StatusConflict, "conflict") } else { writeError(w, http.StatusUnprocessableEntity, "invalid_request") @@ -777,6 +780,7 @@ func (s *Service) serverMutation(w http.ResponseWriter, r *http.Request) { stage := "invalid" if errors.Is(err, domain.ErrResultConflict) || strings.Contains(err.Error(), "conflict") { stage = "conflict" + s.Metrics.ObserveServerConflict("result") writeError(w, http.StatusConflict, "conflict") } else { writeError(w, http.StatusUnprocessableEntity, "invalid_request") diff --git a/server/api/service_test.go b/server/api/service_test.go index 64adfe63..4c4dfdf2 100644 --- a/server/api/service_test.go +++ b/server/api/service_test.go @@ -1447,6 +1447,43 @@ func TestServerRegistrationAPIRequiresBoundWorkloadAndValidDigest(t *testing.T) response.Body.Close() } +func TestServerMutationConflictsAreExportedAsADistinctPrometheusCounter(t *testing.T) { + now := time.Unix(1000, 0).UTC() + binding := domain.WorkloadBinding{AllocationID: "allocation-1", MatchID: "match-1", ServerID: "server-1"} + registrar := &serverRegistrarSpy{err: domain.ErrConflict} + metrics := observability.NewMetrics() + service := &Service{Now: func() time.Time { return now }, Metrics: metrics, WorkloadVerify: func(token string, _ time.Time) (domain.WorkloadBinding, error) { + if token != "workload-token" { + return domain.WorkloadBinding{}, errors.New("bad token") + } + return binding, nil + }, ServerRegistrar: registrar} + server := httptest.NewServer(service.Handler()) + defer server.Close() + body := `{"match_id":"match-1","protocol_version":1,"image_digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","assignment_ready":false}` + req, _ := http.NewRequest(http.MethodPost, server.URL+"/v1/servers/server-1/register", strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer workload-token") + req.Header.Set("Idempotency-Key", "register-key-123456") + response, err := http.DefaultClient.Do(req) + if err != nil || response.StatusCode != http.StatusConflict { + t.Fatalf("status=%v err=%v", response.StatusCode, err) + } + response.Body.Close() + + metricsResponse, err := http.Get(server.URL + "/metrics") + if err != nil { + t.Fatal(err) + } + defer metricsResponse.Body.Close() + exported, err := io.ReadAll(metricsResponse.Body) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(exported), `cosmic_clash_api_server_conflicts_total{kind="register"} 1`) { + t.Fatalf("register conflict was not exported: %s", exported) + } +} + func TestServerShutdownAPIRequiresBoundWorkloadAndDelegatesAcknowledgement(t *testing.T) { now := time.Unix(1000, 0).UTC() binding := domain.WorkloadBinding{AllocationID: "allocation-1", MatchID: "match-1", ServerID: "server-1"} diff --git a/server/observability/metrics.go b/server/observability/metrics.go index 6c4bb6a2..f57704f8 100644 --- a/server/observability/metrics.go +++ b/server/observability/metrics.go @@ -11,20 +11,50 @@ import ( // Metrics is a bounded in-process collector for API request health. Operation // names are normalized to a fixed vocabulary before storage. type Metrics struct { - mu sync.Mutex - counts map[metricKey]uint64 - sums map[metricKey]time.Duration - buckets map[metricKey][]uint64 + mu sync.Mutex + counts map[metricKey]uint64 + sums map[metricKey]time.Duration + buckets map[metricKey][]uint64 + conflicts map[string]uint64 } type metricKey struct{ operation, status string } +// serverConflictKinds is the fixed, bounded label vocabulary for +// ObserveServerConflict, matching the workload-authenticated server mutation +// routes in api.Service.serverMutation. An unrecognized kind is folded into +// "other" so a caller mistake can never grow the label set. +var serverConflictKinds = []string{"register", "connect", "disconnect", "shutdown", "result"} + // apiLatencyBucketsSeconds is deliberately fixed and small. It is wide enough // to query the documented 250 ms API SLO while keeping the exporter bounded. var apiLatencyBucketsSeconds = []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10} func NewMetrics() *Metrics { - return &Metrics{counts: make(map[metricKey]uint64), sums: make(map[metricKey]time.Duration), buckets: make(map[metricKey][]uint64)} + return &Metrics{counts: make(map[metricKey]uint64), sums: make(map[metricKey]time.Duration), buckets: make(map[metricKey][]uint64), conflicts: make(map[string]uint64)} +} + +// ObserveServerConflict records one workload-authenticated server mutation +// (register/connect/disconnect/shutdown/result) that a durable domain.ErrConflict +// or domain.ErrResultConflict rejected. This is a distinct counter from +// ObserveAPI's generic 4xx class specifically so a spike here — duplicate +// registration, a raced reconnect, a replayed result — can be alerted on +// without also firing on ordinary client-side 4xx noise (malformed bodies, +// expired tokens) that shares the same status class. +func (m *Metrics) ObserveServerConflict(kind string) { + if m == nil { + return + } + normalized := "other" + for _, allowed := range serverConflictKinds { + if kind == allowed { + normalized = allowed + break + } + } + m.mu.Lock() + m.conflicts[normalized]++ + m.mu.Unlock() } func (m *Metrics) ObserveAPI(operation string, statusCode int, duration time.Duration) { @@ -74,6 +104,15 @@ func (m *Metrics) WritePrometheus(w io.Writer) error { counts[key], sums[key] = m.counts[key], m.sums[key] buckets[key] = append([]uint64(nil), m.buckets[key]...) } + conflictKinds := make([]string, 0, len(m.conflicts)) + for kind := range m.conflicts { + conflictKinds = append(conflictKinds, kind) + } + sort.Strings(conflictKinds) + conflicts := make(map[string]uint64, len(conflictKinds)) + for _, kind := range conflictKinds { + conflicts[kind] = m.conflicts[kind] + } m.mu.Unlock() if _, err := io.WriteString(w, "# TYPE cosmic_clash_api_requests_total counter\n# TYPE cosmic_clash_api_latency_seconds histogram\n"); err != nil { return err @@ -89,6 +128,16 @@ func (m *Metrics) WritePrometheus(w io.Writer) error { return err } } + if len(conflictKinds) > 0 { + if _, err := io.WriteString(w, "# TYPE cosmic_clash_api_server_conflicts_total counter\n"); err != nil { + return err + } + for _, kind := range conflictKinds { + if _, err := fmt.Fprintf(w, "cosmic_clash_api_server_conflicts_total{kind=\"%s\"} %d\n", kind, conflicts[kind]); err != nil { + return err + } + } + } return nil } diff --git a/server/observability/metrics_test.go b/server/observability/metrics_test.go index c318f996..0b0cc24a 100644 --- a/server/observability/metrics_test.go +++ b/server/observability/metrics_test.go @@ -43,3 +43,48 @@ func TestMetricsHistogramUsesCumulativeBoundarySemantics(t *testing.T) { t.Fatalf("250ms observation entered an earlier bucket: %s", text) } } + +func TestMetricsServerConflictsAreCountedByKindAndBounded(t *testing.T) { + m := NewMetrics() + m.ObserveServerConflict("register") + m.ObserveServerConflict("register") + m.ObserveServerConflict("result") + m.ObserveServerConflict("crafted-unknown-kind") + var output strings.Builder + if err := m.WritePrometheus(&output); err != nil { + t.Fatal(err) + } + text := output.String() + if !strings.Contains(text, "# TYPE cosmic_clash_api_server_conflicts_total counter") { + t.Fatalf("missing conflict counter TYPE line: %s", text) + } + if !strings.Contains(text, `cosmic_clash_api_server_conflicts_total{kind="register"} 2`) { + t.Fatalf("register conflicts not counted correctly: %s", text) + } + if !strings.Contains(text, `cosmic_clash_api_server_conflicts_total{kind="result"} 1`) { + t.Fatalf("result conflicts not counted correctly: %s", text) + } + if !strings.Contains(text, `cosmic_clash_api_server_conflicts_total{kind="other"} 1`) { + t.Fatalf("unknown kind was not folded into the bounded 'other' label: %s", text) + } + if strings.Contains(text, "crafted-unknown-kind") { + t.Fatalf("unbounded conflict kind label leaked: %s", text) + } +} + +func TestMetricsServerConflictAbsentWhenUnobserved(t *testing.T) { + m := NewMetrics() + m.ObserveAPI("queue", 200, time.Millisecond) + var output strings.Builder + if err := m.WritePrometheus(&output); err != nil { + t.Fatal(err) + } + if strings.Contains(output.String(), "cosmic_clash_api_server_conflicts_total") { + t.Fatalf("conflict counter should be omitted entirely until first observed: %s", output.String()) + } +} + +func TestMetricsServerConflictNilReceiverIsANoop(t *testing.T) { + var m *Metrics + m.ObserveServerConflict("register") // must not panic +}