diff --git a/multiplayer-todo.md b/multiplayer-todo.md index 006d7e3e..da435ffd 100644 --- a/multiplayer-todo.md +++ b/multiplayer-todo.md @@ -1237,7 +1237,7 @@ the local/CI/community transport, not a silent production fallback. | # | Task | Acceptance | |---|---|---| | 8.44 `[D:8.3,8.4,8.28,8.31]` | **IN PROGRESS.** Go observability package encodes queue/proposal/match/server IDs and lifecycle stage in structured events while recursively redacting auth/relay tokens and credentials | `server/observability/` covers correlation fields, nested secret redaction and unnamed-event rejection; production logger/metrics/traces/replay integration and secret-canary coverage remain | -| 8.45 `[D:8.2,8.44]` | Dashboards/alerts for wait/MMR/RTT, proposals, allocation/Ready/image pull, connect/no-show, tick/crash/flood, result conflict/lag, abandons and cost | Each SLO and security/cost signal has an exercised alert and runbook | +| 8.45 `[D:8.2,8.44]` | **IN PROGRESS.** Go observability package turns the documented RTT, allocation/connect latency, result-success, API-latency and tick/headroom thresholds into executable window checks | `server/observability/slo.go` covers healthy/violating/empty windows; production metrics export, dashboards, alert routing, wait/MMR/proposal/flood/cost series and runbooks remain | | 8.46 `[D:8.5,8.7,8.9,8.10,8.14,8.18,8.21,8.23,8.25]` | **IN PROGRESS.** Go unit/race coverage spans the current domain/store/supervisor policies, and fuzz targets now exercise queue input, result payload hashing and revision events | `server/domain/*_test.go`, `server/store/*_test.go`, `server/supervisor/*_test.go` and `server/domain/fuzz_test.go` pass normal/race suites; `go test -fuzz`, PostgreSQL concurrency/migration execution, fake Steam/allocator and full lost-Redis/transaction fixtures remain | | 8.47 `[D:8.7,8.30]` | **IN PROGRESS.** Offline testkit provides deterministic fake Steam verification and fake allocation with forced failure injection | `server/testkit/` covers verified identity/replay and cloud-free forced allocation failure; API/Compose integration and exhaustive success/failure matrix remain | | 8.48 `[D:8.10,8.14,8.17,8.18,8.27,8.31,8.35,8.47]` | Second Compose flow: fake backend → queue/proposal → process-ready/allocation/assignment-ready → ENet roster → result ack → shutdown; do not edit Phase 6 fixture | Both server models have independent green gates; existing Make invocations remain unchanged | diff --git a/server/observability/slo.go b/server/observability/slo.go new file mode 100644 index 00000000..42c5f13f --- /dev/null +++ b/server/observability/slo.go @@ -0,0 +1,70 @@ +package observability + +import ( + "sort" + "time" +) + +type SLOWindow struct { + RegionalRTT []time.Duration + Assignment []time.Duration + Connection []time.Duration + APILatency []time.Duration + Matches []bool + TickBacklog bool + Headroom float64 +} + +type SLOViolation struct { + Metric string + Reason string +} + +func EvaluateSLO(window SLOWindow) []SLOViolation { + violations := make([]SLOViolation, 0) + if percentile(window.RegionalRTT, .95) > 80*time.Millisecond { + violations = append(violations, SLOViolation{"regional_rtt_p95", "exceeds 80ms"}) + } + if percentile(window.Assignment, .95) > 5*time.Second || percentile(window.Assignment, .99) > 10*time.Second { + violations = append(violations, SLOViolation{"assignment_latency", "p95/p99 threshold exceeded"}) + } + if percentile(window.Connection, .95) > 5*time.Second { + violations = append(violations, SLOViolation{"connection_latency_p95", "exceeds 5s"}) + } + if ratio(window.Matches) < .999 { + violations = append(violations, SLOViolation{"allocation_result_success", "below 99.9%"}) + } + if percentile(window.APILatency, .95) > 250*time.Millisecond { + violations = append(violations, SLOViolation{"api_latency_p95", "exceeds 250ms"}) + } + if window.TickBacklog { + violations = append(violations, SLOViolation{"tick_health", "physics backlog detected"}) + } + if window.Headroom > 0 && window.Headroom < .30 { + violations = append(violations, SLOViolation{"resource_headroom", "below 30%"}) + } + return violations +} + +func percentile(values []time.Duration, p float64) time.Duration { + if len(values) == 0 { + return 0 + } + ordered := append([]time.Duration(nil), values...) + sort.Slice(ordered, func(i, j int) bool { return ordered[i] < ordered[j] }) + index := int(float64(len(ordered)-1) * p) + return ordered[index] +} + +func ratio(values []bool) float64 { + if len(values) == 0 { + return 1 + } + success := 0 + for _, value := range values { + if value { + success++ + } + } + return float64(success) / float64(len(values)) +} diff --git a/server/observability/slo_test.go b/server/observability/slo_test.go new file mode 100644 index 00000000..6e319306 --- /dev/null +++ b/server/observability/slo_test.go @@ -0,0 +1,27 @@ +package observability + +import ( + "testing" + "time" +) + +func TestEvaluateSLOAcceptsHealthyWindow(t *testing.T) { + window := SLOWindow{RegionalRTT: []time.Duration{20 * time.Millisecond, 40 * time.Millisecond}, Assignment: []time.Duration{time.Second}, Connection: []time.Duration{time.Second}, APILatency: []time.Duration{100 * time.Millisecond}, Matches: []bool{true, true, true}, Headroom: .50} + if violations := EvaluateSLO(window); len(violations) != 0 { + t.Fatalf("healthy window violations = %+v", violations) + } +} + +func TestEvaluateSLOFlagsEveryLaunchGate(t *testing.T) { + window := SLOWindow{RegionalRTT: []time.Duration{101 * time.Millisecond}, Assignment: []time.Duration{11 * time.Second}, Connection: []time.Duration{6 * time.Second}, APILatency: []time.Duration{251 * time.Millisecond}, Matches: []bool{true, false}, TickBacklog: true, Headroom: .29} + violations := EvaluateSLO(window) + if len(violations) != 7 { + t.Fatalf("violations = %+v", violations) + } +} + +func TestEvaluateSLODoesNotInventFailureForEmptyOptionalWindows(t *testing.T) { + if violations := EvaluateSLO(SLOWindow{}); len(violations) != 0 { + t.Fatalf("empty window violations = %+v", violations) + } +}