diff --git a/deploy/k8s/base/control-plane-deployment.yaml b/deploy/k8s/base/control-plane-deployment.yaml index a72096e6..5225cc61 100644 --- a/deploy/k8s/base/control-plane-deployment.yaml +++ b/deploy/k8s/base/control-plane-deployment.yaml @@ -61,7 +61,7 @@ spec: containerPort: 8080 readinessProbe: httpGet: - path: /healthz + path: /readyz port: http periodSeconds: 5 timeoutSeconds: 2 diff --git a/multiplayer-next.md b/multiplayer-next.md index a9c00121..7b61cf14 100644 --- a/multiplayer-next.md +++ b/multiplayer-next.md @@ -1646,3 +1646,5 @@ Per-IP API limiting now resolves the client behind the edge gateway instead of c Allocator probes now distinguish process liveness from useful progress. `/healthz` remains live during dependency outages, while `/readyz` starts unavailable and requires a fully successful provider-list, Ready-registration, and worker cycle within `--readiness-max-stale` (30 seconds in the base deployment). The Kubernetes/Agones HTTP path is bounded by `--provider-timeout=10s`, so an unavailable provider cannot leave readiness green indefinitely; startup rejects a freshness window shorter than the poll interval plus provider timeout, and the probe listener has its own header-read deadline. Boundary and HTTP tests cover startup, exact staleness, clock reversal, recovery, method rejection, and metrics coexistence. The timeout boundary is enforced inside both network adapters as well as in the production allocator wiring: an `agones.Client` or game-server `Supervisor` constructed without an injected HTTP client now receives a ten-second client rather than Go's unbounded `http.DefaultClient`. This prevents alternate binaries, tests, and future callers from restoring an infinite GameServer, roster, registration, or SDK wait by omission. + +Control-plane probes now separate liveness from datastore readiness too. `/healthz` proves the process can serve without restarting it during a PostgreSQL outage; `/readyz` runs a one-second-bounded `PingContext` and the Deployment routes traffic only to replicas whose core durable store responds. Probe and metrics routes bypass the player request limiter, so operator-selected low limits cannot make Kubernetes evict a healthy replica. Missing checks, datastore errors, non-GET methods, and successful recovery are covered by API tests. diff --git a/server/api/rate_limit_test.go b/server/api/rate_limit_test.go index a6139005..8889d018 100644 --- a/server/api/rate_limit_test.go +++ b/server/api/rate_limit_test.go @@ -56,7 +56,7 @@ func TestRateLimitedHTTPBoundaryReturnsGeneric429(t *testing.T) { service := &Service{RateLimiter: limiter, Now: func() time.Time { return time.Unix(1000, 0) }} server := httptest.NewServer(service.Handler()) defer server.Close() - request, err := http.NewRequest(http.MethodGet, server.URL+"/healthz", nil) + request, err := http.NewRequest(http.MethodGet, server.URL+"/unknown", nil) if err != nil { t.Fatal(err) } @@ -66,10 +66,10 @@ func TestRateLimitedHTTPBoundaryReturnsGeneric429(t *testing.T) { t.Fatal(err) } _ = response.Body.Close() - if response.StatusCode != http.StatusOK { + if response.StatusCode != http.StatusNotFound { t.Fatalf("first request status = %d", response.StatusCode) } - request, _ = http.NewRequest(http.MethodGet, server.URL+"/healthz", strings.NewReader("")) + request, _ = http.NewRequest(http.MethodGet, server.URL+"/unknown", strings.NewReader("")) request.Header.Set("Authorization", "Bearer secret-session:secret-token") response, err = server.Client().Do(request) if err != nil { @@ -128,7 +128,7 @@ func TestRateLimiterSeparatesClientsBehindTrustedGateway(t *testing.T) { server := httptest.NewServer(service.Handler()) defer server.Close() request := func(forwarded string) int { - req, requestErr := http.NewRequest(http.MethodGet, server.URL+"/healthz", nil) + req, requestErr := http.NewRequest(http.MethodGet, server.URL+"/unknown", nil) if requestErr != nil { t.Fatal(requestErr) } @@ -140,10 +140,10 @@ func TestRateLimiterSeparatesClientsBehindTrustedGateway(t *testing.T) { response.Body.Close() return response.StatusCode } - if got := request("198.51.100.1"); got != http.StatusOK { + if got := request("198.51.100.1"); got != http.StatusNotFound { t.Fatalf("first client status = %d", got) } - if got := request("198.51.100.2"); got != http.StatusOK { + if got := request("198.51.100.2"); got != http.StatusNotFound { t.Fatalf("second client behind gateway status = %d", got) } if got := request("198.51.100.1"); got != http.StatusTooManyRequests { diff --git a/server/api/service.go b/server/api/service.go index bd038bdc..e80153c2 100644 --- a/server/api/service.go +++ b/server/api/service.go @@ -100,6 +100,7 @@ type AssignmentView struct { type AssignmentProvider func(context.Context, string, string, time.Time) (AssignmentView, error) type RosterProvider func(context.Context, domain.WorkloadBinding, time.Time) ([][]byte, error) +type ReadinessCheck func(context.Context) error type Service struct { Sessions *domain.SessionStore @@ -129,6 +130,7 @@ type Service struct { RateLimiter *RateLimiter ClientIPs *ClientIPResolver Admission AdmissionController + ReadinessCheck ReadinessCheck // Log receives a credential-safe structured event for lifecycle-relevant // reads and mutations. Nil // is a valid, silent no-op -- every call site must stay optional so @@ -185,6 +187,7 @@ func (s *Service) logProposalOutcome(proposalID string, proposal domain.Proposal func (s *Service) Handler() http.Handler { mux := http.NewServeMux() mux.HandleFunc("/healthz", s.health) + mux.HandleFunc("/readyz", s.ready) mux.HandleFunc("/metrics", s.metrics) mux.HandleFunc("/v1/session/steam", s.steamSession) mux.HandleFunc("/v1/queue", s.queueCreate) @@ -208,6 +211,10 @@ func (s *Service) Handler() http.Handler { var handler http.Handler = mux if s.RateLimiter != nil { handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/healthz" || r.URL.Path == "/readyz" || r.URL.Path == "/metrics" { + mux.ServeHTTP(w, r) + return + } if !s.RateLimiter.AllowKeys(requestRateKeys(r, s.ClientIPs), s.now()) { writeError(w, http.StatusTooManyRequests, "rate_limited") return @@ -229,7 +236,7 @@ func (s *Service) Handler() http.Handler { return handler } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/metrics" || r.URL.Path == "/healthz" || strings.HasSuffix(r.URL.Path, "/events") { + if r.URL.Path == "/metrics" || r.URL.Path == "/healthz" || r.URL.Path == "/readyz" || strings.HasSuffix(r.URL.Path, "/events") { handler.ServeHTTP(w, r) return } @@ -337,10 +344,32 @@ func (s *Service) steamSession(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]any{"player_id": session.PlayerID, "expires_at": session.ExpiresAt, "access_token": session.SessionID + ":" + token}) } -func (s *Service) health(w http.ResponseWriter, _ *http.Request) { +func (s *Service) health(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writeError(w, http.StatusMethodNotAllowed, "method_not_allowed") + return + } writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) } +func (s *Service) ready(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writeError(w, http.StatusMethodNotAllowed, "method_not_allowed") + return + } + if s.ReadinessCheck == nil { + writeError(w, http.StatusServiceUnavailable, "not_ready") + return + } + ctx, cancel := context.WithTimeout(r.Context(), time.Second) + defer cancel() + if err := s.ReadinessCheck(ctx); err != nil { + writeError(w, http.StatusServiceUnavailable, "not_ready") + return + } + writeJSON(w, http.StatusOK, map[string]string{"status": "ready"}) +} + type queueCreateRequest struct { TicketID string `json:"ticket_id"` Playlist string `json:"playlist"` diff --git a/server/api/service_test.go b/server/api/service_test.go index 756e299b..92836c63 100644 --- a/server/api/service_test.go +++ b/server/api/service_test.go @@ -1487,6 +1487,59 @@ func TestMetricsEndpointExportsBoundedAPILatencyAndSkipsItsOwnScrape(t *testing. } } +func TestControlPlaneLivenessAndDatastoreReadinessAreIndependent(t *testing.T) { + ready := false + checks := 0 + limiter, err := NewRateLimiter(1, time.Minute, 8) + if err != nil { + t.Fatal(err) + } + service := &Service{ + RateLimiter: limiter, + Now: func() time.Time { return time.Unix(1000, 0) }, + ReadinessCheck: func(context.Context) error { + checks++ + if !ready { + return errors.New("database unavailable") + } + return nil + }, + } + handler := service.Handler() + status := func(method, path string) int { + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(method, path, nil)) + return recorder.Code + } + if got := status(http.MethodGet, "/healthz"); got != http.StatusOK { + t.Fatalf("liveness during datastore outage = %d", got) + } + if got := status(http.MethodGet, "/readyz"); got != http.StatusServiceUnavailable { + t.Fatalf("readiness during datastore outage = %d", got) + } + ready = true + if got := status(http.MethodGet, "/readyz"); got != http.StatusOK { + t.Fatalf("recovered readiness = %d", got) + } + if got := status(http.MethodGet, "/healthz"); got != http.StatusOK { + t.Fatalf("repeated probe was incorrectly rate limited: %d", got) + } + if checks != 2 { + t.Fatalf("readiness checks = %d", checks) + } + if got := status(http.MethodPost, "/readyz"); got != http.StatusMethodNotAllowed { + t.Fatalf("readiness mutation status = %d", got) + } +} + +func TestControlPlaneReadinessFailsClosedWithoutCheck(t *testing.T) { + recorder := httptest.NewRecorder() + (&Service{}).Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/readyz", nil)) + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("unconfigured readiness status = %d", recorder.Code) + } +} + func TestProbeAPIUsesServerEvidenceAndRejectsClientRTTField(t *testing.T) { now := time.Unix(1000, 0).UTC() sessions := domain.NewSessionStore() diff --git a/server/cmd/control-plane/main.go b/server/cmd/control-plane/main.go index 91882dc8..893b4151 100644 --- a/server/cmd/control-plane/main.go +++ b/server/cmd/control-plane/main.go @@ -144,6 +144,7 @@ func newAPIService(db *sql.DB, workloadSecret string, indexes ...api.CandidateIn CandidateIndex: candidateIndex, ProbeRecorder: store.PostgresQueue{DB: db}, WorkloadVerify: api.WorkloadVerifierFromSignedToken([]byte(workloadSecret), db), + ReadinessCheck: db.PingContext, Now: func() time.Time { return time.Now().UTC() }, Log: logEvent, Metrics: observability.NewMetrics(), diff --git a/server/security/test_kubernetes_policies.py b/server/security/test_kubernetes_policies.py index 8c94c170..0043e0ef 100644 --- a/server/security/test_kubernetes_policies.py +++ b/server/security/test_kubernetes_policies.py @@ -47,7 +47,7 @@ class KubernetesPolicyTest(unittest.TestCase): def test_control_plane_has_health_rollout_and_failure_domain_guards(self): deployment = self.read("control-plane-deployment.yaml") for required in ( - "readinessProbe:", "livenessProbe:", "path: /healthz", "port: http", + "readinessProbe:", "livenessProbe:", "path: /readyz", "path: /healthz", "port: http", "type: RollingUpdate", "maxUnavailable: 0", "maxSurge: 1", "terminationGracePeriodSeconds: 10", "topologySpreadConstraints:", "maxSkew: 1",