fix(multiplayer): gate API readiness on database

This commit is contained in:
Josh Creek
2026-09-02 19:14:21 +01:00
parent 1bce603c33
commit 51f8008a38
7 changed files with 95 additions and 10 deletions
+6 -6
View File
@@ -56,7 +56,7 @@ func TestRateLimitedHTTPBoundaryReturnsGeneric429(t *testing.T) {
service := &Service{RateLimiter: limiter, Now: func() time.Time { return time.Unix(1000, 0) }}
server := httptest.NewServer(service.Handler())
defer server.Close()
request, err := http.NewRequest(http.MethodGet, server.URL+"/healthz", nil)
request, err := http.NewRequest(http.MethodGet, server.URL+"/unknown", nil)
if err != nil {
t.Fatal(err)
}
@@ -66,10 +66,10 @@ func TestRateLimitedHTTPBoundaryReturnsGeneric429(t *testing.T) {
t.Fatal(err)
}
_ = response.Body.Close()
if response.StatusCode != http.StatusOK {
if response.StatusCode != http.StatusNotFound {
t.Fatalf("first request status = %d", response.StatusCode)
}
request, _ = http.NewRequest(http.MethodGet, server.URL+"/healthz", strings.NewReader(""))
request, _ = http.NewRequest(http.MethodGet, server.URL+"/unknown", strings.NewReader(""))
request.Header.Set("Authorization", "Bearer secret-session:secret-token")
response, err = server.Client().Do(request)
if err != nil {
@@ -128,7 +128,7 @@ func TestRateLimiterSeparatesClientsBehindTrustedGateway(t *testing.T) {
server := httptest.NewServer(service.Handler())
defer server.Close()
request := func(forwarded string) int {
req, requestErr := http.NewRequest(http.MethodGet, server.URL+"/healthz", nil)
req, requestErr := http.NewRequest(http.MethodGet, server.URL+"/unknown", nil)
if requestErr != nil {
t.Fatal(requestErr)
}
@@ -140,10 +140,10 @@ func TestRateLimiterSeparatesClientsBehindTrustedGateway(t *testing.T) {
response.Body.Close()
return response.StatusCode
}
if got := request("198.51.100.1"); got != http.StatusOK {
if got := request("198.51.100.1"); got != http.StatusNotFound {
t.Fatalf("first client status = %d", got)
}
if got := request("198.51.100.2"); got != http.StatusOK {
if got := request("198.51.100.2"); got != http.StatusNotFound {
t.Fatalf("second client behind gateway status = %d", got)
}
if got := request("198.51.100.1"); got != http.StatusTooManyRequests {
+31 -2
View File
@@ -100,6 +100,7 @@ type AssignmentView struct {
type AssignmentProvider func(context.Context, string, string, time.Time) (AssignmentView, error)
type RosterProvider func(context.Context, domain.WorkloadBinding, time.Time) ([][]byte, error)
type ReadinessCheck func(context.Context) error
type Service struct {
Sessions *domain.SessionStore
@@ -129,6 +130,7 @@ type Service struct {
RateLimiter *RateLimiter
ClientIPs *ClientIPResolver
Admission AdmissionController
ReadinessCheck ReadinessCheck
// Log receives a credential-safe structured event for lifecycle-relevant
// reads and mutations. Nil
// is a valid, silent no-op -- every call site must stay optional so
@@ -185,6 +187,7 @@ func (s *Service) logProposalOutcome(proposalID string, proposal domain.Proposal
func (s *Service) Handler() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/healthz", s.health)
mux.HandleFunc("/readyz", s.ready)
mux.HandleFunc("/metrics", s.metrics)
mux.HandleFunc("/v1/session/steam", s.steamSession)
mux.HandleFunc("/v1/queue", s.queueCreate)
@@ -208,6 +211,10 @@ func (s *Service) Handler() http.Handler {
var handler http.Handler = mux
if s.RateLimiter != nil {
handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/healthz" || r.URL.Path == "/readyz" || r.URL.Path == "/metrics" {
mux.ServeHTTP(w, r)
return
}
if !s.RateLimiter.AllowKeys(requestRateKeys(r, s.ClientIPs), s.now()) {
writeError(w, http.StatusTooManyRequests, "rate_limited")
return
@@ -229,7 +236,7 @@ func (s *Service) Handler() http.Handler {
return handler
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/metrics" || r.URL.Path == "/healthz" || strings.HasSuffix(r.URL.Path, "/events") {
if r.URL.Path == "/metrics" || r.URL.Path == "/healthz" || r.URL.Path == "/readyz" || strings.HasSuffix(r.URL.Path, "/events") {
handler.ServeHTTP(w, r)
return
}
@@ -337,10 +344,32 @@ func (s *Service) steamSession(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]any{"player_id": session.PlayerID, "expires_at": session.ExpiresAt, "access_token": session.SessionID + ":" + token})
}
func (s *Service) health(w http.ResponseWriter, _ *http.Request) {
func (s *Service) health(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeError(w, http.StatusMethodNotAllowed, "method_not_allowed")
return
}
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
}
func (s *Service) ready(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeError(w, http.StatusMethodNotAllowed, "method_not_allowed")
return
}
if s.ReadinessCheck == nil {
writeError(w, http.StatusServiceUnavailable, "not_ready")
return
}
ctx, cancel := context.WithTimeout(r.Context(), time.Second)
defer cancel()
if err := s.ReadinessCheck(ctx); err != nil {
writeError(w, http.StatusServiceUnavailable, "not_ready")
return
}
writeJSON(w, http.StatusOK, map[string]string{"status": "ready"})
}
type queueCreateRequest struct {
TicketID string `json:"ticket_id"`
Playlist string `json:"playlist"`
+53
View File
@@ -1487,6 +1487,59 @@ func TestMetricsEndpointExportsBoundedAPILatencyAndSkipsItsOwnScrape(t *testing.
}
}
func TestControlPlaneLivenessAndDatastoreReadinessAreIndependent(t *testing.T) {
ready := false
checks := 0
limiter, err := NewRateLimiter(1, time.Minute, 8)
if err != nil {
t.Fatal(err)
}
service := &Service{
RateLimiter: limiter,
Now: func() time.Time { return time.Unix(1000, 0) },
ReadinessCheck: func(context.Context) error {
checks++
if !ready {
return errors.New("database unavailable")
}
return nil
},
}
handler := service.Handler()
status := func(method, path string) int {
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(method, path, nil))
return recorder.Code
}
if got := status(http.MethodGet, "/healthz"); got != http.StatusOK {
t.Fatalf("liveness during datastore outage = %d", got)
}
if got := status(http.MethodGet, "/readyz"); got != http.StatusServiceUnavailable {
t.Fatalf("readiness during datastore outage = %d", got)
}
ready = true
if got := status(http.MethodGet, "/readyz"); got != http.StatusOK {
t.Fatalf("recovered readiness = %d", got)
}
if got := status(http.MethodGet, "/healthz"); got != http.StatusOK {
t.Fatalf("repeated probe was incorrectly rate limited: %d", got)
}
if checks != 2 {
t.Fatalf("readiness checks = %d", checks)
}
if got := status(http.MethodPost, "/readyz"); got != http.StatusMethodNotAllowed {
t.Fatalf("readiness mutation status = %d", got)
}
}
func TestControlPlaneReadinessFailsClosedWithoutCheck(t *testing.T) {
recorder := httptest.NewRecorder()
(&Service{}).Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/readyz", nil))
if recorder.Code != http.StatusServiceUnavailable {
t.Fatalf("unconfigured readiness status = %d", recorder.Code)
}
}
func TestProbeAPIUsesServerEvidenceAndRejectsClientRTTField(t *testing.T) {
now := time.Unix(1000, 0).UTC()
sessions := domain.NewSessionStore()