fix(multiplayer): report allocator readiness

server/allocator/health.go

@@ -0,0 +1,65 @@
+package allocator
+
+import (
+	"net/http"
+	"sync"
+	"time"
+)
+
+// Health records only complete successful cycles. Readiness ages out when
+// provider or durable-store work repeatedly fails or stalls, while liveness
+// remains independent so Kubernetes does not restart a healthy process for a
+// dependency outage.
+type Health struct {
+	mu                  sync.RWMutex
+	lastSuccessfulCycle time.Time
+}
+
+func (h *Health) ObserveSuccessfulCycle(at time.Time) {
+	if h == nil || at.IsZero() {
+		return
+	}
+	h.mu.Lock()
+	h.lastSuccessfulCycle = at
+	h.mu.Unlock()
+}
+
+func (h *Health) Ready(now time.Time, maxStale time.Duration) bool {
+	if h == nil || now.IsZero() || maxStale <= 0 {
+		return false
+	}
+	h.mu.RLock()
+	lastSuccess := h.lastSuccessfulCycle
+	h.mu.RUnlock()
+	return !lastSuccess.IsZero() && !now.Before(lastSuccess) && now.Sub(lastSuccess) <= maxStale
+}
+
+// RoleHandler exposes metrics plus distinct process-liveness and dependency-
+// progress readiness endpoints on the allocator's private listener.
+func RoleHandler(metrics *Metrics, health *Health, maxStale time.Duration, now func() time.Time) http.Handler {
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", MetricsHandler(metrics))
+	mux.HandleFunc("/healthz", methodGet(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		_, _ = w.Write([]byte("ok\n"))
+	}))
+	mux.HandleFunc("/readyz", methodGet(func(w http.ResponseWriter, _ *http.Request) {
+		if now == nil || !health.Ready(now(), maxStale) {
+			http.Error(w, "allocator has no recent successful cycle", http.StatusServiceUnavailable)
+			return
+		}
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		_, _ = w.Write([]byte("ready\n"))
+	}))
+	return mux
+}
+
+func methodGet(next http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodGet {
+			w.WriteHeader(http.StatusMethodNotAllowed)
+			return
+		}
+		next(w, r)
+	}
+}

server/allocator/health_test.go

@@ -0,0 +1,53 @@
+package allocator
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestHealthRequiresRecentSuccessfulCycle(t *testing.T) {
+	health := &Health{}
+	now := time.Unix(1000, 0)
+	if health.Ready(now, 30*time.Second) {
+		t.Fatal("allocator was ready before a successful cycle")
+	}
+	health.ObserveSuccessfulCycle(now)
+	if !health.Ready(now.Add(30*time.Second), 30*time.Second) {
+		t.Fatal("allocator was not ready at the staleness boundary")
+	}
+	if health.Ready(now.Add(30*time.Second+time.Nanosecond), 30*time.Second) {
+		t.Fatal("stale allocator remained ready")
+	}
+	if health.Ready(now.Add(-time.Second), 30*time.Second) {
+		t.Fatal("clock reversal was accepted as ready")
+	}
+}
+
+func TestRoleHandlerSeparatesLivenessReadinessAndMetrics(t *testing.T) {
+	health := &Health{}
+	now := time.Unix(1000, 0)
+	handler := RoleHandler(NewMetrics(), health, 30*time.Second, func() time.Time { return now })
+	status := func(method, path string) int {
+		recorder := httptest.NewRecorder()
+		handler.ServeHTTP(recorder, httptest.NewRequest(method, path, nil))
+		return recorder.Code
+	}
+	if got := status(http.MethodGet, "/healthz"); got != http.StatusOK {
+		t.Fatalf("liveness status = %d", got)
+	}
+	if got := status(http.MethodGet, "/readyz"); got != http.StatusServiceUnavailable {
+		t.Fatalf("startup readiness status = %d", got)
+	}
+	health.ObserveSuccessfulCycle(now)
+	if got := status(http.MethodGet, "/readyz"); got != http.StatusOK {
+		t.Fatalf("successful-cycle readiness status = %d", got)
+	}
+	if got := status(http.MethodGet, "/metrics"); got != http.StatusOK {
+		t.Fatalf("metrics status = %d", got)
+	}
+	if got := status(http.MethodPost, "/readyz"); got != http.StatusMethodNotAllowed {
+		t.Fatalf("readiness mutation status = %d", got)
+	}
+}