feat(multiplayer): alert on workload server-mutation conflicts

Closes the 'live duplicate/conflict alerting also remains' gap noted
in §8.10: a durable domain.ErrConflict/ErrResultConflict rejection on
/v1/servers/{id}/{register,connect,disconnect,shutdown,result} was
already logged as a structured 'conflict' stage event, but had no
Prometheus signal distinct from the generic 4xx-class counter, which
also catches ordinary client noise (malformed bodies, expired
tokens). A real duplicate registration, raced reconnect, or replayed
result would have been invisible to alerting until someone went
looking through logs.

observability.Metrics gains ObserveServerConflict(kind), a bounded
counter keyed to serverMutation's own five routes (an unrecognized
kind folds into "other", so a caller mistake can't grow the label
set), exported as cosmic_clash_api_server_conflicts_total. Wired at
each of serverMutation's four conflict branches in server/api/service.go.
deploy/observability/prometheus-rules.yaml adds
CosmicClashControlPlaneServerConflicts, mirroring the existing
allocator quota-denial alert shape, firing on >3 conflicts of one
kind in 15 minutes.

Verified: go build/vet/test -race clean across every server package;
new unit tests cover per-kind counting, the bounded 'other' fallback,
the counter's absence until first observed, and a nil-receiver no-op;
a service-level test proves a real register conflict is exported
through the live /metrics endpoint. scripts/verify_observability_manifests.py
passes against the edited rules file.

Remaining, and explicitly out of scope here: this alert has only been
validated statically, never against a live Prometheus/Alertmanager
firing on real traffic — that requires the same live cluster this
sandbox has never had.
This commit is contained in:
Josh Creek
2026-09-04 17:24:09 +01:00
parent 817572a6ce
commit ce17a45afb
6 changed files with 162 additions and 6 deletions
+4
View File
@@ -652,6 +652,7 @@ func (s *Service) serverMutation(w http.ResponseWriter, r *http.Request) {
stage := "invalid"
if errors.Is(err, domain.ErrConflict) {
stage = "conflict"
s.Metrics.ObserveServerConflict("register")
writeError(w, http.StatusConflict, "conflict")
} else {
writeError(w, http.StatusUnprocessableEntity, "invalid_request")
@@ -702,6 +703,7 @@ func (s *Service) serverMutation(w http.ResponseWriter, r *http.Request) {
}
if err != nil {
if errors.Is(err, domain.ErrConflict) {
s.Metrics.ObserveServerConflict(parts[1])
writeError(w, http.StatusConflict, "conflict")
} else {
// The request has already passed schema and workload checks. An
@@ -747,6 +749,7 @@ func (s *Service) serverMutation(w http.ResponseWriter, r *http.Request) {
stage := "invalid"
if errors.Is(err, domain.ErrConflict) {
stage = "conflict"
s.Metrics.ObserveServerConflict("shutdown")
writeError(w, http.StatusConflict, "conflict")
} else {
writeError(w, http.StatusUnprocessableEntity, "invalid_request")
@@ -777,6 +780,7 @@ func (s *Service) serverMutation(w http.ResponseWriter, r *http.Request) {
stage := "invalid"
if errors.Is(err, domain.ErrResultConflict) || strings.Contains(err.Error(), "conflict") {
stage = "conflict"
s.Metrics.ObserveServerConflict("result")
writeError(w, http.StatusConflict, "conflict")
} else {
writeError(w, http.StatusUnprocessableEntity, "invalid_request")
+37
View File
@@ -1447,6 +1447,43 @@ func TestServerRegistrationAPIRequiresBoundWorkloadAndValidDigest(t *testing.T)
response.Body.Close()
}
func TestServerMutationConflictsAreExportedAsADistinctPrometheusCounter(t *testing.T) {
now := time.Unix(1000, 0).UTC()
binding := domain.WorkloadBinding{AllocationID: "allocation-1", MatchID: "match-1", ServerID: "server-1"}
registrar := &serverRegistrarSpy{err: domain.ErrConflict}
metrics := observability.NewMetrics()
service := &Service{Now: func() time.Time { return now }, Metrics: metrics, WorkloadVerify: func(token string, _ time.Time) (domain.WorkloadBinding, error) {
if token != "workload-token" {
return domain.WorkloadBinding{}, errors.New("bad token")
}
return binding, nil
}, ServerRegistrar: registrar}
server := httptest.NewServer(service.Handler())
defer server.Close()
body := `{"match_id":"match-1","protocol_version":1,"image_digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","assignment_ready":false}`
req, _ := http.NewRequest(http.MethodPost, server.URL+"/v1/servers/server-1/register", strings.NewReader(body))
req.Header.Set("Authorization", "Bearer workload-token")
req.Header.Set("Idempotency-Key", "register-key-123456")
response, err := http.DefaultClient.Do(req)
if err != nil || response.StatusCode != http.StatusConflict {
t.Fatalf("status=%v err=%v", response.StatusCode, err)
}
response.Body.Close()
metricsResponse, err := http.Get(server.URL + "/metrics")
if err != nil {
t.Fatal(err)
}
defer metricsResponse.Body.Close()
exported, err := io.ReadAll(metricsResponse.Body)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(exported), `cosmic_clash_api_server_conflicts_total{kind="register"} 1`) {
t.Fatalf("register conflict was not exported: %s", exported)
}
}
func TestServerShutdownAPIRequiresBoundWorkloadAndDelegatesAcknowledgement(t *testing.T) {
now := time.Unix(1000, 0).UTC()
binding := domain.WorkloadBinding{AllocationID: "allocation-1", MatchID: "match-1", ServerID: "server-1"}