feat(multiplayer): alert on workload server-mutation conflicts

Closes the 'live duplicate/conflict alerting also remains' gap noted
in §8.10: a durable domain.ErrConflict/ErrResultConflict rejection on
/v1/servers/{id}/{register,connect,disconnect,shutdown,result} was
already logged as a structured 'conflict' stage event, but had no
Prometheus signal distinct from the generic 4xx-class counter, which
also catches ordinary client noise (malformed bodies, expired
tokens). A real duplicate registration, raced reconnect, or replayed
result would have been invisible to alerting until someone went
looking through logs.

observability.Metrics gains ObserveServerConflict(kind), a bounded
counter keyed to serverMutation's own five routes (an unrecognized
kind folds into "other", so a caller mistake can't grow the label
set), exported as cosmic_clash_api_server_conflicts_total. Wired at
each of serverMutation's four conflict branches in server/api/service.go.
deploy/observability/prometheus-rules.yaml adds
CosmicClashControlPlaneServerConflicts, mirroring the existing
allocator quota-denial alert shape, firing on >3 conflicts of one
kind in 15 minutes.

Verified: go build/vet/test -race clean across every server package;
new unit tests cover per-kind counting, the bounded 'other' fallback,
the counter's absence until first observed, and a nil-receiver no-op;
a service-level test proves a real register conflict is exported
through the live /metrics endpoint. scripts/verify_observability_manifests.py
passes against the edited rules file.

Remaining, and explicitly out of scope here: this alert has only been
validated statically, never against a live Prometheus/Alertmanager
firing on real traffic — that requires the same live cluster this
sandbox has never had.

server/api/service.go

@@ -652,6 +652,7 @@ func (s *Service) serverMutation(w http.ResponseWriter, r *http.Request) {
 			stage := "invalid"
 			if errors.Is(err, domain.ErrConflict) {
 				stage = "conflict"
+				s.Metrics.ObserveServerConflict("register")
 				writeError(w, http.StatusConflict, "conflict")
 			} else {
 				writeError(w, http.StatusUnprocessableEntity, "invalid_request")
@@ -702,6 +703,7 @@ func (s *Service) serverMutation(w http.ResponseWriter, r *http.Request) {
 		}
 		if err != nil {
 			if errors.Is(err, domain.ErrConflict) {
+				s.Metrics.ObserveServerConflict(parts[1])
 				writeError(w, http.StatusConflict, "conflict")
 			} else {
 				// The request has already passed schema and workload checks. An
@@ -747,6 +749,7 @@ func (s *Service) serverMutation(w http.ResponseWriter, r *http.Request) {
 			stage := "invalid"
 			if errors.Is(err, domain.ErrConflict) {
 				stage = "conflict"
+				s.Metrics.ObserveServerConflict("shutdown")
 				writeError(w, http.StatusConflict, "conflict")
 			} else {
 				writeError(w, http.StatusUnprocessableEntity, "invalid_request")
@@ -777,6 +780,7 @@ func (s *Service) serverMutation(w http.ResponseWriter, r *http.Request) {
 		stage := "invalid"
 		if errors.Is(err, domain.ErrResultConflict) || strings.Contains(err.Error(), "conflict") {
 			stage = "conflict"
+			s.Metrics.ObserveServerConflict("result")
 			writeError(w, http.StatusConflict, "conflict")
 		} else {
 			writeError(w, http.StatusUnprocessableEntity, "invalid_request")

server/api/service_test.go

@@ -1447,6 +1447,43 @@ func TestServerRegistrationAPIRequiresBoundWorkloadAndValidDigest(t *testing.T)
 	response.Body.Close()
 }
 
+func TestServerMutationConflictsAreExportedAsADistinctPrometheusCounter(t *testing.T) {
+	now := time.Unix(1000, 0).UTC()
+	binding := domain.WorkloadBinding{AllocationID: "allocation-1", MatchID: "match-1", ServerID: "server-1"}
+	registrar := &serverRegistrarSpy{err: domain.ErrConflict}
+	metrics := observability.NewMetrics()
+	service := &Service{Now: func() time.Time { return now }, Metrics: metrics, WorkloadVerify: func(token string, _ time.Time) (domain.WorkloadBinding, error) {
+		if token != "workload-token" {
+			return domain.WorkloadBinding{}, errors.New("bad token")
+		}
+		return binding, nil
+	}, ServerRegistrar: registrar}
+	server := httptest.NewServer(service.Handler())
+	defer server.Close()
+	body := `{"match_id":"match-1","protocol_version":1,"image_digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","assignment_ready":false}`
+	req, _ := http.NewRequest(http.MethodPost, server.URL+"/v1/servers/server-1/register", strings.NewReader(body))
+	req.Header.Set("Authorization", "Bearer workload-token")
+	req.Header.Set("Idempotency-Key", "register-key-123456")
+	response, err := http.DefaultClient.Do(req)
+	if err != nil || response.StatusCode != http.StatusConflict {
+		t.Fatalf("status=%v err=%v", response.StatusCode, err)
+	}
+	response.Body.Close()
+
+	metricsResponse, err := http.Get(server.URL + "/metrics")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer metricsResponse.Body.Close()
+	exported, err := io.ReadAll(metricsResponse.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(exported), `cosmic_clash_api_server_conflicts_total{kind="register"} 1`) {
+		t.Fatalf("register conflict was not exported: %s", exported)
+	}
+}
+
 func TestServerShutdownAPIRequiresBoundWorkloadAndDelegatesAcknowledgement(t *testing.T) {
 	now := time.Unix(1000, 0).UTC()
 	binding := domain.WorkloadBinding{AllocationID: "allocation-1", MatchID: "match-1", ServerID: "server-1"}

server/observability/metrics.go

@@ -11,20 +11,50 @@ import (
 // Metrics is a bounded in-process collector for API request health. Operation
 // names are normalized to a fixed vocabulary before storage.
 type Metrics struct {
-	mu      sync.Mutex
-	counts  map[metricKey]uint64
-	sums    map[metricKey]time.Duration
-	buckets map[metricKey][]uint64
+	mu        sync.Mutex
+	counts    map[metricKey]uint64
+	sums      map[metricKey]time.Duration
+	buckets   map[metricKey][]uint64
+	conflicts map[string]uint64
 }
 
 type metricKey struct{ operation, status string }
 
+// serverConflictKinds is the fixed, bounded label vocabulary for
+// ObserveServerConflict, matching the workload-authenticated server mutation
+// routes in api.Service.serverMutation. An unrecognized kind is folded into
+// "other" so a caller mistake can never grow the label set.
+var serverConflictKinds = []string{"register", "connect", "disconnect", "shutdown", "result"}
+
 // apiLatencyBucketsSeconds is deliberately fixed and small. It is wide enough
 // to query the documented 250 ms API SLO while keeping the exporter bounded.
 var apiLatencyBucketsSeconds = []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10}
 
 func NewMetrics() *Metrics {
-	return &Metrics{counts: make(map[metricKey]uint64), sums: make(map[metricKey]time.Duration), buckets: make(map[metricKey][]uint64)}
+	return &Metrics{counts: make(map[metricKey]uint64), sums: make(map[metricKey]time.Duration), buckets: make(map[metricKey][]uint64), conflicts: make(map[string]uint64)}
+}
+
+// ObserveServerConflict records one workload-authenticated server mutation
+// (register/connect/disconnect/shutdown/result) that a durable domain.ErrConflict
+// or domain.ErrResultConflict rejected. This is a distinct counter from
+// ObserveAPI's generic 4xx class specifically so a spike here — duplicate
+// registration, a raced reconnect, a replayed result — can be alerted on
+// without also firing on ordinary client-side 4xx noise (malformed bodies,
+// expired tokens) that shares the same status class.
+func (m *Metrics) ObserveServerConflict(kind string) {
+	if m == nil {
+		return
+	}
+	normalized := "other"
+	for _, allowed := range serverConflictKinds {
+		if kind == allowed {
+			normalized = allowed
+			break
+		}
+	}
+	m.mu.Lock()
+	m.conflicts[normalized]++
+	m.mu.Unlock()
 }
 
 func (m *Metrics) ObserveAPI(operation string, statusCode int, duration time.Duration) {
@@ -74,6 +104,15 @@ func (m *Metrics) WritePrometheus(w io.Writer) error {
 		counts[key], sums[key] = m.counts[key], m.sums[key]
 		buckets[key] = append([]uint64(nil), m.buckets[key]...)
 	}
+	conflictKinds := make([]string, 0, len(m.conflicts))
+	for kind := range m.conflicts {
+		conflictKinds = append(conflictKinds, kind)
+	}
+	sort.Strings(conflictKinds)
+	conflicts := make(map[string]uint64, len(conflictKinds))
+	for _, kind := range conflictKinds {
+		conflicts[kind] = m.conflicts[kind]
+	}
 	m.mu.Unlock()
 	if _, err := io.WriteString(w, "# TYPE cosmic_clash_api_requests_total counter\n# TYPE cosmic_clash_api_latency_seconds histogram\n"); err != nil {
 		return err
@@ -89,6 +128,16 @@ func (m *Metrics) WritePrometheus(w io.Writer) error {
 			return err
 		}
 	}
+	if len(conflictKinds) > 0 {
+		if _, err := io.WriteString(w, "# TYPE cosmic_clash_api_server_conflicts_total counter\n"); err != nil {
+			return err
+		}
+		for _, kind := range conflictKinds {
+			if _, err := fmt.Fprintf(w, "cosmic_clash_api_server_conflicts_total{kind=\"%s\"} %d\n", kind, conflicts[kind]); err != nil {
+				return err
+			}
+		}
+	}
 	return nil
 }
 

server/observability/metrics_test.go

@@ -43,3 +43,48 @@ func TestMetricsHistogramUsesCumulativeBoundarySemantics(t *testing.T) {
 		t.Fatalf("250ms observation entered an earlier bucket: %s", text)
 	}
 }
+
+func TestMetricsServerConflictsAreCountedByKindAndBounded(t *testing.T) {
+	m := NewMetrics()
+	m.ObserveServerConflict("register")
+	m.ObserveServerConflict("register")
+	m.ObserveServerConflict("result")
+	m.ObserveServerConflict("crafted-unknown-kind")
+	var output strings.Builder
+	if err := m.WritePrometheus(&output); err != nil {
+		t.Fatal(err)
+	}
+	text := output.String()
+	if !strings.Contains(text, "# TYPE cosmic_clash_api_server_conflicts_total counter") {
+		t.Fatalf("missing conflict counter TYPE line: %s", text)
+	}
+	if !strings.Contains(text, `cosmic_clash_api_server_conflicts_total{kind="register"} 2`) {
+		t.Fatalf("register conflicts not counted correctly: %s", text)
+	}
+	if !strings.Contains(text, `cosmic_clash_api_server_conflicts_total{kind="result"} 1`) {
+		t.Fatalf("result conflicts not counted correctly: %s", text)
+	}
+	if !strings.Contains(text, `cosmic_clash_api_server_conflicts_total{kind="other"} 1`) {
+		t.Fatalf("unknown kind was not folded into the bounded 'other' label: %s", text)
+	}
+	if strings.Contains(text, "crafted-unknown-kind") {
+		t.Fatalf("unbounded conflict kind label leaked: %s", text)
+	}
+}
+
+func TestMetricsServerConflictAbsentWhenUnobserved(t *testing.T) {
+	m := NewMetrics()
+	m.ObserveAPI("queue", 200, time.Millisecond)
+	var output strings.Builder
+	if err := m.WritePrometheus(&output); err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(output.String(), "cosmic_clash_api_server_conflicts_total") {
+		t.Fatalf("conflict counter should be omitted entirely until first observed: %s", output.String())
+	}
+}
+
+func TestMetricsServerConflictNilReceiverIsANoop(t *testing.T) {
+	var m *Metrics
+	m.ObserveServerConflict("register") // must not panic
+}