feat(multiplayer): export queryable API latency histograms

server/observability/metrics.go

@@ -11,15 +11,20 @@ import (
 // Metrics is a bounded in-process collector for API request health. Operation
 // names are normalized to a fixed vocabulary before storage.
 type Metrics struct {
-	mu     sync.Mutex
-	counts map[metricKey]uint64
-	sums   map[metricKey]time.Duration
+	mu      sync.Mutex
+	counts  map[metricKey]uint64
+	sums    map[metricKey]time.Duration
+	buckets map[metricKey][]uint64
 }
 
 type metricKey struct{ operation, status string }
 
+// apiLatencyBucketsSeconds is deliberately fixed and small. It is wide enough
+// to query the documented 250 ms API SLO while keeping the exporter bounded.
+var apiLatencyBucketsSeconds = []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10}
+
 func NewMetrics() *Metrics {
-	return &Metrics{counts: make(map[metricKey]uint64), sums: make(map[metricKey]time.Duration)}
+	return &Metrics{counts: make(map[metricKey]uint64), sums: make(map[metricKey]time.Duration), buckets: make(map[metricKey][]uint64)}
 }
 
 func (m *Metrics) ObserveAPI(operation string, statusCode int, duration time.Duration) {
@@ -33,6 +38,17 @@ func (m *Metrics) ObserveAPI(operation string, statusCode int, duration time.Dur
 	m.mu.Lock()
 	m.counts[key]++
 	m.sums[key] += duration
+	bucketCounts := m.buckets[key]
+	if bucketCounts == nil {
+		bucketCounts = make([]uint64, len(apiLatencyBucketsSeconds))
+		m.buckets[key] = bucketCounts
+	}
+	seconds := duration.Seconds()
+	for index, upperBound := range apiLatencyBucketsSeconds {
+		if seconds <= upperBound {
+			bucketCounts[index]++
+		}
+	}
 	m.mu.Unlock()
 }
 
@@ -53,16 +69,23 @@ func (m *Metrics) WritePrometheus(w io.Writer) error {
 	})
 	counts := make(map[metricKey]uint64, len(keys))
 	sums := make(map[metricKey]time.Duration, len(keys))
+	buckets := make(map[metricKey][]uint64, len(keys))
 	for _, key := range keys {
 		counts[key], sums[key] = m.counts[key], m.sums[key]
+		buckets[key] = append([]uint64(nil), m.buckets[key]...)
 	}
 	m.mu.Unlock()
-	if _, err := io.WriteString(w, "# TYPE cosmic_clash_api_requests_total counter\n# TYPE cosmic_clash_api_latency_seconds summary\n"); err != nil {
+	if _, err := io.WriteString(w, "# TYPE cosmic_clash_api_requests_total counter\n# TYPE cosmic_clash_api_latency_seconds histogram\n"); err != nil {
 		return err
 	}
 	for _, key := range keys {
 		labels := fmt.Sprintf(`operation="%s",status="%s"`, key.operation, key.status)
-		if _, err := fmt.Fprintf(w, "cosmic_clash_api_requests_total{%s} %d\ncosmic_clash_api_latency_seconds_count{%s} %d\ncosmic_clash_api_latency_seconds_sum{%s} %.9f\n", labels, counts[key], labels, counts[key], labels, sums[key].Seconds()); err != nil {
+		for index, upperBound := range apiLatencyBucketsSeconds {
+			if _, err := fmt.Fprintf(w, "cosmic_clash_api_latency_seconds_bucket{%s,le=\"%g\"} %d\n", labels, upperBound, buckets[key][index]); err != nil {
+				return err
+			}
+		}
+		if _, err := fmt.Fprintf(w, "cosmic_clash_api_latency_seconds_bucket{%s,le=\"+Inf\"} %d\ncosmic_clash_api_requests_total{%s} %d\ncosmic_clash_api_latency_seconds_count{%s} %d\ncosmic_clash_api_latency_seconds_sum{%s} %.9f\n", labels, counts[key], labels, counts[key], labels, counts[key], labels, sums[key].Seconds()); err != nil {
 			return err
 		}
 	}

server/observability/metrics_test.go

@@ -18,7 +18,28 @@ func TestMetricsNormalizesOperationsAndExportsBoundedLabels(t *testing.T) {
 	if !strings.Contains(text, `operation="queue",status="2xx"`) || !strings.Contains(text, `operation="other",status="5xx"`) {
 		t.Fatalf("metrics output = %s", text)
 	}
+	if !strings.Contains(text, "# TYPE cosmic_clash_api_latency_seconds histogram") ||
+		!strings.Contains(text, `cosmic_clash_api_latency_seconds_bucket{operation="queue",status="2xx",le="0.25"} 1`) ||
+		!strings.Contains(text, `cosmic_clash_api_latency_seconds_bucket{operation="queue",status="2xx",le="+Inf"} 1`) {
+		t.Fatalf("latency histogram missing expected buckets: %s", text)
+	}
 	if strings.Contains(text, "crafted") || strings.Contains(text, "secret") {
 		t.Fatalf("unbounded operation label leaked: %s", text)
 	}
 }
+
+func TestMetricsHistogramUsesCumulativeBoundarySemantics(t *testing.T) {
+	m := NewMetrics()
+	m.ObserveAPI("queue", 200, 250*time.Millisecond)
+	var output strings.Builder
+	if err := m.WritePrometheus(&output); err != nil {
+		t.Fatal(err)
+	}
+	text := output.String()
+	if !strings.Contains(text, `le="0.25"} 1`) || !strings.Contains(text, `le="0.5"} 1`) {
+		t.Fatalf("boundary observation was not cumulative: %s", text)
+	}
+	if strings.Contains(text, `le="0.1"} 1`) {
+		t.Fatalf("250ms observation entered an earlier bucket: %s", text)
+	}
+}