mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
Josh Creek
ce17a45afb
Closes the 'live duplicate/conflict alerting also remains' gap noted
in §8.10: a durable domain.ErrConflict/ErrResultConflict rejection on
/v1/servers/{id}/{register,connect,disconnect,shutdown,result} was
already logged as a structured 'conflict' stage event, but had no
Prometheus signal distinct from the generic 4xx-class counter, which
also catches ordinary client noise (malformed bodies, expired
tokens). A real duplicate registration, raced reconnect, or replayed
result would have been invisible to alerting until someone went
looking through logs.
observability.Metrics gains ObserveServerConflict(kind), a bounded
counter keyed to serverMutation's own five routes (an unrecognized
kind folds into "other", so a caller mistake can't grow the label
set), exported as cosmic_clash_api_server_conflicts_total. Wired at
each of serverMutation's four conflict branches in server/api/service.go.
deploy/observability/prometheus-rules.yaml adds
CosmicClashControlPlaneServerConflicts, mirroring the existing
allocator quota-denial alert shape, firing on >3 conflicts of one
kind in 15 minutes.
Verified: go build/vet/test -race clean across every server package;
new unit tests cover per-kind counting, the bounded 'other' fallback,
the counter's absence until first observed, and a nil-receiver no-op;
a service-level test proves a real register conflict is exported
through the live /metrics endpoint. scripts/verify_observability_manifests.py
passes against the edited rules file.
Remaining, and explicitly out of scope here: this alert has only been
validated statically, never against a live Prometheus/Alertmanager
firing on real traffic — that requires the same live cluster this
sandbox has never had.
91 lines
3.2 KiB
Go
91 lines
3.2 KiB
Go
package observability
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestMetricsNormalizesOperationsAndExportsBoundedLabels(t *testing.T) {
|
|
m := NewMetrics()
|
|
m.ObserveAPI("queue", 201, 10*time.Millisecond)
|
|
m.ObserveAPI("/crafted/path/with-secret", 500, time.Second)
|
|
var output strings.Builder
|
|
if err := m.WritePrometheus(&output); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
text := output.String()
|
|
if !strings.Contains(text, `operation="queue",status="2xx"`) || !strings.Contains(text, `operation="other",status="5xx"`) {
|
|
t.Fatalf("metrics output = %s", text)
|
|
}
|
|
if !strings.Contains(text, "# TYPE cosmic_clash_api_latency_seconds histogram") ||
|
|
!strings.Contains(text, `cosmic_clash_api_latency_seconds_bucket{operation="queue",status="2xx",le="0.25"} 1`) ||
|
|
!strings.Contains(text, `cosmic_clash_api_latency_seconds_bucket{operation="queue",status="2xx",le="+Inf"} 1`) {
|
|
t.Fatalf("latency histogram missing expected buckets: %s", text)
|
|
}
|
|
if strings.Contains(text, "crafted") || strings.Contains(text, "secret") {
|
|
t.Fatalf("unbounded operation label leaked: %s", text)
|
|
}
|
|
}
|
|
|
|
func TestMetricsHistogramUsesCumulativeBoundarySemantics(t *testing.T) {
|
|
m := NewMetrics()
|
|
m.ObserveAPI("queue", 200, 250*time.Millisecond)
|
|
var output strings.Builder
|
|
if err := m.WritePrometheus(&output); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
text := output.String()
|
|
if !strings.Contains(text, `le="0.25"} 1`) || !strings.Contains(text, `le="0.5"} 1`) {
|
|
t.Fatalf("boundary observation was not cumulative: %s", text)
|
|
}
|
|
if strings.Contains(text, `le="0.1"} 1`) {
|
|
t.Fatalf("250ms observation entered an earlier bucket: %s", text)
|
|
}
|
|
}
|
|
|
|
func TestMetricsServerConflictsAreCountedByKindAndBounded(t *testing.T) {
|
|
m := NewMetrics()
|
|
m.ObserveServerConflict("register")
|
|
m.ObserveServerConflict("register")
|
|
m.ObserveServerConflict("result")
|
|
m.ObserveServerConflict("crafted-unknown-kind")
|
|
var output strings.Builder
|
|
if err := m.WritePrometheus(&output); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
text := output.String()
|
|
if !strings.Contains(text, "# TYPE cosmic_clash_api_server_conflicts_total counter") {
|
|
t.Fatalf("missing conflict counter TYPE line: %s", text)
|
|
}
|
|
if !strings.Contains(text, `cosmic_clash_api_server_conflicts_total{kind="register"} 2`) {
|
|
t.Fatalf("register conflicts not counted correctly: %s", text)
|
|
}
|
|
if !strings.Contains(text, `cosmic_clash_api_server_conflicts_total{kind="result"} 1`) {
|
|
t.Fatalf("result conflicts not counted correctly: %s", text)
|
|
}
|
|
if !strings.Contains(text, `cosmic_clash_api_server_conflicts_total{kind="other"} 1`) {
|
|
t.Fatalf("unknown kind was not folded into the bounded 'other' label: %s", text)
|
|
}
|
|
if strings.Contains(text, "crafted-unknown-kind") {
|
|
t.Fatalf("unbounded conflict kind label leaked: %s", text)
|
|
}
|
|
}
|
|
|
|
func TestMetricsServerConflictAbsentWhenUnobserved(t *testing.T) {
|
|
m := NewMetrics()
|
|
m.ObserveAPI("queue", 200, time.Millisecond)
|
|
var output strings.Builder
|
|
if err := m.WritePrometheus(&output); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(output.String(), "cosmic_clash_api_server_conflicts_total") {
|
|
t.Fatalf("conflict counter should be omitted entirely until first observed: %s", output.String())
|
|
}
|
|
}
|
|
|
|
func TestMetricsServerConflictNilReceiverIsANoop(t *testing.T) {
|
|
var m *Metrics
|
|
m.ObserveServerConflict("register") // must not panic
|
|
}
|