fix(server): fan outbox events out to every control-plane replica

The Deployment runs two replicas, but WebSocket subscribers live only in
each process's in-memory hub. Every replica races to read the same
global unpublished outbox rows, and publishing succeeded even when the
winning replica held no matching local subscriber -- that replica then
set the single global published_at. A client connected to the other
replica never received the event, and delivery degraded further with
each replica added. REST recovery eventually converged, but short-lived
proposal transitions could be observed late or not at all.

Publish committed events through PostgreSQL LISTEN/NOTIFY so the replica
that owns the subscriber's connection delivers it, regardless of which
replica drained the row. The listener holds its own pgx connection --
LISTEN is session state, so a pooled database/sql connection cannot
carry it -- and reconnects with backoff, since losing it would silently
downgrade that replica's subscribers to REST-only recovery.

The fan-out is optional: without EventFanout configured, behaviour is
unchanged local-hub publication, which stays correct for a single
replica and for tests. Only outbox-sourced events are routed through it;
the in-request-path publishes remain local, as those are a latency
optimisation for the caller's own connection.

Fan-out needs a wire shape of its own because ControlPlaneEvent hides
PlayerID from clients, and the recipient is exactly what a peer replica
needs to route on.
This commit is contained in:
Josh Creek
2026-09-05 10:29:23 +01:00
parent 4248e51c60
commit 129b0c7ef0
6 changed files with 233 additions and 5 deletions
+68
View File
@@ -2006,3 +2006,71 @@ func TestPostgreSQLBanWithoutRevocationStillBlocksExistingSessions(t *testing.T)
t.Fatal("a ban applied without revocation left the existing session usable")
}
}
// The WebSocket hub is per-process, but any control-plane replica may drain a
// given outbox row, and the winner sets the single global published_at even
// with no matching local subscriber. Fan-out through LISTEN/NOTIFY is what
// lets the replica that actually owns the connection deliver the event.
func TestPostgreSQLControlPlaneEventFanoutReachesEveryReplica(t *testing.T) {
db := openIntegrationPostgres(t)
applyIntegrationMigrations(t, db)
dsn := os.Getenv("COSMIC_CLASH_POSTGRES_DSN")
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
// Two listeners stand in for two replicas; neither is the one that will
// perform the notify.
type replica struct {
name string
received chan []byte
}
replicas := []*replica{
{name: "replica-a", received: make(chan []byte, 4)},
{name: "replica-b", received: make(chan []byte, 4)},
}
for _, r := range replicas {
target := r
go ListenControlPlaneEvents(ctx, dsn, func(payload []byte) {
select {
case target.received <- payload:
default:
}
}, nil)
}
// LISTEN is asynchronous; retry the notify until both listeners are
// attached rather than sleeping an arbitrary amount.
payload := []byte(`{"event":"state_changed","resource_id":"match-fanout","player_id":"player-fanout"}`)
deadline := time.Now().Add(15 * time.Second)
pending := map[string]bool{"replica-a": true, "replica-b": true}
for len(pending) > 0 && time.Now().Before(deadline) {
if err := NotifyControlPlaneEvent(ctx, db, payload); err != nil {
t.Fatalf("notify: %v", err)
}
for _, r := range replicas {
select {
case got := <-r.received:
if string(got) != string(payload) {
t.Fatalf("%s received %q, want %q", r.name, got, payload)
}
delete(pending, r.name)
case <-time.After(200 * time.Millisecond):
}
}
}
if len(pending) > 0 {
t.Fatalf("replicas never received the fanned-out event: %v", pending)
}
}
func TestNotifyControlPlaneEventRejectsOversizedPayloads(t *testing.T) {
db := openIntegrationPostgres(t)
oversized := make([]byte, MaxNotifyPayloadBytes+1)
for i := range oversized {
oversized[i] = 'x'
}
if err := NotifyControlPlaneEvent(context.Background(), db, oversized); err == nil {
t.Fatal("payload over the NOTIFY limit was accepted")
}
}