mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-11 00:14:00 +00:00
fix(server): fan outbox events out to every control-plane replica
The Deployment runs two replicas, but WebSocket subscribers live only in each process's in-memory hub. Every replica races to read the same global unpublished outbox rows, and publishing succeeded even when the winning replica held no matching local subscriber -- that replica then set the single global published_at. A client connected to the other replica never received the event, and delivery degraded further with each replica added. REST recovery eventually converged, but short-lived proposal transitions could be observed late or not at all. Publish committed events through PostgreSQL LISTEN/NOTIFY so the replica that owns the subscriber's connection delivers it, regardless of which replica drained the row. The listener holds its own pgx connection -- LISTEN is session state, so a pooled database/sql connection cannot carry it -- and reconnects with backoff, since losing it would silently downgrade that replica's subscribers to REST-only recovery. The fan-out is optional: without EventFanout configured, behaviour is unchanged local-hub publication, which stays correct for a single replica and for tests. Only outbox-sourced events are routed through it; the in-request-path publishes remain local, as those are a latency optimisation for the caller's own connection. Fan-out needs a wire shape of its own because ControlPlaneEvent hides PlayerID from clients, and the recipient is exactly what a peer replica needs to route on.
This commit is contained in:
@@ -2006,3 +2006,71 @@ func TestPostgreSQLBanWithoutRevocationStillBlocksExistingSessions(t *testing.T)
|
||||
t.Fatal("a ban applied without revocation left the existing session usable")
|
||||
}
|
||||
}
|
||||
|
||||
// The WebSocket hub is per-process, but any control-plane replica may drain a
|
||||
// given outbox row, and the winner sets the single global published_at even
|
||||
// with no matching local subscriber. Fan-out through LISTEN/NOTIFY is what
|
||||
// lets the replica that actually owns the connection deliver the event.
|
||||
func TestPostgreSQLControlPlaneEventFanoutReachesEveryReplica(t *testing.T) {
|
||||
db := openIntegrationPostgres(t)
|
||||
applyIntegrationMigrations(t, db)
|
||||
|
||||
dsn := os.Getenv("COSMIC_CLASH_POSTGRES_DSN")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Two listeners stand in for two replicas; neither is the one that will
|
||||
// perform the notify.
|
||||
type replica struct {
|
||||
name string
|
||||
received chan []byte
|
||||
}
|
||||
replicas := []*replica{
|
||||
{name: "replica-a", received: make(chan []byte, 4)},
|
||||
{name: "replica-b", received: make(chan []byte, 4)},
|
||||
}
|
||||
for _, r := range replicas {
|
||||
target := r
|
||||
go ListenControlPlaneEvents(ctx, dsn, func(payload []byte) {
|
||||
select {
|
||||
case target.received <- payload:
|
||||
default:
|
||||
}
|
||||
}, nil)
|
||||
}
|
||||
|
||||
// LISTEN is asynchronous; retry the notify until both listeners are
|
||||
// attached rather than sleeping an arbitrary amount.
|
||||
payload := []byte(`{"event":"state_changed","resource_id":"match-fanout","player_id":"player-fanout"}`)
|
||||
deadline := time.Now().Add(15 * time.Second)
|
||||
pending := map[string]bool{"replica-a": true, "replica-b": true}
|
||||
for len(pending) > 0 && time.Now().Before(deadline) {
|
||||
if err := NotifyControlPlaneEvent(ctx, db, payload); err != nil {
|
||||
t.Fatalf("notify: %v", err)
|
||||
}
|
||||
for _, r := range replicas {
|
||||
select {
|
||||
case got := <-r.received:
|
||||
if string(got) != string(payload) {
|
||||
t.Fatalf("%s received %q, want %q", r.name, got, payload)
|
||||
}
|
||||
delete(pending, r.name)
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(pending) > 0 {
|
||||
t.Fatalf("replicas never received the fanned-out event: %v", pending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotifyControlPlaneEventRejectsOversizedPayloads(t *testing.T) {
|
||||
db := openIntegrationPostgres(t)
|
||||
oversized := make([]byte, MaxNotifyPayloadBytes+1)
|
||||
for i := range oversized {
|
||||
oversized[i] = 'x'
|
||||
}
|
||||
if err := NotifyControlPlaneEvent(context.Background(), db, oversized); err == nil {
|
||||
t.Fatal("payload over the NOTIFY limit was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user