mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-11 08:23:45 +00:00
fix(server): fan outbox events out to every control-plane replica
The Deployment runs two replicas, but WebSocket subscribers live only in each process's in-memory hub. Every replica races to read the same global unpublished outbox rows, and publishing succeeded even when the winning replica held no matching local subscriber -- that replica then set the single global published_at. A client connected to the other replica never received the event, and delivery degraded further with each replica added. REST recovery eventually converged, but short-lived proposal transitions could be observed late or not at all. Publish committed events through PostgreSQL LISTEN/NOTIFY so the replica that owns the subscriber's connection delivers it, regardless of which replica drained the row. The listener holds its own pgx connection -- LISTEN is session state, so a pooled database/sql connection cannot carry it -- and reconnects with backoff, since losing it would silently downgrade that replica's subscribers to REST-only recovery. The fan-out is optional: without EventFanout configured, behaviour is unchanged local-hub publication, which stays correct for a single replica and for tests. Only outbox-sourced events are routed through it; the in-request-path publishes remain local, as those are a latency optimisation for the caller's own connection. Fan-out needs a wire shape of its own because ControlPlaneEvent hides PlayerID from clients, and the recipient is exactly what a peer replica needs to route on.
This commit is contained in:
@@ -105,6 +105,28 @@ func main() {
|
||||
}
|
||||
}
|
||||
}()
|
||||
// Fan committed outbox events out to every replica. Subscribers live in
|
||||
// each process's in-memory hub, but any replica may drain a given outbox
|
||||
// row, so without this a client connected elsewhere never sees the event
|
||||
// and delivery degrades as replicas are added.
|
||||
service.EventFanout = func(event api.ControlPlaneEvent) error {
|
||||
payload, err := api.EncodeFannedOutEvent(event)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return store.NotifyControlPlaneEvent(ctx, db, payload)
|
||||
}
|
||||
go store.ListenControlPlaneEvents(ctx, *dsn, func(payload []byte) {
|
||||
event, err := api.DecodeFannedOutEvent(payload)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// Publishing to a player with no local subscriber is a no-op, so every
|
||||
// replica can handle every notification.
|
||||
_ = service.PublishControlPlaneEvent(event)
|
||||
}, func(err error) {
|
||||
fmt.Fprintf(os.Stderr, "control-plane: event fan-out listener: %v\n", err)
|
||||
})
|
||||
go api.RunProposalOutboxDispatcher(ctx, db, service)
|
||||
go api.RunResultOutboxDispatcher(ctx, db, service)
|
||||
go api.RunStateOutboxDispatcher(ctx, db, service)
|
||||
|
||||
Reference in New Issue
Block a user