mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-13 08:52:03 +00:00
fix(server): fan outbox events out to every control-plane replica
The Deployment runs two replicas, but WebSocket subscribers live only in each process's in-memory hub. Every replica races to read the same global unpublished outbox rows, and publishing succeeded even when the winning replica held no matching local subscriber -- that replica then set the single global published_at. A client connected to the other replica never received the event, and delivery degraded further with each replica added. REST recovery eventually converged, but short-lived proposal transitions could be observed late or not at all. Publish committed events through PostgreSQL LISTEN/NOTIFY so the replica that owns the subscriber's connection delivers it, regardless of which replica drained the row. The listener holds its own pgx connection -- LISTEN is session state, so a pooled database/sql connection cannot carry it -- and reconnects with backoff, since losing it would silently downgrade that replica's subscribers to REST-only recovery. The fan-out is optional: without EventFanout configured, behaviour is unchanged local-hub publication, which stays correct for a single replica and for tests. Only outbox-sourced events are routed through it; the in-request-path publishes remain local, as those are a latency optimisation for the caller's own connection. Fan-out needs a wire shape of its own because ControlPlaneEvent hides PlayerID from clients, and the recipient is exactly what a peer replica needs to route on.
This commit is contained in:
+42
-2
@@ -224,12 +224,52 @@ func (s *Service) getEventHub() *eventHub {
|
||||
}
|
||||
|
||||
// PublishControlPlaneEvent routes an already-authorized event to the matching
|
||||
// authenticated player connection. Durable callers should publish from their
|
||||
// outbox after commit; this in-memory hub is deliberately non-authoritative.
|
||||
// authenticated player connection on THIS replica. Durable callers should
|
||||
// publish from their outbox after commit; this in-memory hub is deliberately
|
||||
// non-authoritative.
|
||||
func (s *Service) PublishControlPlaneEvent(event ControlPlaneEvent) error {
|
||||
return s.getEventHub().publish(event)
|
||||
}
|
||||
|
||||
// fannedOutEvent is the fan-out wire shape. It cannot reuse ControlPlaneEvent
|
||||
// directly because that type hides PlayerID from clients (json:"-"), and the
|
||||
// recipient is precisely what a peer replica needs in order to route.
|
||||
type fannedOutEvent struct {
|
||||
ControlPlaneEvent
|
||||
PlayerID string `json:"player_id"`
|
||||
}
|
||||
|
||||
// EncodeFannedOutEvent and DecodeFannedOutEvent are exported for the
|
||||
// control-plane binary, which owns the transport wiring.
|
||||
func EncodeFannedOutEvent(event ControlPlaneEvent) ([]byte, error) {
|
||||
return json.Marshal(fannedOutEvent{ControlPlaneEvent: event, PlayerID: event.PlayerID})
|
||||
}
|
||||
|
||||
func DecodeFannedOutEvent(payload []byte) (ControlPlaneEvent, error) {
|
||||
var decoded fannedOutEvent
|
||||
if err := json.Unmarshal(payload, &decoded); err != nil {
|
||||
return ControlPlaneEvent{}, err
|
||||
}
|
||||
event := decoded.ControlPlaneEvent
|
||||
event.PlayerID = decoded.PlayerID
|
||||
if event.Event == "" || event.ResourceID == "" || event.PlayerID == "" {
|
||||
return ControlPlaneEvent{}, fmt.Errorf("invalid fanned-out control-plane event")
|
||||
}
|
||||
return event, nil
|
||||
}
|
||||
|
||||
// publishOutboxEvent is how the outbox dispatchers publish. When EventFanout
|
||||
// is configured it hands the event to the shared transport so every replica --
|
||||
// including whichever one holds the subscriber's WebSocket -- can deliver it.
|
||||
// Without it, behaviour is unchanged: local-hub only, correct for a single
|
||||
// replica and for tests.
|
||||
func (s *Service) publishOutboxEvent(event ControlPlaneEvent) error {
|
||||
if s.EventFanout != nil {
|
||||
return s.EventFanout(event)
|
||||
}
|
||||
return s.PublishControlPlaneEvent(event)
|
||||
}
|
||||
|
||||
func (s *Service) publishTicketEvent(ticket domain.QueueTicket, now time.Time) {
|
||||
_ = s.PublishControlPlaneEvent(ControlPlaneEvent{
|
||||
Event: "state_changed", Revision: ticket.Revision, ResourceID: ticket.TicketID,
|
||||
|
||||
Reference in New Issue
Block a user