fix(server): fan outbox events out to every control-plane replica

The Deployment runs two replicas, but WebSocket subscribers live only in
each process's in-memory hub. Every replica races to read the same
global unpublished outbox rows, and publishing succeeded even when the
winning replica held no matching local subscriber -- that replica then
set the single global published_at. A client connected to the other
replica never received the event, and delivery degraded further with
each replica added. REST recovery eventually converged, but short-lived
proposal transitions could be observed late or not at all.

Publish committed events through PostgreSQL LISTEN/NOTIFY so the replica
that owns the subscriber's connection delivers it, regardless of which
replica drained the row. The listener holds its own pgx connection --
LISTEN is session state, so a pooled database/sql connection cannot
carry it -- and reconnects with backoff, since losing it would silently
downgrade that replica's subscribers to REST-only recovery.

The fan-out is optional: without EventFanout configured, behaviour is
unchanged local-hub publication, which stays correct for a single
replica and for tests. Only outbox-sourced events are routed through it;
the in-request-path publishes remain local, as those are a latency
optimisation for the caller's own connection.

Fan-out needs a wire shape of its own because ControlPlaneEvent hides
PlayerID from clients, and the recipient is exactly what a peer replica
needs to route on.

server/api/events.go

@@ -224,12 +224,52 @@ func (s *Service) getEventHub() *eventHub {
 }
 
 // PublishControlPlaneEvent routes an already-authorized event to the matching
-// authenticated player connection. Durable callers should publish from their
-// outbox after commit; this in-memory hub is deliberately non-authoritative.
+// authenticated player connection on THIS replica. Durable callers should
+// publish from their outbox after commit; this in-memory hub is deliberately
+// non-authoritative.
 func (s *Service) PublishControlPlaneEvent(event ControlPlaneEvent) error {
 	return s.getEventHub().publish(event)
 }
 
+// fannedOutEvent is the fan-out wire shape. It cannot reuse ControlPlaneEvent
+// directly because that type hides PlayerID from clients (json:"-"), and the
+// recipient is precisely what a peer replica needs in order to route.
+type fannedOutEvent struct {
+	ControlPlaneEvent
+	PlayerID string `json:"player_id"`
+}
+
+// EncodeFannedOutEvent and DecodeFannedOutEvent are exported for the
+// control-plane binary, which owns the transport wiring.
+func EncodeFannedOutEvent(event ControlPlaneEvent) ([]byte, error) {
+	return json.Marshal(fannedOutEvent{ControlPlaneEvent: event, PlayerID: event.PlayerID})
+}
+
+func DecodeFannedOutEvent(payload []byte) (ControlPlaneEvent, error) {
+	var decoded fannedOutEvent
+	if err := json.Unmarshal(payload, &decoded); err != nil {
+		return ControlPlaneEvent{}, err
+	}
+	event := decoded.ControlPlaneEvent
+	event.PlayerID = decoded.PlayerID
+	if event.Event == "" || event.ResourceID == "" || event.PlayerID == "" {
+		return ControlPlaneEvent{}, fmt.Errorf("invalid fanned-out control-plane event")
+	}
+	return event, nil
+}
+
+// publishOutboxEvent is how the outbox dispatchers publish. When EventFanout
+// is configured it hands the event to the shared transport so every replica --
+// including whichever one holds the subscriber's WebSocket -- can deliver it.
+// Without it, behaviour is unchanged: local-hub only, correct for a single
+// replica and for tests.
+func (s *Service) publishOutboxEvent(event ControlPlaneEvent) error {
+	if s.EventFanout != nil {
+		return s.EventFanout(event)
+	}
+	return s.PublishControlPlaneEvent(event)
+}
+
 func (s *Service) publishTicketEvent(ticket domain.QueueTicket, now time.Time) {
 	_ = s.PublishControlPlaneEvent(ControlPlaneEvent{
 		Event: "state_changed", Revision: ticket.Revision, ResourceID: ticket.TicketID,

server/api/outbox.go

@@ -155,7 +155,7 @@ func deliverProposalOutboxEvent(_ context.Context, event store.OutboxEvent, serv
 		return fmt.Errorf("invalid proposal outbox event")
 	}
 	for _, playerID := range envelope.PlayerIDs {
-		if err := service.PublishControlPlaneEvent(ControlPlaneEvent{
+		if err := service.publishOutboxEvent(ControlPlaneEvent{
 			Event: envelope.Event, Revision: envelope.Revision, ResourceID: envelope.ResourceID,
 			OccurredAt: envelope.OccurredAt, State: envelope.State, PlayerID: playerID,
 		}); err != nil {
@@ -181,7 +181,7 @@ func deliverResultOutboxEvent(ctx context.Context, db *sql.DB, event store.Outbo
 		return fmt.Errorf("result outbox event has no participants")
 	}
 	for _, playerID := range players {
-		if err := service.PublishControlPlaneEvent(ControlPlaneEvent{
+		if err := service.publishOutboxEvent(ControlPlaneEvent{
 			Event: "state_changed", Revision: event.Revision, ResourceID: event.AggregateID,
 			OccurredAt: event.CreatedAt, State: "COMPLETED", MatchID: event.AggregateID,
 			PlayerID: playerID,
@@ -215,7 +215,7 @@ func deliverStateOutboxEvent(_ context.Context, event store.OutboxEvent, service
 		if playerID == "" {
 			return fmt.Errorf("state outbox event has empty participant")
 		}
-		if err := service.PublishControlPlaneEvent(ControlPlaneEvent{Event: "state_changed", Revision: envelope.Revision, ResourceID: envelope.ResourceID, OccurredAt: envelope.OccurredAt, State: envelope.State, MatchID: envelope.MatchID, PlayerID: playerID}); err != nil {
+		if err := service.publishOutboxEvent(ControlPlaneEvent{Event: "state_changed", Revision: envelope.Revision, ResourceID: envelope.ResourceID, OccurredAt: envelope.OccurredAt, State: envelope.State, MatchID: envelope.MatchID, PlayerID: playerID}); err != nil {
 			return err
 		}
 	}

server/api/service.go

@@ -118,6 +118,11 @@ type Service struct {
 	CandidateV2           CandidateProviderV2
 	QueueBackend          QueueBackend
 	CandidateIndex        CandidateIndex
+	// EventFanout, when set, publishes outbox-sourced events through a shared
+	// transport instead of only this replica's in-memory hub. Without it a
+	// client connected to a replica other than the one that drained the outbox
+	// row never receives the event.
+	EventFanout func(ControlPlaneEvent) error
 	Probe                 ProbeProvider
 	ProbeRecorder         ProbeRecorder
 	WorkloadVerify        WorkloadVerifier