Files
CosmicClash/server/api/outbox.go
T
Josh Creek 1dd05c75f1 fix(server): repair the initial-connect outbox envelope and unblock dispatch
ApplyInitialConnectPlan wrote a payload of {match_id,state,action},
omitting event, revision, resource_id, occurred_at and player_ids --
every field deliverStateOutboxEvent requires. Delivery rejected the row,
dispatch returned on the first error so it was never acknowledged, and
because reads are ordered oldest-first it was retried ahead of every
later state_changed event on every 100ms poll. One initial-connect
transition therefore blocked lifecycle delivery for all matches, not
just its own.

Two independent fixes, since either alone leaves the system fragile:

Build envelopes through one validating helper (MarshalOutboxEnvelope)
and convert all five writers to it. A writer that omits a required
field now fails its own transaction instead of committing a row that
can only ever poison the queue. The helper takes revision as int64 so
the -1 "nothing matched" sentinel some CTEs return surfaces as an error
rather than wrapping to a huge uint64.

Make dispatch resilient regardless: a delivery failure is now counted
against that row and the batch continues, with the row dead-lettered
after MaxOutboxDeliveryAttempts so a poison event degrades to one lost
notification instead of a stalled queue. Ordering within an aggregate
is still honoured -- later events of a failed match are deferred, so no
client observes that match's newer state before its older state. An ack
failure still stops the batch, being a database rather than a payload
problem.

Initial-connect events now address every participant, not just the
connected ones: a no-show needs to learn their ticket was failed and a
penalty applied.
2026-09-05 10:17:16 +01:00

224 lines
7.8 KiB
Go

package api
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"time"
"github.com/cosmic-clash/cosmic-clash/server/store"
)
// RunProposalOutboxDispatcher delivers committed proposal changes to the
// authenticated WebSocket subscribers. It only reads proposal_changed rows;
// result and other outbox event types remain owned by their own consumers.
// The outbox guarantees after-commit publication into this replica's bounded
// transient hub; WebSocket receipt is deliberately best-effort. Clients use
// owner-scoped periodic REST recovery for correctness across disconnects and
// replicas, so a socket notification is only a latency optimization.
func RunProposalOutboxDispatcher(ctx context.Context, db *sql.DB, service *Service) {
if db == nil || service == nil {
return
}
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
dispatcher := store.NewOutboxDispatcher(db, func(deliveryCtx context.Context, event store.OutboxEvent) error {
return deliverProposalOutboxEvent(deliveryCtx, event, service)
})
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
events, err := store.ReadUnpublishedProposalOutbox(ctx, db, 100)
if err != nil {
continue
}
_ = dispatchOutboxEvents(ctx, db, dispatcher, events)
}
}
}
// RunResultOutboxDispatcher delivers committed match results as targeted
// COMPLETED state events. It owns only match_completed rows; proposal rows
// remain with RunProposalOutboxDispatcher.
func RunResultOutboxDispatcher(ctx context.Context, db *sql.DB, service *Service) {
if db == nil || service == nil {
return
}
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
dispatcher := store.NewOutboxDispatcher(db, func(deliveryCtx context.Context, event store.OutboxEvent) error {
return deliverResultOutboxEvent(deliveryCtx, db, event, service)
})
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
events, err := store.ReadUnpublishedResultOutbox(ctx, db, 100)
if err != nil {
continue
}
_ = dispatchOutboxEvents(ctx, db, dispatcher, events)
}
}
}
// RunStateOutboxDispatcher delivers committed allocation/no-show lifecycle
// transitions to each participant without acknowledging proposal or result
// events owned by the other dispatchers.
func RunStateOutboxDispatcher(ctx context.Context, db *sql.DB, service *Service) {
if db == nil || service == nil {
return
}
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
dispatcher := store.NewOutboxDispatcher(db, func(deliveryCtx context.Context, event store.OutboxEvent) error {
return deliverStateOutboxEvent(deliveryCtx, event, service)
})
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
events, err := store.ReadUnpublishedStateOutbox(ctx, db, 100)
if err != nil {
continue
}
_ = dispatchOutboxEvents(ctx, db, dispatcher, events)
}
}
}
func dispatchOutboxEvents(ctx context.Context, db *sql.DB, dispatcher *store.OutboxDispatcher, events []store.OutboxEvent) error {
if len(events) == 0 {
return nil
}
// Use the same delivery-before-ack contract as the general dispatcher,
// while keeping the already-filtered batch from being read a second time.
//
// A delivery failure does not abort the batch. Returning here meant one
// undeliverable payload -- reads are oldest-first -- was retried ahead of
// every later event of its type on every poll, forever. Instead the failure
// is counted against that row (dead-lettering it once exhausted) and the
// batch continues.
//
// Ordering within one aggregate is still honoured: once an event for a
// match fails, its later events are left for a subsequent poll so a client
// can never observe that match's newer state before its older state. Other
// aggregates are independent and proceed.
blocked := make(map[string]struct{})
var firstErr error
for _, event := range events {
if event.EventID == "" {
return fmt.Errorf("outbox event has no ID")
}
if _, skip := blocked[event.AggregateID]; skip {
continue
}
if err := dispatcher.Deliver(ctx, event); err != nil {
blocked[event.AggregateID] = struct{}{}
if firstErr == nil {
firstErr = err
}
if db != nil {
if _, failErr := store.RecordOutboxDeliveryFailure(ctx, db, event.EventID, err, time.Now().UTC()); failErr != nil {
return failErr
}
}
continue
}
if err := dispatcher.Ack(ctx, event.EventID, time.Now().UTC()); err != nil {
// An ack failure is a database problem, not a payload problem;
// stop rather than counting it against the event.
return err
}
}
return firstErr
}
func deliverProposalOutboxEvent(_ context.Context, event store.OutboxEvent, service *Service) error {
var envelope struct {
Event string `json:"event"`
Revision uint64 `json:"revision"`
ResourceID string `json:"resource_id"`
OccurredAt time.Time `json:"occurred_at"`
State string `json:"state"`
PlayerIDs []string `json:"player_ids"`
}
if err := json.Unmarshal(event.Payload, &envelope); err != nil {
return fmt.Errorf("decode proposal outbox event: %w", err)
}
if envelope.Event != "proposal_changed" || envelope.ResourceID == "" || len(envelope.PlayerIDs) == 0 {
return fmt.Errorf("invalid proposal outbox event")
}
for _, playerID := range envelope.PlayerIDs {
if err := service.PublishControlPlaneEvent(ControlPlaneEvent{
Event: envelope.Event, Revision: envelope.Revision, ResourceID: envelope.ResourceID,
OccurredAt: envelope.OccurredAt, State: envelope.State, PlayerID: playerID,
}); err != nil {
return err
}
}
return nil
}
func deliverResultOutboxEvent(ctx context.Context, db *sql.DB, event store.OutboxEvent, service *Service) error {
if event.EventType != "match_completed" || event.AggregateID == "" || event.Revision == 0 || len(event.Payload) == 0 {
return fmt.Errorf("invalid result outbox event")
}
var payload map[string]any
if err := json.Unmarshal(event.Payload, &payload); err != nil || payload == nil {
return fmt.Errorf("decode result outbox event: %w", err)
}
players, err := store.ReadMatchParticipantIDs(ctx, db, event.AggregateID)
if err != nil {
return err
}
if len(players) == 0 {
return fmt.Errorf("result outbox event has no participants")
}
for _, playerID := range players {
if err := service.PublishControlPlaneEvent(ControlPlaneEvent{
Event: "state_changed", Revision: event.Revision, ResourceID: event.AggregateID,
OccurredAt: event.CreatedAt, State: "COMPLETED", MatchID: event.AggregateID,
PlayerID: playerID,
}); err != nil {
return err
}
}
return nil
}
func deliverStateOutboxEvent(_ context.Context, event store.OutboxEvent, service *Service) error {
if event.EventType != "state_changed" || event.AggregateID == "" || event.Revision == 0 || len(event.Payload) == 0 {
return fmt.Errorf("invalid state outbox event")
}
var envelope struct {
Event string `json:"event"`
Revision uint64 `json:"revision"`
ResourceID string `json:"resource_id"`
OccurredAt time.Time `json:"occurred_at"`
State string `json:"state"`
MatchID string `json:"match_id"`
PlayerIDs []string `json:"player_ids"`
}
if err := json.Unmarshal(event.Payload, &envelope); err != nil {
return fmt.Errorf("decode state outbox event: %w", err)
}
if envelope.Event != "state_changed" || envelope.ResourceID != event.AggregateID || envelope.Revision != event.Revision || envelope.State == "" || len(envelope.PlayerIDs) == 0 {
return fmt.Errorf("invalid state outbox payload")
}
for _, playerID := range envelope.PlayerIDs {
if playerID == "" {
return fmt.Errorf("state outbox event has empty participant")
}
if err := service.PublishControlPlaneEvent(ControlPlaneEvent{Event: "state_changed", Revision: envelope.Revision, ResourceID: envelope.ResourceID, OccurredAt: envelope.OccurredAt, State: envelope.State, MatchID: envelope.MatchID, PlayerID: playerID}); err != nil {
return err
}
}
return nil
}