fix(server): repair the initial-connect outbox envelope and unblock dispatch

ApplyInitialConnectPlan wrote a payload of {match_id,state,action},
omitting event, revision, resource_id, occurred_at and player_ids --
every field deliverStateOutboxEvent requires. Delivery rejected the row,
dispatch returned on the first error so it was never acknowledged, and
because reads are ordered oldest-first it was retried ahead of every
later state_changed event on every 100ms poll. One initial-connect
transition therefore blocked lifecycle delivery for all matches, not
just its own.

Two independent fixes, since either alone leaves the system fragile:

Build envelopes through one validating helper (MarshalOutboxEnvelope)
and convert all five writers to it. A writer that omits a required
field now fails its own transaction instead of committing a row that
can only ever poison the queue. The helper takes revision as int64 so
the -1 "nothing matched" sentinel some CTEs return surfaces as an error
rather than wrapping to a huge uint64.

Make dispatch resilient regardless: a delivery failure is now counted
against that row and the batch continues, with the row dead-lettered
after MaxOutboxDeliveryAttempts so a poison event degrades to one lost
notification instead of a stalled queue. Ordering within an aggregate
is still honoured -- later events of a failed match are deferred, so no
client observes that match's newer state before its older state. An ack
failure still stops the batch, being a database rather than a payload
problem.

Initial-connect events now address every participant, not just the
connected ones: a no-show needs to learn their ticket was failed and a
penalty applied.
This commit is contained in:
Josh Creek
2026-09-05 10:14:43 +01:00
parent 2c648514ba
commit 1dd05c75f1
12 changed files with 379 additions and 31 deletions
+61 -4
View File
@@ -25,28 +25,28 @@ type OutboxEvent struct {
const OutboxUnpublishedSelectSQL = `SELECT event_id, aggregate_type, aggregate_id, revision,
event_type, payload, created_at, published_at
FROM outbox
WHERE published_at IS NULL
WHERE published_at IS NULL AND dead_lettered_at IS NULL
ORDER BY created_at, event_id
LIMIT $1`
const OutboxUnpublishedProposalSelectSQL = `SELECT event_id, aggregate_type, aggregate_id, revision,
event_type, payload, created_at, published_at
FROM outbox
WHERE published_at IS NULL AND event_type = 'proposal_changed'
WHERE published_at IS NULL AND dead_lettered_at IS NULL AND event_type = 'proposal_changed'
ORDER BY created_at, event_id
LIMIT $1`
const OutboxUnpublishedResultSelectSQL = `SELECT event_id, aggregate_type, aggregate_id, revision,
event_type, payload, created_at, published_at
FROM outbox
WHERE published_at IS NULL AND event_type = 'match_completed'
WHERE published_at IS NULL AND dead_lettered_at IS NULL AND event_type = 'match_completed'
ORDER BY created_at, event_id
LIMIT $1`
const OutboxUnpublishedStateSelectSQL = `SELECT event_id, aggregate_type, aggregate_id, revision,
event_type, payload, created_at, published_at
FROM outbox
WHERE published_at IS NULL AND event_type = 'state_changed'
WHERE published_at IS NULL AND dead_lettered_at IS NULL AND event_type = 'state_changed'
ORDER BY created_at, event_id
LIMIT $1`
@@ -59,8 +59,65 @@ const OutboxMarkPublishedSQL = `UPDATE outbox
SET published_at = $2
WHERE event_id = $1 AND published_at IS NULL`
// MaxOutboxDeliveryAttempts bounds how long one undeliverable row may hold up
// its event type. It must exceed any plausible transient outage of the local
// fan-out adapter, since a healthy event is retried through the same counter.
const MaxOutboxDeliveryAttempts = 20
// OutboxRecordFailureSQL increments the attempt counter and dead-letters the
// row once it is exhausted, in one statement so a crash between the two cannot
// leave a row that is retried forever.
const OutboxRecordFailureSQL = `UPDATE outbox
SET delivery_attempts = delivery_attempts + 1,
last_delivery_error = $2,
dead_lettered_at = CASE WHEN delivery_attempts + 1 >= $3 THEN $4 ELSE dead_lettered_at END
WHERE event_id = $1 AND published_at IS NULL AND dead_lettered_at IS NULL
RETURNING dead_lettered_at IS NOT NULL`
const OutboxDeadLetteredCountSQL = `SELECT count(*) FROM outbox WHERE dead_lettered_at IS NOT NULL`
var ErrOutboxEventNotFound = fmt.Errorf("outbox event not found or already published")
// RecordOutboxDeliveryFailure notes one failed delivery attempt and reports
// whether the row was dead-lettered as a result. Callers should keep going to
// the next event: the whole point is that one poison row must not stall the
// others.
func RecordOutboxDeliveryFailure(ctx context.Context, db *sql.DB, eventID string, cause error, now time.Time) (bool, error) {
if db == nil || eventID == "" || now.IsZero() {
return false, fmt.Errorf("invalid outbox failure arguments")
}
message := ""
if cause != nil {
message = cause.Error()
}
if len(message) > 500 {
message = message[:500]
}
var deadLettered bool
err := db.QueryRowContext(ctx, OutboxRecordFailureSQL, eventID, message, MaxOutboxDeliveryAttempts, now).Scan(&deadLettered)
if err == sql.ErrNoRows {
// Published or already dead-lettered by another replica; nothing owed.
return false, nil
}
if err != nil {
return false, err
}
return deadLettered, nil
}
// CountDeadLetteredOutboxEvents backs the deletion-lag/poison-row metric. A
// non-zero value means at least one lifecycle event was never delivered.
func CountDeadLetteredOutboxEvents(ctx context.Context, db *sql.DB) (int64, error) {
if db == nil {
return 0, fmt.Errorf("invalid outbox count arguments")
}
var count int64
if err := db.QueryRowContext(ctx, OutboxDeadLetteredCountSQL).Scan(&count); err != nil {
return 0, err
}
return count, nil
}
type OutboxDelivery func(context.Context, OutboxEvent) error
// OutboxDispatcher is the durable-to-transient bridge. Read and Ack are