mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-16 10:02:08 +00:00
fix(server): repair the initial-connect outbox envelope and unblock dispatch
ApplyInitialConnectPlan wrote a payload of {match_id,state,action},
omitting event, revision, resource_id, occurred_at and player_ids --
every field deliverStateOutboxEvent requires. Delivery rejected the row,
dispatch returned on the first error so it was never acknowledged, and
because reads are ordered oldest-first it was retried ahead of every
later state_changed event on every 100ms poll. One initial-connect
transition therefore blocked lifecycle delivery for all matches, not
just its own.
Two independent fixes, since either alone leaves the system fragile:
Build envelopes through one validating helper (MarshalOutboxEnvelope)
and convert all five writers to it. A writer that omits a required
field now fails its own transaction instead of committing a row that
can only ever poison the queue. The helper takes revision as int64 so
the -1 "nothing matched" sentinel some CTEs return surfaces as an error
rather than wrapping to a huge uint64.
Make dispatch resilient regardless: a delivery failure is now counted
against that row and the batch continues, with the row dead-lettered
after MaxOutboxDeliveryAttempts so a poison event degrades to one lost
notification instead of a stalled queue. Ordering within an aggregate
is still honoured -- later events of a failed match are deferred, so no
client observes that match's newer state before its older state. An ack
failure still stops the batch, being a database rather than a payload
problem.
Initial-connect events now address every participant, not just the
connected ones: a no-show needs to learn their ticket was failed and a
penalty applied.
This commit is contained in:
+61
-4
@@ -25,28 +25,28 @@ type OutboxEvent struct {
|
||||
const OutboxUnpublishedSelectSQL = `SELECT event_id, aggregate_type, aggregate_id, revision,
|
||||
event_type, payload, created_at, published_at
|
||||
FROM outbox
|
||||
WHERE published_at IS NULL
|
||||
WHERE published_at IS NULL AND dead_lettered_at IS NULL
|
||||
ORDER BY created_at, event_id
|
||||
LIMIT $1`
|
||||
|
||||
const OutboxUnpublishedProposalSelectSQL = `SELECT event_id, aggregate_type, aggregate_id, revision,
|
||||
event_type, payload, created_at, published_at
|
||||
FROM outbox
|
||||
WHERE published_at IS NULL AND event_type = 'proposal_changed'
|
||||
WHERE published_at IS NULL AND dead_lettered_at IS NULL AND event_type = 'proposal_changed'
|
||||
ORDER BY created_at, event_id
|
||||
LIMIT $1`
|
||||
|
||||
const OutboxUnpublishedResultSelectSQL = `SELECT event_id, aggregate_type, aggregate_id, revision,
|
||||
event_type, payload, created_at, published_at
|
||||
FROM outbox
|
||||
WHERE published_at IS NULL AND event_type = 'match_completed'
|
||||
WHERE published_at IS NULL AND dead_lettered_at IS NULL AND event_type = 'match_completed'
|
||||
ORDER BY created_at, event_id
|
||||
LIMIT $1`
|
||||
|
||||
const OutboxUnpublishedStateSelectSQL = `SELECT event_id, aggregate_type, aggregate_id, revision,
|
||||
event_type, payload, created_at, published_at
|
||||
FROM outbox
|
||||
WHERE published_at IS NULL AND event_type = 'state_changed'
|
||||
WHERE published_at IS NULL AND dead_lettered_at IS NULL AND event_type = 'state_changed'
|
||||
ORDER BY created_at, event_id
|
||||
LIMIT $1`
|
||||
|
||||
@@ -59,8 +59,65 @@ const OutboxMarkPublishedSQL = `UPDATE outbox
|
||||
SET published_at = $2
|
||||
WHERE event_id = $1 AND published_at IS NULL`
|
||||
|
||||
// MaxOutboxDeliveryAttempts bounds how long one undeliverable row may hold up
|
||||
// its event type. It must exceed any plausible transient outage of the local
|
||||
// fan-out adapter, since a healthy event is retried through the same counter.
|
||||
const MaxOutboxDeliveryAttempts = 20
|
||||
|
||||
// OutboxRecordFailureSQL increments the attempt counter and dead-letters the
|
||||
// row once it is exhausted, in one statement so a crash between the two cannot
|
||||
// leave a row that is retried forever.
|
||||
const OutboxRecordFailureSQL = `UPDATE outbox
|
||||
SET delivery_attempts = delivery_attempts + 1,
|
||||
last_delivery_error = $2,
|
||||
dead_lettered_at = CASE WHEN delivery_attempts + 1 >= $3 THEN $4 ELSE dead_lettered_at END
|
||||
WHERE event_id = $1 AND published_at IS NULL AND dead_lettered_at IS NULL
|
||||
RETURNING dead_lettered_at IS NOT NULL`
|
||||
|
||||
const OutboxDeadLetteredCountSQL = `SELECT count(*) FROM outbox WHERE dead_lettered_at IS NOT NULL`
|
||||
|
||||
var ErrOutboxEventNotFound = fmt.Errorf("outbox event not found or already published")
|
||||
|
||||
// RecordOutboxDeliveryFailure notes one failed delivery attempt and reports
|
||||
// whether the row was dead-lettered as a result. Callers should keep going to
|
||||
// the next event: the whole point is that one poison row must not stall the
|
||||
// others.
|
||||
func RecordOutboxDeliveryFailure(ctx context.Context, db *sql.DB, eventID string, cause error, now time.Time) (bool, error) {
|
||||
if db == nil || eventID == "" || now.IsZero() {
|
||||
return false, fmt.Errorf("invalid outbox failure arguments")
|
||||
}
|
||||
message := ""
|
||||
if cause != nil {
|
||||
message = cause.Error()
|
||||
}
|
||||
if len(message) > 500 {
|
||||
message = message[:500]
|
||||
}
|
||||
var deadLettered bool
|
||||
err := db.QueryRowContext(ctx, OutboxRecordFailureSQL, eventID, message, MaxOutboxDeliveryAttempts, now).Scan(&deadLettered)
|
||||
if err == sql.ErrNoRows {
|
||||
// Published or already dead-lettered by another replica; nothing owed.
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return deadLettered, nil
|
||||
}
|
||||
|
||||
// CountDeadLetteredOutboxEvents backs the deletion-lag/poison-row metric. A
|
||||
// non-zero value means at least one lifecycle event was never delivered.
|
||||
func CountDeadLetteredOutboxEvents(ctx context.Context, db *sql.DB) (int64, error) {
|
||||
if db == nil {
|
||||
return 0, fmt.Errorf("invalid outbox count arguments")
|
||||
}
|
||||
var count int64
|
||||
if err := db.QueryRowContext(ctx, OutboxDeadLetteredCountSQL).Scan(&count); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
type OutboxDelivery func(context.Context, OutboxEvent) error
|
||||
|
||||
// OutboxDispatcher is the durable-to-transient bridge. Read and Ack are
|
||||
|
||||
Reference in New Issue
Block a user