fix(multiplayer): terminate proposal offenders atomically

This commit is contained in:
Josh Creek
2026-09-03 00:09:07 +01:00
parent aa446cfbfe
commit f8af212e3f
6 changed files with 206 additions and 67 deletions
+64 -26
View File
@@ -20,22 +20,24 @@ WHERE proposal_id = $1 AND state = 'OPEN' AND expires_at <= $2`
const ProposalParticipantExpireSQL = `UPDATE proposal_participants
SET response = 'TIMED_OUT', responded_at = $2
WHERE proposal_id = $1 AND response = 'PENDING'
AND EXISTS (SELECT 1 FROM proposals WHERE proposals.proposal_id = proposal_participants.proposal_id AND proposals.expires_at <= $2)`
AND EXISTS (SELECT 1 FROM proposals WHERE proposals.proposal_id = proposal_participants.proposal_id
AND proposals.state = 'EXPIRED' AND proposals.expires_at <= $2)`
// ProposalExpireRequeueSQL is the timeout sibling of
// ProposalDeclineRequeueSQL: a proposal that simply times out (no unanimous
// response inside the 10s window) leaves any participant still holding a
// PROPOSED ticket exactly as stranded as an explicit decline does, and for
// the identical reason -- nothing else ever moves a PROPOSED ticket back to
// QUEUED. The `state = 'EXPIRED'` guard makes this safe to call
// unconditionally right after ProposalExpireSQL: it's a no-op on a proposal
// that was already OPEN and stays OPEN (nothing to requeue) or one that was
// already EXPIRED on a prior pass (its participants' tickets, if any were
// still PROPOSED, were already requeued then).
// ProposalExpireRequeueSQL preserves queue precedence only for participants
// who accepted. Participants who did not respond are offenders and their
// tickets are terminated separately by ProposalTimeoutTicketExpireSQL.
const ProposalExpireRequeueSQL = `UPDATE queue_tickets q
SET state = 'QUEUED', expires_at = $2, revision = revision + 1
FROM proposal_participants pp
WHERE pp.proposal_id = $1 AND q.ticket_id = pp.ticket_id AND q.player_id = pp.player_id AND q.state = 'PROPOSED'
AND pp.response = 'ACCEPTED'
AND EXISTS (SELECT 1 FROM proposals WHERE proposals.proposal_id = $1 AND proposals.state = 'EXPIRED')`
const ProposalTimeoutTicketExpireSQL = `UPDATE queue_tickets q
SET state = 'EXPIRED', revision = revision + 1
FROM proposal_participants pp
WHERE pp.proposal_id = $1 AND q.ticket_id = pp.ticket_id AND q.player_id = pp.player_id AND q.state = 'PROPOSED'
AND pp.response = 'TIMED_OUT'
AND EXISTS (SELECT 1 FROM proposals WHERE proposals.proposal_id = $1 AND proposals.state = 'EXPIRED')`
const OpenProposalForCancelledTicketSQL = `SELECT pp.proposal_id
@@ -47,8 +49,8 @@ WHERE pp.ticket_id = $1 AND pp.player_id = $2 AND p.state = 'OPEN'`
// immediately when one of its participants cancels their own queue ticket
// directly, rather than leaving every other participant to wait out the
// full response window for something the system already knows can't happen
// -- ProposalExpireRequeueSQL would eventually rescue them anyway, but not
// for up to ProposalWindow's full duration for no reason. Must run inside
// -- expiry recovery would eventually release them anyway, but not for up to
// ProposalWindow's full duration for no reason. Must run inside
// the same transaction as the ticket cancel itself; a no-op if the ticket
// wasn't part of any currently-OPEN proposal.
func CascadeCancelToOpenProposal(ctx context.Context, tx *sql.Tx, ticketID, playerID string, now time.Time) error {
@@ -63,7 +65,7 @@ func CascadeCancelToOpenProposal(ctx context.Context, tx *sql.Tx, ticketID, play
if _, err := tx.ExecContext(ctx, ProposalDeclineSQL, proposalID); err != nil {
return err
}
_, err = tx.ExecContext(ctx, ProposalDeclineRequeueSQL, proposalID, now.Add(domain.QueueExpiryWindow))
_, err = tx.ExecContext(ctx, ProposalAbortRequeueSQL, proposalID, now.Add(domain.QueueExpiryWindow))
return err
}
@@ -89,6 +91,9 @@ FROM idempotency_keys
WHERE scope = $1 AND idempotency_key = $2
FOR UPDATE`
const ProposalResponseIdempotencyDeleteSQL = `DELETE FROM idempotency_keys
WHERE scope = $1 AND idempotency_key = $2`
const ProposalLockSQL = `SELECT playlist, state, revision, expires_at
FROM proposals
WHERE proposal_id = $1
@@ -115,21 +120,33 @@ const ProposalDeclineSQL = `UPDATE proposals
SET state = 'DECLINED', revision = revision + 1
WHERE proposal_id = $1 AND state = 'OPEN'`
// ProposalDeclineRequeueSQL requeues every participant's ticket, including
// the decliner's own. The durable decline penalty separately prevents that
// player from immediately creating a replacement ticket; leaving this ticket
// at PROPOSED would not implement a cooldown, it would strand the player and
// hide the ticket from the matcher.
const ProposalDeclineActorCancelSQL = `UPDATE queue_tickets q
SET state = 'CANCELLED', revision = revision + 1
FROM proposal_participants pp
WHERE pp.proposal_id = $1 AND pp.player_id = $2
AND q.ticket_id = pp.ticket_id AND q.player_id = pp.player_id AND q.state = 'PROPOSED'`
// ProposalDeclineRequeueSQL preserves the original queue precedence of every
// innocent participant while terminating the declining player's ticket.
const ProposalDeclineRequeueSQL = `UPDATE queue_tickets q
SET state = 'QUEUED', expires_at = $2, revision = revision + 1
FROM proposal_participants pp
WHERE pp.proposal_id = $1 AND pp.player_id <> $3
AND q.ticket_id = pp.ticket_id AND q.player_id = pp.player_id AND q.state = 'PROPOSED'`
// ProposalAbortRequeueSQL is used when a participant has already cancelled
// their own ticket. It requeues every remaining PROPOSED ticket; the cancelled
// ticket cannot be selected by the state predicate.
const ProposalAbortRequeueSQL = `UPDATE queue_tickets q
SET state = 'QUEUED', expires_at = $2, revision = revision + 1
FROM proposal_participants pp
WHERE pp.proposal_id = $1 AND q.ticket_id = pp.ticket_id AND q.player_id = pp.player_id AND q.state = 'PROPOSED'`
const ProposalCooldownEventsSQL = `SELECT kind, starts_at
FROM penalties
WHERE player_id = $1 AND playlist = $2
AND kind IN ('PROPOSAL_DECLINED', 'PROPOSAL_TIMEOUT')
AND starts_at >= $3
AND starts_at >= $3 AND starts_at <= $4
ORDER BY starts_at`
const ProposalCooldownInsertSQL = `INSERT INTO penalties
@@ -143,7 +160,7 @@ WHERE proposal_id = $1 AND response = 'TIMED_OUT' AND responded_at = $2
ORDER BY player_id`
func recordProposalCooldown(ctx context.Context, tx *sql.Tx, playerID string, playlist domain.Playlist, proposalID, kind string, response domain.Response, now time.Time) error {
rows, err := tx.QueryContext(ctx, ProposalCooldownEventsSQL, playerID, string(playlist), now.Add(-30*time.Minute))
rows, err := tx.QueryContext(ctx, ProposalCooldownEventsSQL, playerID, string(playlist), now.Add(-30*time.Minute), now)
if err != nil {
return err
}
@@ -162,6 +179,10 @@ func recordProposalCooldown(ctx context.Context, tx *sql.Tx, playerID string, pl
events = append(events, domain.CooldownEvent{At: at, Playlist: playlist, Kind: response})
}
if err := rows.Err(); err != nil {
rows.Close()
return err
}
if err := rows.Close(); err != nil {
return err
}
events = append(events, domain.CooldownEvent{At: now, Playlist: playlist, Kind: response})
@@ -235,6 +256,9 @@ func GetProposal(ctx context.Context, db *sql.DB, playerID, proposalID string, n
return domain.Proposal{}, err
}
}
if _, err := tx.ExecContext(ctx, ProposalTimeoutTicketExpireSQL, proposalID); err != nil {
return domain.Proposal{}, err
}
if _, err := tx.ExecContext(ctx, ProposalExpireRequeueSQL, proposalID, now.Add(domain.QueueExpiryWindow)); err != nil {
return domain.Proposal{}, err
}
@@ -278,7 +302,9 @@ func RespondToProposal(ctx context.Context, db *sql.DB, playerID, proposalID, id
}
digest := sha256.Sum256([]byte(fmt.Sprintf("%s|%s|%t|%d", playerID, proposalID, accept, expectedRevision)))
var proposal domain.Proposal
closed := false
err := RunSerializable(ctx, db, DefaultSerializableAttempts, func(ctx context.Context, tx *sql.Tx) error {
closed = false
result, err := tx.ExecContext(ctx, ProposalResponseIdempotencyInsertSQL, ProposalResponseIdempotencyScope, idempotencyKey, digest[:], []byte("{}"))
if err != nil {
return err
@@ -321,14 +347,20 @@ func RespondToProposal(ctx context.Context, db *sql.DB, playerID, proposalID, id
if err := recordProposalTimeoutCooldowns(ctx, tx, proposalID, domain.Playlist(playlist), now); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, ProposalTimeoutTicketExpireSQL, proposalID); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, ProposalExpireRequeueSQL, proposalID, now.Add(domain.QueueExpiryWindow)); err != nil {
return err
}
if !now.Before(expiresAt) {
return domain.ErrProposalClosed
}
if state != string(domain.Open) || !now.Before(expiresAt) {
return domain.ErrProposalClosed
// Commit any expiry recovery above, but do not retain a placeholder
// idempotency result for a mutation that was rejected as closed.
if _, err := tx.ExecContext(ctx, ProposalResponseIdempotencyDeleteSQL, ProposalResponseIdempotencyScope, idempotencyKey); err != nil {
return err
}
closed = true
return nil
}
if revision != expectedRevision {
return domain.ErrStaleRevision
@@ -374,7 +406,10 @@ func RespondToProposal(ctx context.Context, db *sql.DB, playerID, proposalID, id
if err := recordProposalDeclineCooldown(ctx, tx, playerID, domain.Playlist(playlist), proposalID, now); err != nil {
return err
}
_, err = tx.ExecContext(ctx, ProposalDeclineRequeueSQL, proposalID, now.Add(domain.QueueExpiryWindow))
if _, err = tx.ExecContext(ctx, ProposalDeclineActorCancelSQL, proposalID, playerID); err != nil {
return err
}
_, err = tx.ExecContext(ctx, ProposalDeclineRequeueSQL, proposalID, now.Add(domain.QueueExpiryWindow), playerID)
}
if err != nil {
return err
@@ -411,5 +446,8 @@ func RespondToProposal(ctx context.Context, db *sql.DB, playerID, proposalID, id
_, err = tx.ExecContext(ctx, `UPDATE idempotency_keys SET result = $3 WHERE scope = $1 AND idempotency_key = $2`, ProposalResponseIdempotencyScope, idempotencyKey, stored)
return err
})
if err == nil && closed {
return domain.Proposal{}, domain.ErrProposalClosed
}
return proposal, err
}