Files
CosmicClash/server/store/proposal_recovery_sql.go
T

454 lines
17 KiB
Go

package store
import (
"bytes"
"context"
"crypto/sha256"
"database/sql"
"encoding/json"
"fmt"
"strings"
"time"
"github.com/cosmic-clash/cosmic-clash/server/domain"
)
const ProposalExpireSQL = `UPDATE proposals
SET state = 'EXPIRED', revision = revision + 1
WHERE proposal_id = $1 AND state = 'OPEN' AND expires_at <= $2`
const ProposalParticipantExpireSQL = `UPDATE proposal_participants
SET response = 'TIMED_OUT', responded_at = $2
WHERE proposal_id = $1 AND response = 'PENDING'
AND EXISTS (SELECT 1 FROM proposals WHERE proposals.proposal_id = proposal_participants.proposal_id
AND proposals.state = 'EXPIRED' AND proposals.expires_at <= $2)`
// ProposalExpireRequeueSQL preserves queue precedence only for participants
// who accepted. Participants who did not respond are offenders and their
// tickets are terminated separately by ProposalTimeoutTicketExpireSQL.
const ProposalExpireRequeueSQL = `UPDATE queue_tickets q
SET state = 'QUEUED', expires_at = $2, revision = revision + 1
FROM proposal_participants pp
WHERE pp.proposal_id = $1 AND q.ticket_id = pp.ticket_id AND q.player_id = pp.player_id AND q.state = 'PROPOSED'
AND pp.response = 'ACCEPTED'
AND EXISTS (SELECT 1 FROM proposals WHERE proposals.proposal_id = $1 AND proposals.state = 'EXPIRED')`
const ProposalTimeoutTicketExpireSQL = `UPDATE queue_tickets q
SET state = 'EXPIRED', revision = revision + 1
FROM proposal_participants pp
WHERE pp.proposal_id = $1 AND q.ticket_id = pp.ticket_id AND q.player_id = pp.player_id AND q.state = 'PROPOSED'
AND pp.response = 'TIMED_OUT'
AND EXISTS (SELECT 1 FROM proposals WHERE proposals.proposal_id = $1 AND proposals.state = 'EXPIRED')`
const OpenProposalForCancelledTicketSQL = `SELECT pp.proposal_id
FROM proposal_participants pp
JOIN proposals p ON p.proposal_id = pp.proposal_id
WHERE pp.ticket_id = $1 AND pp.player_id = $2 AND p.state = 'OPEN'`
// CascadeCancelToOpenProposal declines and requeues an OPEN proposal
// immediately when one of its participants cancels their own queue ticket
// directly, rather than leaving every other participant to wait out the
// full response window for something the system already knows can't happen
// -- expiry recovery would eventually release them anyway, but not for up to
// ProposalWindow's full duration for no reason. Must run inside
// the same transaction as the ticket cancel itself; a no-op if the ticket
// wasn't part of any currently-OPEN proposal.
func CascadeCancelToOpenProposal(ctx context.Context, tx *sql.Tx, ticketID, playerID string, now time.Time) error {
var proposalID string
err := tx.QueryRowContext(ctx, OpenProposalForCancelledTicketSQL, ticketID, playerID).Scan(&proposalID)
if err == sql.ErrNoRows {
return nil
}
if err != nil {
return err
}
if _, err := tx.ExecContext(ctx, ProposalDeclineSQL, proposalID); err != nil {
return err
}
_, err = tx.ExecContext(ctx, ProposalAbortRequeueSQL, proposalID, now.Add(domain.QueueExpiryWindow))
return err
}
const ProposalRecoverySelectSQL = `SELECT proposal_id, playlist, state, revision, expires_at
FROM proposals
WHERE proposal_id = $1
AND EXISTS (SELECT 1 FROM proposal_participants WHERE proposal_id = proposals.proposal_id AND player_id = $2)`
const ProposalParticipantsSelectSQL = `SELECT player_id, response
FROM proposal_participants
WHERE proposal_id = $1
ORDER BY player_id`
const ProposalResponseIdempotencyScope = "proposal.respond"
const ProposalResponseIdempotencyInsertSQL = `INSERT INTO idempotency_keys
(scope, idempotency_key, payload_digest, result)
VALUES ($1, $2, $3, $4)
ON CONFLICT (scope, idempotency_key) DO NOTHING`
const ProposalResponseIdempotencySelectSQL = `SELECT payload_digest, result
FROM idempotency_keys
WHERE scope = $1 AND idempotency_key = $2
FOR UPDATE`
const ProposalResponseIdempotencyDeleteSQL = `DELETE FROM idempotency_keys
WHERE scope = $1 AND idempotency_key = $2`
const ProposalLockSQL = `SELECT playlist, state, revision, expires_at
FROM proposals
WHERE proposal_id = $1
FOR UPDATE`
const ProposalParticipantLockSQL = `SELECT response
FROM proposal_participants
WHERE proposal_id = $1 AND player_id = $2
FOR UPDATE`
const ProposalParticipantRespondSQL = `UPDATE proposal_participants
SET response = $3, responded_at = $4
WHERE proposal_id = $1 AND player_id = $2 AND response = 'PENDING'`
const ProposalCountPendingSQL = `SELECT COUNT(*)
FROM proposal_participants
WHERE proposal_id = $1 AND response = 'PENDING'`
const ProposalAcceptSQL = `UPDATE proposals
SET state = 'ACCEPTED', revision = revision + 1
WHERE proposal_id = $1 AND state = 'OPEN'`
const ProposalDeclineSQL = `UPDATE proposals
SET state = 'DECLINED', revision = revision + 1
WHERE proposal_id = $1 AND state = 'OPEN'`
const ProposalDeclineActorCancelSQL = `UPDATE queue_tickets q
SET state = 'CANCELLED', revision = revision + 1
FROM proposal_participants pp
WHERE pp.proposal_id = $1 AND pp.player_id = $2
AND q.ticket_id = pp.ticket_id AND q.player_id = pp.player_id AND q.state = 'PROPOSED'`
// ProposalDeclineRequeueSQL preserves the original queue precedence of every
// innocent participant while terminating the declining player's ticket.
const ProposalDeclineRequeueSQL = `UPDATE queue_tickets q
SET state = 'QUEUED', expires_at = $2, revision = revision + 1
FROM proposal_participants pp
WHERE pp.proposal_id = $1 AND pp.player_id <> $3
AND q.ticket_id = pp.ticket_id AND q.player_id = pp.player_id AND q.state = 'PROPOSED'`
// ProposalAbortRequeueSQL is used when a participant has already cancelled
// their own ticket. It requeues every remaining PROPOSED ticket; the cancelled
// ticket cannot be selected by the state predicate.
const ProposalAbortRequeueSQL = `UPDATE queue_tickets q
SET state = 'QUEUED', expires_at = $2, revision = revision + 1
FROM proposal_participants pp
WHERE pp.proposal_id = $1 AND q.ticket_id = pp.ticket_id AND q.player_id = pp.player_id AND q.state = 'PROPOSED'`
const ProposalCooldownEventsSQL = `SELECT kind, starts_at
FROM penalties
WHERE player_id = $1 AND playlist = $2
AND kind IN ('PROPOSAL_DECLINED', 'PROPOSAL_TIMEOUT')
AND starts_at >= $3 AND starts_at <= $4
ORDER BY starts_at`
const ProposalCooldownInsertSQL = `INSERT INTO penalties
(penalty_id, player_id, playlist, kind, starts_at, ends_at)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (penalty_id) DO NOTHING`
const ProposalTimedOutParticipantsSQL = `SELECT player_id
FROM proposal_participants
WHERE proposal_id = $1 AND response = 'TIMED_OUT' AND responded_at = $2
ORDER BY player_id`
func recordProposalCooldown(ctx context.Context, tx *sql.Tx, playerID string, playlist domain.Playlist, proposalID, kind string, response domain.Response, now time.Time) error {
rows, err := tx.QueryContext(ctx, ProposalCooldownEventsSQL, playerID, string(playlist), now.Add(-30*time.Minute), now)
if err != nil {
return err
}
defer rows.Close()
events := make([]domain.CooldownEvent, 0)
for rows.Next() {
var kind string
var at time.Time
if err := rows.Scan(&kind, &at); err != nil {
return err
}
response := domain.TimedOutResponse
if kind == "PROPOSAL_DECLINED" {
response = domain.DeclinedResponse
}
events = append(events, domain.CooldownEvent{At: at, Playlist: playlist, Kind: response})
}
if err := rows.Err(); err != nil {
rows.Close()
return err
}
if err := rows.Close(); err != nil {
return err
}
events = append(events, domain.CooldownEvent{At: now, Playlist: playlist, Kind: response})
until := domain.CooldownUntil(events, playlist, now)
_, err = tx.ExecContext(ctx, ProposalCooldownInsertSQL, "proposal-"+strings.ToLower(kind)+":"+proposalID+":"+playerID, playerID, string(playlist), kind, now, until)
return err
}
func recordProposalDeclineCooldown(ctx context.Context, tx *sql.Tx, playerID string, playlist domain.Playlist, proposalID string, now time.Time) error {
return recordProposalCooldown(ctx, tx, playerID, playlist, proposalID, "PROPOSAL_DECLINED", domain.DeclinedResponse, now)
}
func recordProposalTimeoutCooldowns(ctx context.Context, tx *sql.Tx, proposalID string, playlist domain.Playlist, now time.Time) error {
rows, err := tx.QueryContext(ctx, ProposalTimedOutParticipantsSQL, proposalID, now)
if err != nil {
return err
}
defer rows.Close()
players := make([]string, 0)
for rows.Next() {
var playerID string
if err := rows.Scan(&playerID); err != nil {
return err
}
players = append(players, playerID)
}
if err := rows.Err(); err != nil {
return err
}
if err := rows.Close(); err != nil {
return err
}
for _, playerID := range players {
if err := recordProposalCooldown(ctx, tx, playerID, playlist, proposalID, "PROPOSAL_TIMEOUT", domain.TimedOutResponse, now); err != nil {
return err
}
}
return nil
}
const ProposalRevisionBumpSQL = `UPDATE proposals
SET revision = revision + 1
WHERE proposal_id = $1 AND state = 'OPEN'`
var ErrProposalResponseConflict = fmt.Errorf("proposal response conflict")
// GetProposal recovers the full proposal only after proving the caller is a
// participant. Expiry is advanced in the same transaction as the read so a
// missed event cannot leave a durable proposal indefinitely OPEN.
func GetProposal(ctx context.Context, db *sql.DB, playerID, proposalID string, now time.Time) (domain.Proposal, error) {
if db == nil || playerID == "" || proposalID == "" || now.IsZero() {
return domain.Proposal{}, fmt.Errorf("invalid proposal recovery arguments")
}
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return domain.Proposal{}, err
}
defer tx.Rollback()
if _, err := tx.ExecContext(ctx, ProposalExpireSQL, proposalID, now); err != nil {
return domain.Proposal{}, err
}
if _, err := tx.ExecContext(ctx, ProposalParticipantExpireSQL, proposalID, now); err != nil {
return domain.Proposal{}, err
}
var cooldownPlaylist string
if err := tx.QueryRowContext(ctx, `SELECT playlist FROM proposals WHERE proposal_id = $1`, proposalID).Scan(&cooldownPlaylist); err != nil && err != sql.ErrNoRows {
return domain.Proposal{}, err
}
if cooldownPlaylist != "" {
if err := recordProposalTimeoutCooldowns(ctx, tx, proposalID, domain.Playlist(cooldownPlaylist), now); err != nil {
return domain.Proposal{}, err
}
}
if _, err := tx.ExecContext(ctx, ProposalTimeoutTicketExpireSQL, proposalID); err != nil {
return domain.Proposal{}, err
}
if _, err := tx.ExecContext(ctx, ProposalExpireRequeueSQL, proposalID, now.Add(domain.QueueExpiryWindow)); err != nil {
return domain.Proposal{}, err
}
var proposal domain.Proposal
var playlist, state string
if err := tx.QueryRowContext(ctx, ProposalRecoverySelectSQL, proposalID, playerID).Scan(&proposal.ProposalID, &playlist, &state, &proposal.Revision, &proposal.ExpiresAt); err != nil {
return domain.Proposal{}, err
}
proposal.Playlist = domain.Playlist(playlist)
proposal.State = domain.State(state)
rows, err := tx.QueryContext(ctx, ProposalParticipantsSelectSQL, proposalID)
if err != nil {
return domain.Proposal{}, err
}
defer rows.Close()
for rows.Next() {
var participant domain.ProposalParticipant
if err := rows.Scan(&participant.PlayerID, &participant.Response); err != nil {
return domain.Proposal{}, err
}
proposal.Participants = append(proposal.Participants, participant)
}
if err := rows.Err(); err != nil {
return domain.Proposal{}, err
}
if len(proposal.Participants) == 0 {
return domain.Proposal{}, fmt.Errorf("proposal has no participants")
}
if err := tx.Commit(); err != nil {
return domain.Proposal{}, err
}
return proposal, nil
}
// RespondToProposal is the durable mutation counterpart to GetProposal. The
// proposal row and participant row are locked in one transaction; the result
// is stored under the idempotency key before the transaction commits.
func RespondToProposal(ctx context.Context, db *sql.DB, playerID, proposalID, idempotencyKey string, accept bool, expectedRevision uint64, now time.Time) (domain.Proposal, error) {
if db == nil || playerID == "" || proposalID == "" || len(idempotencyKey) < 16 || len(idempotencyKey) > 128 || now.IsZero() {
return domain.Proposal{}, fmt.Errorf("invalid proposal response arguments")
}
digest := sha256.Sum256([]byte(fmt.Sprintf("%s|%s|%t|%d", playerID, proposalID, accept, expectedRevision)))
var proposal domain.Proposal
closed := false
err := RunSerializable(ctx, db, DefaultSerializableAttempts, func(ctx context.Context, tx *sql.Tx) error {
closed = false
result, err := tx.ExecContext(ctx, ProposalResponseIdempotencyInsertSQL, ProposalResponseIdempotencyScope, idempotencyKey, digest[:], []byte("{}"))
if err != nil {
return err
}
inserted, err := result.RowsAffected()
if err != nil {
return err
}
if inserted == 0 {
var priorDigest, priorResult []byte
if err := tx.QueryRowContext(ctx, ProposalResponseIdempotencySelectSQL, ProposalResponseIdempotencyScope, idempotencyKey).Scan(&priorDigest, &priorResult); err != nil {
return err
}
if !bytes.Equal(priorDigest, digest[:]) {
return ErrProposalResponseConflict
}
if err := json.Unmarshal(priorResult, &proposal); err != nil {
return fmt.Errorf("invalid stored proposal response: %w", err)
}
return nil
}
var playlist, state string
var revision uint64
var expiresAt time.Time
if err := tx.QueryRowContext(ctx, ProposalLockSQL, proposalID).Scan(&playlist, &state, &revision, &expiresAt); err != nil {
return err
}
// A mutation is also a recovery boundary. If the response arrives after
// the window, advance both the proposal and its pending participants in
// this same transaction before returning the closed error. Otherwise a
// client that missed the expiry event could observe OPEN/PENDING forever
// when its first durable interaction is an accept/decline.
if _, err := tx.ExecContext(ctx, ProposalExpireSQL, proposalID, now); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, ProposalParticipantExpireSQL, proposalID, now); err != nil {
return err
}
if err := recordProposalTimeoutCooldowns(ctx, tx, proposalID, domain.Playlist(playlist), now); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, ProposalTimeoutTicketExpireSQL, proposalID); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, ProposalExpireRequeueSQL, proposalID, now.Add(domain.QueueExpiryWindow)); err != nil {
return err
}
if state != string(domain.Open) || !now.Before(expiresAt) {
// Commit any expiry recovery above, but do not retain a placeholder
// idempotency result for a mutation that was rejected as closed.
if _, err := tx.ExecContext(ctx, ProposalResponseIdempotencyDeleteSQL, ProposalResponseIdempotencyScope, idempotencyKey); err != nil {
return err
}
closed = true
return nil
}
if revision != expectedRevision {
return domain.ErrStaleRevision
}
var response string
if err := tx.QueryRowContext(ctx, ProposalParticipantLockSQL, proposalID, playerID).Scan(&response); err != nil {
return domain.ErrNotParticipant
}
if response != string(domain.Pending) {
return domain.ErrConflict
}
response = string(domain.DeclinedResponse)
if accept {
response = string(domain.AcceptedResponse)
}
changed, err := tx.ExecContext(ctx, ProposalParticipantRespondSQL, proposalID, playerID, response, now)
if err != nil {
return err
}
if count, err := changed.RowsAffected(); err != nil || count != 1 {
return ErrProposalResponseConflict
}
targetState := string(domain.Declined)
if accept {
var pending int
if err := tx.QueryRowContext(ctx, ProposalCountPendingSQL, proposalID).Scan(&pending); err != nil {
return err
}
if pending == 0 {
targetState = string(domain.Accepted)
} else {
targetState = state
}
}
if targetState != state {
if targetState == string(domain.Accepted) {
_, err = tx.ExecContext(ctx, ProposalAcceptSQL, proposalID)
} else {
_, err = tx.ExecContext(ctx, ProposalDeclineSQL, proposalID)
if err != nil {
return err
}
if err := recordProposalDeclineCooldown(ctx, tx, playerID, domain.Playlist(playlist), proposalID, now); err != nil {
return err
}
if _, err = tx.ExecContext(ctx, ProposalDeclineActorCancelSQL, proposalID, playerID); err != nil {
return err
}
_, err = tx.ExecContext(ctx, ProposalDeclineRequeueSQL, proposalID, now.Add(domain.QueueExpiryWindow), playerID)
}
if err != nil {
return err
}
revision++
} else {
if _, err := tx.ExecContext(ctx, ProposalRevisionBumpSQL, proposalID); err != nil {
return err
}
revision++
}
proposal = domain.Proposal{ProposalID: proposalID, Playlist: domain.Playlist(playlist), State: domain.State(targetState), Revision: revision, ExpiresAt: expiresAt}
rows, err := tx.QueryContext(ctx, ProposalParticipantsSelectSQL, proposalID)
if err != nil {
return err
}
for rows.Next() {
var participant domain.ProposalParticipant
if err := rows.Scan(&participant.PlayerID, &participant.Response); err != nil {
rows.Close()
return err
}
proposal.Participants = append(proposal.Participants, participant)
}
if err := rows.Err(); err != nil {
rows.Close()
return err
}
rows.Close()
stored, err := json.Marshal(proposal)
if err != nil {
return err
}
_, err = tx.ExecContext(ctx, `UPDATE idempotency_keys SET result = $3 WHERE scope = $1 AND idempotency_key = $2`, ProposalResponseIdempotencyScope, idempotencyKey, stored)
return err
})
if err == nil && closed {
return domain.Proposal{}, domain.ErrProposalClosed
}
return proposal, err
}