mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
feat: persist queue probe metadata
This commit is contained in:
Josh Creek
+3
-1
@@ -47,7 +47,9 @@ product policy are in [`docs/MATCHMAKING.md`](docs/MATCHMAKING.md).
|
||||
state; proposal/result transactions and live Redis restart/failover gates
|
||||
remain. The matcher package now performs bounded candidate formation and
|
||||
delegates the final proposal claim to the durable transaction boundary;
|
||||
ranked metadata/provider wiring and a long-running worker role remain.
|
||||
queue tickets now also retain server-derived probe RTT metadata for
|
||||
authoritative matcher reads; ranked metadata/provider wiring and live
|
||||
Redis repair remain.
|
||||
- [ ] **IN PROGRESS:** Run the Go control plane against PostgreSQL/Redis with
|
||||
independently runnable API, matcher, allocator and maintenance roles. The
|
||||
`cmd/control-plane` API role now opens PostgreSQL, applies migrations, wires
|
||||
|
||||
+2
-2
@@ -1173,7 +1173,7 @@ the local/CI/community transport, not a silent production fallback.
|
||||
| 8.2 `[D:8.1]` | **DONE.** Encode the launch SLOs from `docs/MATCHMAKING.md`: RTT, allocation/connect latency, 99.9% allocation/result success, API latency and tick health | [`docs/MATCHMAKING-SLOs.md`](docs/MATCHMAKING-SLOs.md) defines each metric, denominator, percentile/window, owner, alert threshold and release evidence |
|
||||
| 8.3 `[D:8.1]` | **DONE.** Publish versioned OpenAPI + WebSocket contracts for Steam login/session, profile/rating, queue create/heartbeat/cancel/resume, proposal accept/decline, assignment/status, server registration/roster/result/shutdown | [`server/contracts/v1/`](server/contracts/v1/) contains machine-readable REST/events contracts and dependency-free structural tests; REST resync is specified by the contract; `server/api/service.go` also exposes the documented `/api/v1` route names (including server-assigned idempotent queue ticket IDs and DELETE cancellation) alongside the existing client `/v1` routes, covered by `TestDocumentedContractRoutesAdaptToServiceAPI` |
|
||||
| 8.4 `[D:8.3]` | **DONE.** Define opaque IDs, legal queue/match state transitions, revisions and idempotency keys | [`server/contracts/v1/state-transitions.json`](server/contracts/v1/state-transitions.json) locks terminal states, legal edges, stale-revision handling and same-key replay/conflict behavior; contract tests cover the invariants |
|
||||
| 8.5 `[D:8.4]` | **IN PROGRESS.** Initial PostgreSQL migration now defines durable idempotency keys, queue ownership/active-participation fencing, identities, sessions/revocations, ranked seasons, ratings/events, matches/participants, penalties, results, audits and outbox | `server/migrations/0001_initial.sql`, `migrations/runner.go`, `cmd/migrate` and static checks cover the durable tables, uniqueness/check constraints, Redis-as-cache boundary and serialized forward migration recording; opt-in `scripts/run_postgres_integration.sh` now runs the runner and real queue/assignment ownership, idempotency, revision and expiry checks through pgx; rollback/down migration, the remaining serializable adapters and cache-loss repair remain |
|
||||
| 8.5 `[D:8.4]` | **IN PROGRESS.** Initial PostgreSQL migration now defines durable idempotency keys, queue ownership/active-participation fencing, identities, sessions/revocations, ranked seasons, ratings/events, matches/participants, penalties, results, audits and outbox; follow-up migration persists server-derived queue probe RTT metadata | `server/migrations/0001_initial.sql`, `0003_queue_probe_metadata.sql`, `migrations/runner.go`, `cmd/migrate` and static checks cover the durable tables, uniqueness/check constraints, Redis-as-cache boundary and serialized forward migration recording; opt-in `scripts/run_postgres_integration.sh` now runs the runner and real queue/assignment ownership, idempotency, revision and expiry checks through pgx; rollback/down migration, the remaining serializable adapters and cache-loss repair remain |
|
||||
| 8.6 `[D:8.3,8.4]` | **IN PROGRESS.** Add allocated-mode `ServerConfig` compatibility fields as opt-in defaults | `ServerConfig` now validates allocation mode, match/server IDs, playlist version, client build, future assignment expiry, image digest, transport and EU/NA region; `server_boot.gd` fails closed for the not-yet-wired Steam SDR transport, constrains allocated processes to one match, and emits allocation identity/transport in `server_started`; signed-authorisation admission, dynamic endpoint wiring and full manifest/runtime tests remain |
|
||||
|
||||
#### 8B — Authentication and secure control plane
|
||||
@@ -1193,7 +1193,7 @@ the local/CI/community transport, not a silent production fallback.
|
||||
| # | Task | Acceptance |
|
||||
|---|---|---|
|
||||
| 8.14 `[D:8.4,8.5,8.8]` | **IN PROGRESS.** Pure Go queue domain enforces one active ticket per verified player under concurrent mutation, 10 s heartbeat/30 s expiry, retry-safe create/heartbeat/cancel, owner-only recovery reads and deterministic candidate projection; store layer adds a rebuildable candidate-cache boundary and authenticated HTTP queue adapter with playlist/build/protocol compatibility metadata | `server/domain/queue.go`, `server/store/candidates.go`, `server/store/queue_sql.go`, `server/store/redis_candidates.go` and `server/api/service.go` cover ownership/expiry/idempotency, concurrent create fencing, candidate/player ownership binding, owner/revision-scoped SQL heartbeat/cancel/recovery, injectable PostgreSQL queue backend selected by the HTTP service, authoritative queue-to-cache rebuild, expired recovery as a terminal error, owner-scoped SQL recovery with authoritative expiry handling, server-owned candidate resolution, strict compatibility metadata, TTL-bound Redis upsert/remove/snapshot, API create/heartbeat/cancel projection hooks, optional control-plane Redis configuration and atomic durable-source repair on partial/malformed cache state; opt-in real PostgreSQL execution now covers create/replay/active-player fencing, owner recovery, revision-fenced heartbeat/cancel and expiry, while miniredis covers Redis behavior and repair-source failure; live Redis restart/failover and worker integration remain |
|
||||
| 8.15 `[D:7.8,8.3]` | **IN PROGRESS.** Pure Go probe validation treats Steam location as opaque, requires nonce/freshness/region and server-computed RTT, and implements discrepancy quarantine/release; authenticated HTTP now accepts only opaque location/nonce input through a server-owned probe provider | `server/domain/probes.go`, adversarial fixtures and `server/api/service.go` cover stale/wrong/forged evidence, the 25 ms/30% threshold, three-sample quarantine, five-clean release, authenticated provider arguments and rejection of client RTT fields; Steam coordinator and regional probe adapters remain |
|
||||
| 8.15 `[D:7.8,8.3]` | **IN PROGRESS.** Pure Go probe validation treats Steam location as opaque, requires nonce/freshness/region and server-computed RTT, and implements discrepancy quarantine/release; authenticated HTTP now accepts only opaque location/nonce input through a server-owned probe provider; durable queue projections now have a server-derived RTT JSON field for matcher reads | `server/domain/probes.go`, `server/migrations/0003_queue_probe_metadata.sql`, adversarial fixtures and `server/api/service.go`/`store/queue_sql.go` cover stale/wrong/forged evidence, the 25 ms/30% threshold, three-sample quarantine, five-clean release, authenticated provider arguments, rejection of client RTT fields, playlist-scoped candidate reads and bounded metadata decoding; Steam coordinator, regional probe adapters and API-to-queue probe population remain |
|
||||
| 8.16 `[D:8.14,8.15]` | **IN PROGRESS.** Pure Go candidate/team selection implements the <=100 ms ceiling, pairwise widening tolerance, anchor inclusion, deterministic set/region scoring and balanced team partitioning; queue-backed formation now consumes the server-owned projection, fences duplicate player identities and rejects playlist/build/protocol mixing; the matcher worker performs bounded formation and delegates the final claim to the durable proposal transaction | `server/domain/matcher.go`, `teams.go`, `server/matcher/worker.go` and adversarial fixtures cover no-common-region, tolerance boundaries, lexical ties, mean-rating balance, malformed candidates, duplicate identities, compatibility mismatches, incomplete batches, source failure, durable claim failure and queue-backed oldest-anchor formation; ranked provider and long-running worker integration remain |
|
||||
| 8.17 `[D:8.14,8.16]` | **IN PROGRESS.** Pure Go proposal policy sends a 10-second response window to every selected human, requires unanimous acceptance, applies exact decline/timeout cooldowns and ranked escalation; authenticated API exposes revisioned accept/decline mutations; formed matches now pass through a playlist-aware proposal boundary | `server/domain/proposal.go`, `formation.go` and `server/api/service.go` plus adversarial fixtures cover partial/unanimous response, expiry, replay/conflict, stale API revision, casual lineup preparation and ranked metadata validation; queue precedence and allocation integration remain |
|
||||
| 8.18 `[D:8.5,8.14,8.17]` | **IN PROGRESS.** Go store layer defines PostgreSQL SERIALIZABLE whole-transaction retries and queue candidate/proposal claim SQL using `FOR UPDATE SKIP LOCKED` plus durable uniqueness/revision fences; proposal creation now inserts proposal/participants and promotes every ticket in one rollback-safe transaction with player- and playlist-bound claim predicates, queue creation has a durable idempotency/owner-read adapter, participant-scoped proposal recovery now expires OPEN proposals and pending participants transactionally at read time, and proposal accept/decline now uses participant/proposal locks, revision fencing and durable idempotency; response attempts also advance expired proposals and pending participants before returning closed; runnable casual matcher polling now reads an authoritative PostgreSQL candidate batch and delegates its final claim to this transaction | `server/store/serializable.go`, `queue_sql.go`, `proposal_sql.go`, `proposal_recovery_sql.go`, `server/matcher/worker.go` and tests cover retry classification, claim-boundary invariants, player/ticket/playlist mapping, durable queue replay/conflict, owner-scoped queue/proposal recovery, expiry at read and mutation boundaries, response replay/conflict, stale revisions, zero-row claim aborts, atomic statement ordering, incomplete matcher batches and source/claim failures; opt-in PostgreSQL execution now covers queue create/replay/fencing, assignment persistence, proposal claim/promotion, participant recovery, unanimous response and rollback of partial claims; ranked provider, Redis-backed worker repair, worker-failure and concurrent two-matcher integration tests remain |
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
-- Persist only server-derived placement metadata alongside queue ownership.
|
||||
-- Redis remains a rebuildable index; this JSON projection is durable source
|
||||
-- data and may be empty until the authenticated probe completes.
|
||||
ALTER TABLE queue_tickets
|
||||
ADD COLUMN predicted_rtt JSONB NOT NULL DEFAULT '{}'::jsonb;
|
||||
+36
-19
@@ -22,23 +22,23 @@ FROM idempotency_keys
|
||||
WHERE scope = $1 AND idempotency_key = $2
|
||||
FOR UPDATE`
|
||||
QueueTicketSelectSQL = `SELECT ticket_id, player_id, playlist, state, client_build,
|
||||
protocol_version, enqueued_at, expires_at, revision
|
||||
protocol_version, enqueued_at, expires_at, revision, predicted_rtt
|
||||
FROM queue_tickets
|
||||
WHERE ticket_id = $1 AND player_id = $2`
|
||||
QueueTicketHeartbeatSQL = `UPDATE queue_tickets SET revision = revision + 1,
|
||||
expires_at = $4 + INTERVAL '30 seconds'
|
||||
WHERE ticket_id = $1 AND player_id = $2 AND revision = $3
|
||||
AND state IN ('QUEUED', 'PROPOSED') AND expires_at > $4
|
||||
RETURNING ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at, revision`
|
||||
RETURNING ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at, revision, predicted_rtt`
|
||||
QueueTicketCancelSQL = `UPDATE queue_tickets SET state = 'CANCELLED',
|
||||
revision = revision + 1, expires_at = $4
|
||||
WHERE ticket_id = $1 AND player_id = $2 AND revision = $3
|
||||
AND state NOT IN ('COMPLETED', 'CANCELLED', 'EXPIRED', 'FAILED')
|
||||
RETURNING ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at, revision`
|
||||
RETURNING ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at, revision, predicted_rtt`
|
||||
)
|
||||
|
||||
const QueueCandidateProjectionSQL = `SELECT ticket_id, player_id, playlist, client_build,
|
||||
protocol_version, enqueued_at
|
||||
protocol_version, enqueued_at, predicted_rtt
|
||||
FROM queue_tickets
|
||||
WHERE state = 'QUEUED' AND playlist = $1 AND expires_at > $2
|
||||
ORDER BY enqueued_at, ticket_id
|
||||
@@ -60,9 +60,13 @@ func ListQueuedCandidates(ctx context.Context, db *sql.DB, playlist domain.Playl
|
||||
for rows.Next() {
|
||||
var candidate domain.Candidate
|
||||
var playlist string
|
||||
if err := rows.Scan(&candidate.TicketID, &candidate.PlayerID, &playlist, &candidate.ClientBuild, &candidate.ProtocolVersion, &candidate.EnqueuedAt); err != nil {
|
||||
var predictedRTT []byte
|
||||
if err := rows.Scan(&candidate.TicketID, &candidate.PlayerID, &playlist, &candidate.ClientBuild, &candidate.ProtocolVersion, &candidate.EnqueuedAt, &predictedRTT); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(predictedRTT, &candidate.PredictedRTT); err != nil {
|
||||
return nil, fmt.Errorf("decode candidate RTT: %w", err)
|
||||
}
|
||||
candidate.Playlist = domain.Playlist(playlist)
|
||||
candidates = append(candidates, candidate)
|
||||
}
|
||||
@@ -108,22 +112,27 @@ func CreateQueueTicket(ctx context.Context, db *sql.DB, ticketID, playerID, idem
|
||||
ticket = queueTicketRecordToDomain(prior)
|
||||
return nil
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, QueueTicketInsertSQL, ticketID, playerID, string(spec.Playlist), string(domain.Queued), spec.ClientBuild, spec.ProtocolVersion, now, ticket.ExpiresAt)
|
||||
predictedRTT, err := json.Marshal(candidate.PredictedRTT)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, QueueTicketInsertSQL, ticketID, playerID, string(spec.Playlist), string(domain.Queued), spec.ClientBuild, spec.ProtocolVersion, now, ticket.ExpiresAt, predictedRTT)
|
||||
return err
|
||||
})
|
||||
return ticket, err
|
||||
}
|
||||
|
||||
type queueTicketRecord struct {
|
||||
TicketID string `json:"ticket_id"`
|
||||
PlayerID string `json:"player_id"`
|
||||
Playlist string `json:"playlist"`
|
||||
State string `json:"state"`
|
||||
ClientBuild string `json:"client_build"`
|
||||
ProtocolVersion int `json:"protocol_version"`
|
||||
EnqueuedAt time.Time `json:"enqueued_at"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
Revision uint64 `json:"revision"`
|
||||
TicketID string `json:"ticket_id"`
|
||||
PlayerID string `json:"player_id"`
|
||||
Playlist string `json:"playlist"`
|
||||
State string `json:"state"`
|
||||
ClientBuild string `json:"client_build"`
|
||||
ProtocolVersion int `json:"protocol_version"`
|
||||
EnqueuedAt time.Time `json:"enqueued_at"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
Revision uint64 `json:"revision"`
|
||||
PredictedRTT map[string]float64 `json:"predicted_rtt"`
|
||||
}
|
||||
|
||||
type PostgresQueue struct{ DB *sql.DB }
|
||||
@@ -146,9 +155,13 @@ func GetQueueTicket(ctx context.Context, db *sql.DB, playerID, ticketID string,
|
||||
return domain.QueueTicket{}, fmt.Errorf("invalid queue recovery arguments")
|
||||
}
|
||||
var record queueTicketRecord
|
||||
if err := db.QueryRowContext(ctx, QueueTicketSelectSQL, ticketID, playerID).Scan(&record.TicketID, &record.PlayerID, &record.Playlist, &record.State, &record.ClientBuild, &record.ProtocolVersion, &record.EnqueuedAt, &record.ExpiresAt, &record.Revision); err != nil {
|
||||
var predictedRTT []byte
|
||||
if err := db.QueryRowContext(ctx, QueueTicketSelectSQL, ticketID, playerID).Scan(&record.TicketID, &record.PlayerID, &record.Playlist, &record.State, &record.ClientBuild, &record.ProtocolVersion, &record.EnqueuedAt, &record.ExpiresAt, &record.Revision, &predictedRTT); err != nil {
|
||||
return domain.QueueTicket{}, err
|
||||
}
|
||||
if err := json.Unmarshal(predictedRTT, &record.PredictedRTT); err != nil {
|
||||
return domain.QueueTicket{}, fmt.Errorf("decode queue RTT: %w", err)
|
||||
}
|
||||
ticket := queueTicketRecordToDomain(record)
|
||||
if (ticket.State == domain.Queued || ticket.State == domain.Proposed) && !now.Before(ticket.ExpiresAt) {
|
||||
return domain.QueueTicket{}, domain.ErrTicketExpired
|
||||
@@ -194,9 +207,13 @@ func mutateQueueTicket(ctx context.Context, db *sql.DB, playerID, ticketID, idem
|
||||
return nil
|
||||
}
|
||||
var record queueTicketRecord
|
||||
if err := tx.QueryRowContext(ctx, mutationSQL, ticketID, playerID, expectedRevision, now).Scan(&record.TicketID, &record.PlayerID, &record.Playlist, &record.State, &record.ClientBuild, &record.ProtocolVersion, &record.EnqueuedAt, &record.ExpiresAt, &record.Revision); err != nil {
|
||||
var predictedRTT []byte
|
||||
if err := tx.QueryRowContext(ctx, mutationSQL, ticketID, playerID, expectedRevision, now).Scan(&record.TicketID, &record.PlayerID, &record.Playlist, &record.State, &record.ClientBuild, &record.ProtocolVersion, &record.EnqueuedAt, &record.ExpiresAt, &record.Revision, &predictedRTT); err != nil {
|
||||
return fmt.Errorf("queue mutation rejected: %w", err)
|
||||
}
|
||||
if err := json.Unmarshal(predictedRTT, &record.PredictedRTT); err != nil {
|
||||
return fmt.Errorf("decode queue RTT: %w", err)
|
||||
}
|
||||
ticket = queueTicketRecordToDomain(record)
|
||||
stored, err := json.Marshal(record)
|
||||
if err != nil {
|
||||
@@ -209,9 +226,9 @@ func mutateQueueTicket(ctx context.Context, db *sql.DB, playerID, ticketID, idem
|
||||
}
|
||||
|
||||
func queueTicketRecordFromDomain(ticket domain.QueueTicket) queueTicketRecord {
|
||||
return queueTicketRecord{ticket.TicketID, ticket.PlayerID, string(ticket.Playlist), string(ticket.State), ticket.Candidate.ClientBuild, ticket.Candidate.ProtocolVersion, ticket.EnqueuedAt, ticket.ExpiresAt, ticket.Revision}
|
||||
return queueTicketRecord{TicketID: ticket.TicketID, PlayerID: ticket.PlayerID, Playlist: string(ticket.Playlist), State: string(ticket.State), ClientBuild: ticket.Candidate.ClientBuild, ProtocolVersion: ticket.Candidate.ProtocolVersion, EnqueuedAt: ticket.EnqueuedAt, ExpiresAt: ticket.ExpiresAt, Revision: ticket.Revision, PredictedRTT: ticket.Candidate.PredictedRTT}
|
||||
}
|
||||
func queueTicketRecordToDomain(record queueTicketRecord) domain.QueueTicket {
|
||||
candidate := domain.Candidate{TicketID: record.TicketID, PlayerID: record.PlayerID, Playlist: domain.Playlist(record.Playlist), ClientBuild: record.ClientBuild, ProtocolVersion: record.ProtocolVersion, EnqueuedAt: record.EnqueuedAt}
|
||||
candidate := domain.Candidate{TicketID: record.TicketID, PlayerID: record.PlayerID, Playlist: domain.Playlist(record.Playlist), ClientBuild: record.ClientBuild, ProtocolVersion: record.ProtocolVersion, EnqueuedAt: record.EnqueuedAt, PredictedRTT: record.PredictedRTT}
|
||||
return domain.QueueTicket{TicketID: record.TicketID, PlayerID: record.PlayerID, Candidate: candidate, Playlist: domain.Playlist(record.Playlist), State: domain.State(record.State), Revision: record.Revision, EnqueuedAt: record.EnqueuedAt, ExpiresAt: record.ExpiresAt}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ func TestQueueSQLUsesDurableIdempotencyAndOwnerScopedRecovery(t *testing.T) {
|
||||
QueueTicketInsertSQL: {"player_id", "playlist", "client_build", "protocol_version"},
|
||||
QueueTicketHeartbeatSQL: {"player_id = $2", "revision = $3", "expires_at > $4", "RETURNING"},
|
||||
QueueTicketCancelSQL: {"player_id = $2", "revision = $3", "state NOT IN", "RETURNING"},
|
||||
QueueCandidateProjectionSQL: {"playlist = $1", "predicted_rtt", "expires_at > $2", "LIMIT $3"},
|
||||
} {
|
||||
for _, fragment := range fragments {
|
||||
if !contains(query, fragment) {
|
||||
@@ -23,6 +24,21 @@ func TestQueueSQLUsesDurableIdempotencyAndOwnerScopedRecovery(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestListQueuedCandidatesRejectsUnscopedOrUnboundedReads(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
for _, playlist := range []domain.Playlist{"", "invalid"} {
|
||||
if _, err := ListQueuedCandidates(nil, nil, playlist, now, 4); err == nil {
|
||||
t.Fatalf("playlist %q accepted", playlist)
|
||||
}
|
||||
}
|
||||
if _, err := ListQueuedCandidates(nil, nil, domain.Casual, now, 0); err == nil {
|
||||
t.Fatal("zero limit accepted")
|
||||
}
|
||||
if _, err := ListQueuedCandidates(nil, nil, domain.Casual, now, 1001); err == nil {
|
||||
t.Fatal("unbounded limit accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueMutationAdaptersRejectInvalidArgumentsWithoutDatabase(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
if _, err := HeartbeatQueueTicket(nil, nil, "player-1", "ticket-1", "short", 0, now); err == nil {
|
||||
|
||||
@@ -62,8 +62,8 @@ var (
|
||||
// QueueTicketInsertSQL relies on the partial unique index in migration 0001
|
||||
// as the cross-replica one-active-ticket fence.
|
||||
QueueTicketInsertSQL = `INSERT INTO queue_tickets
|
||||
(ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at)
|
||||
VALUES ($1, $2, $3, 'QUEUED', $4, $5, $6, $7)`
|
||||
(ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at, predicted_rtt)
|
||||
VALUES ($1, $2, $3, 'QUEUED', $4, $5, $6, $7, $8)`
|
||||
|
||||
// CandidateClaimSQL must run in the same serializable transaction as
|
||||
// ProposalParticipantInsertSQL. SKIP LOCKED lets another matcher continue,
|
||||
|
||||
Reference in New Issue
Block a user