feat: persist authenticated queue probe RTT

This commit is contained in:
Josh Creek
2026-09-01 09:40:25 +01:00
parent 25cc182793
commit def60169a8
7 changed files with 131 additions and 2 deletions
+4 -1
View File
@@ -49,7 +49,10 @@ product policy are in [`docs/MATCHMAKING.md`](docs/MATCHMAKING.md).
delegates the final proposal claim to the durable transaction boundary; delegates the final proposal claim to the durable transaction boundary;
queue tickets now also retain server-derived probe RTT metadata for queue tickets now also retain server-derived probe RTT metadata for
authoritative matcher reads; ranked metadata/provider wiring and live authoritative matcher reads; ranked metadata/provider wiring and live
Redis repair remain. Redis repair remain. The authenticated probe API now records validated
server-computed RTT values into the active player's durable queue ticket and
fails closed when that write is unavailable; Steam/coordinator evidence
acquisition and multi-region probe population remain.
- [ ] **IN PROGRESS:** Run the Go control plane against PostgreSQL/Redis with - [ ] **IN PROGRESS:** Run the Go control plane against PostgreSQL/Redis with
independently runnable API, matcher, allocator and maintenance roles. The independently runnable API, matcher, allocator and maintenance roles. The
`cmd/control-plane` API role now opens PostgreSQL, applies migrations, wires `cmd/control-plane` API role now opens PostgreSQL, applies migrations, wires
+1 -1
View File
@@ -1193,7 +1193,7 @@ the local/CI/community transport, not a silent production fallback.
| # | Task | Acceptance | | # | Task | Acceptance |
|---|---|---| |---|---|---|
| 8.14 `[D:8.4,8.5,8.8]` | **IN PROGRESS.** Pure Go queue domain enforces one active ticket per verified player under concurrent mutation, 10 s heartbeat/30 s expiry, retry-safe create/heartbeat/cancel, owner-only recovery reads and deterministic candidate projection; store layer adds a rebuildable candidate-cache boundary and authenticated HTTP queue adapter with playlist/build/protocol compatibility metadata | `server/domain/queue.go`, `server/store/candidates.go`, `server/store/queue_sql.go`, `server/store/redis_candidates.go` and `server/api/service.go` cover ownership/expiry/idempotency, concurrent create fencing, candidate/player ownership binding, owner/revision-scoped SQL heartbeat/cancel/recovery, injectable PostgreSQL queue backend selected by the HTTP service, authoritative queue-to-cache rebuild, expired recovery as a terminal error, owner-scoped SQL recovery with authoritative expiry handling, server-owned candidate resolution, strict compatibility metadata, TTL-bound Redis upsert/remove/snapshot, API create/heartbeat/cancel projection hooks, optional control-plane Redis configuration and atomic durable-source repair on partial/malformed cache state; opt-in real PostgreSQL execution now covers create/replay/active-player fencing, owner recovery, revision-fenced heartbeat/cancel and expiry, while miniredis covers Redis behavior and repair-source failure; live Redis restart/failover and worker integration remain | | 8.14 `[D:8.4,8.5,8.8]` | **IN PROGRESS.** Pure Go queue domain enforces one active ticket per verified player under concurrent mutation, 10 s heartbeat/30 s expiry, retry-safe create/heartbeat/cancel, owner-only recovery reads and deterministic candidate projection; store layer adds a rebuildable candidate-cache boundary and authenticated HTTP queue adapter with playlist/build/protocol compatibility metadata | `server/domain/queue.go`, `server/store/candidates.go`, `server/store/queue_sql.go`, `server/store/redis_candidates.go` and `server/api/service.go` cover ownership/expiry/idempotency, concurrent create fencing, candidate/player ownership binding, owner/revision-scoped SQL heartbeat/cancel/recovery, injectable PostgreSQL queue backend selected by the HTTP service, authoritative queue-to-cache rebuild, expired recovery as a terminal error, owner-scoped SQL recovery with authoritative expiry handling, server-owned candidate resolution, strict compatibility metadata, TTL-bound Redis upsert/remove/snapshot, API create/heartbeat/cancel projection hooks, optional control-plane Redis configuration and atomic durable-source repair on partial/malformed cache state; opt-in real PostgreSQL execution now covers create/replay/active-player fencing, owner recovery, revision-fenced heartbeat/cancel and expiry, while miniredis covers Redis behavior and repair-source failure; live Redis restart/failover and worker integration remain |
| 8.15 `[D:7.8,8.3]` | **IN PROGRESS.** Pure Go probe validation treats Steam location as opaque, requires nonce/freshness/region and server-computed RTT, and implements discrepancy quarantine/release; authenticated HTTP now accepts only opaque location/nonce input through a server-owned probe provider; durable queue projections now have a server-derived RTT JSON field for matcher reads | `server/domain/probes.go`, `server/migrations/0003_queue_probe_metadata.sql`, adversarial fixtures and `server/api/service.go`/`store/queue_sql.go` cover stale/wrong/forged evidence, the 25 ms/30% threshold, three-sample quarantine, five-clean release, authenticated provider arguments, rejection of client RTT fields, playlist-scoped candidate reads and bounded metadata decoding; Steam coordinator, regional probe adapters and API-to-queue probe population remain | | 8.15 `[D:7.8,8.3]` | **IN PROGRESS.** Pure Go probe validation treats Steam location as opaque, requires nonce/freshness/region and server-computed RTT, and implements discrepancy quarantine/release; authenticated HTTP now accepts only opaque location/nonce input through a server-owned probe provider and records validated RTT into the active player's durable queue ticket; durable queue projections have a server-derived RTT JSON field for matcher reads | `server/domain/probes.go`, `server/migrations/0003_queue_probe_metadata.sql`, adversarial fixtures and `server/api/service.go`/`store/queue_sql.go` cover stale/wrong/forged evidence, the 25 ms/30% threshold, three-sample quarantine, five-clean release, authenticated provider arguments, rejection of client RTT fields, player/ticket/expiry binding, persistence failure, playlist-scoped candidate reads and bounded metadata decoding; Steam coordinator, regional probe adapters and multi-region probe population remain |
| 8.16 `[D:8.14,8.15]` | **IN PROGRESS.** Pure Go candidate/team selection implements the <=100 ms ceiling, pairwise widening tolerance, anchor inclusion, deterministic set/region scoring and balanced team partitioning; queue-backed formation now consumes the server-owned projection, fences duplicate player identities and rejects playlist/build/protocol mixing; the matcher worker performs bounded formation and delegates the final claim to the durable proposal transaction | `server/domain/matcher.go`, `teams.go`, `server/matcher/worker.go` and adversarial fixtures cover no-common-region, tolerance boundaries, lexical ties, mean-rating balance, malformed candidates, duplicate identities, compatibility mismatches, incomplete batches, source failure, durable claim failure and queue-backed oldest-anchor formation; ranked provider and long-running worker integration remain | | 8.16 `[D:8.14,8.15]` | **IN PROGRESS.** Pure Go candidate/team selection implements the <=100 ms ceiling, pairwise widening tolerance, anchor inclusion, deterministic set/region scoring and balanced team partitioning; queue-backed formation now consumes the server-owned projection, fences duplicate player identities and rejects playlist/build/protocol mixing; the matcher worker performs bounded formation and delegates the final claim to the durable proposal transaction | `server/domain/matcher.go`, `teams.go`, `server/matcher/worker.go` and adversarial fixtures cover no-common-region, tolerance boundaries, lexical ties, mean-rating balance, malformed candidates, duplicate identities, compatibility mismatches, incomplete batches, source failure, durable claim failure and queue-backed oldest-anchor formation; ranked provider and long-running worker integration remain |
| 8.17 `[D:8.14,8.16]` | **IN PROGRESS.** Pure Go proposal policy sends a 10-second response window to every selected human, requires unanimous acceptance, applies exact decline/timeout cooldowns and ranked escalation; authenticated API exposes revisioned accept/decline mutations; formed matches now pass through a playlist-aware proposal boundary | `server/domain/proposal.go`, `formation.go` and `server/api/service.go` plus adversarial fixtures cover partial/unanimous response, expiry, replay/conflict, stale API revision, casual lineup preparation and ranked metadata validation; queue precedence and allocation integration remain | | 8.17 `[D:8.14,8.16]` | **IN PROGRESS.** Pure Go proposal policy sends a 10-second response window to every selected human, requires unanimous acceptance, applies exact decline/timeout cooldowns and ranked escalation; authenticated API exposes revisioned accept/decline mutations; formed matches now pass through a playlist-aware proposal boundary | `server/domain/proposal.go`, `formation.go` and `server/api/service.go` plus adversarial fixtures cover partial/unanimous response, expiry, replay/conflict, stale API revision, casual lineup preparation and ranked metadata validation; queue precedence and allocation integration remain |
| 8.18 `[D:8.5,8.14,8.17]` | **IN PROGRESS.** Go store layer defines PostgreSQL SERIALIZABLE whole-transaction retries and queue candidate/proposal claim SQL using `FOR UPDATE SKIP LOCKED` plus durable uniqueness/revision fences; proposal creation now inserts proposal/participants and promotes every ticket in one rollback-safe transaction with player- and playlist-bound claim predicates, queue creation has a durable idempotency/owner-read adapter, participant-scoped proposal recovery now expires OPEN proposals and pending participants transactionally at read time, and proposal accept/decline now uses participant/proposal locks, revision fencing and durable idempotency; response attempts also advance expired proposals and pending participants before returning closed; runnable casual matcher polling now reads an authoritative PostgreSQL candidate batch and delegates its final claim to this transaction | `server/store/serializable.go`, `queue_sql.go`, `proposal_sql.go`, `proposal_recovery_sql.go`, `server/matcher/worker.go` and tests cover retry classification, claim-boundary invariants, player/ticket/playlist mapping, durable queue replay/conflict, owner-scoped queue/proposal recovery, expiry at read and mutation boundaries, response replay/conflict, stale revisions, zero-row claim aborts, atomic statement ordering, incomplete matcher batches and source/claim failures; opt-in PostgreSQL execution now covers queue create/replay/fencing, assignment persistence, proposal claim/promotion, participant recovery, unanimous response and rollback of partial claims; ranked provider, Redis-backed worker repair, worker-failure and concurrent two-matcher integration tests remain | | 8.18 `[D:8.5,8.14,8.17]` | **IN PROGRESS.** Go store layer defines PostgreSQL SERIALIZABLE whole-transaction retries and queue candidate/proposal claim SQL using `FOR UPDATE SKIP LOCKED` plus durable uniqueness/revision fences; proposal creation now inserts proposal/participants and promotes every ticket in one rollback-safe transaction with player- and playlist-bound claim predicates, queue creation has a durable idempotency/owner-read adapter, participant-scoped proposal recovery now expires OPEN proposals and pending participants transactionally at read time, and proposal accept/decline now uses participant/proposal locks, revision fencing and durable idempotency; response attempts also advance expired proposals and pending participants before returning closed; runnable casual matcher polling now reads an authoritative PostgreSQL candidate batch and delegates its final claim to this transaction | `server/store/serializable.go`, `queue_sql.go`, `proposal_sql.go`, `proposal_recovery_sql.go`, `server/matcher/worker.go` and tests cover retry classification, claim-boundary invariants, player/ticket/playlist mapping, durable queue replay/conflict, owner-scoped queue/proposal recovery, expiry at read and mutation boundaries, response replay/conflict, stale revisions, zero-row claim aborts, atomic statement ordering, incomplete matcher batches and source/claim failures; opt-in PostgreSQL execution now covers queue create/replay/fencing, assignment persistence, proposal claim/promotion, participant recovery, unanimous response and rollback of partial claims; ranked provider, Redis-backed worker repair, worker-failure and concurrent two-matcher integration tests remain |
+10
View File
@@ -26,6 +26,9 @@ const maxBodyBytes = 8 << 10
type CandidateProvider func(playerID, ticketID string) (domain.Candidate, error) type CandidateProvider func(playerID, ticketID string) (domain.Candidate, error)
type CandidateProviderV2 func(playerID, ticketID string, spec domain.QueueSpec) (domain.Candidate, error) type CandidateProviderV2 func(playerID, ticketID string, spec domain.QueueSpec) (domain.Candidate, error)
type ProbeProvider func(playerID, region string, opaqueLocation, nonce []byte, receivedAt time.Time) (domain.ProbeEvidence, []byte, error) type ProbeProvider func(playerID, region string, opaqueLocation, nonce []byte, receivedAt time.Time) (domain.ProbeEvidence, []byte, error)
type ProbeRecorder interface {
RecordProbe(context.Context, string, string, time.Duration, time.Time) error
}
type QueueBackend interface { type QueueBackend interface {
Create(context.Context, string, string, string, domain.QueueSpec, time.Time) (domain.QueueTicket, error) Create(context.Context, string, string, string, domain.QueueSpec, time.Time) (domain.QueueTicket, error)
@@ -86,6 +89,7 @@ type Service struct {
QueueBackend QueueBackend QueueBackend QueueBackend
CandidateIndex CandidateIndex CandidateIndex CandidateIndex
Probe ProbeProvider Probe ProbeProvider
ProbeRecorder ProbeRecorder
Assignment AssignmentProvider Assignment AssignmentProvider
Now func() time.Time Now func() time.Time
Proposals map[string]*domain.Proposal Proposals map[string]*domain.Proposal
@@ -653,6 +657,12 @@ func (s *Service) probe(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusUnprocessableEntity, "invalid_probe") writeError(w, http.StatusUnprocessableEntity, "invalid_probe")
return return
} }
if s.ProbeRecorder != nil {
if err := s.ProbeRecorder.RecordProbe(r.Context(), playerID, region, evidence.ServerRTT, receivedAt); err != nil {
writeError(w, http.StatusServiceUnavailable, "probe_persistence_failed")
return
}
}
writeJSON(w, http.StatusAccepted, map[string]any{"region": region, "server_rtt_ms": evidence.ServerRTT.Milliseconds(), "status": "accepted"}) writeJSON(w, http.StatusAccepted, map[string]any{"region": region, "server_rtt_ms": evidence.ServerRTT.Milliseconds(), "status": "accepted"})
} }
+45
View File
@@ -25,6 +25,21 @@ type candidateIndexSpy struct {
last domain.Candidate last domain.Candidate
} }
type probeRecorderSpy struct {
calls int
err error
last struct {
player, region string
rtt time.Duration
}
}
func (p *probeRecorderSpy) RecordProbe(_ context.Context, player, region string, rtt time.Duration, _ time.Time) error {
p.calls++
p.last.player, p.last.region, p.last.rtt = player, region, rtt
return p.err
}
func (i *candidateIndexSpy) Upsert(_ context.Context, candidate domain.Candidate) error { func (i *candidateIndexSpy) Upsert(_ context.Context, candidate domain.Candidate) error {
i.upsertCalls++ i.upsertCalls++
i.last = candidate i.last = candidate
@@ -936,6 +951,36 @@ func TestProbeAPIUsesServerEvidenceAndRejectsClientRTTField(t *testing.T) {
_ = response.Body.Close() _ = response.Body.Close()
} }
func TestProbeAPIRecordsOnlyValidatedServerEvidence(t *testing.T) {
now := time.Unix(1000, 0).UTC()
sessions := domain.NewSessionStore()
session, token, _ := sessions.Issue("player-a", time.Hour, now)
recorder := &probeRecorderSpy{}
service := &Service{Sessions: sessions, Now: func() time.Time { return now }, ProbeRecorder: recorder, Probe: func(_ string, region string, location, nonce []byte, _ time.Time) (domain.ProbeEvidence, []byte, error) {
return domain.ProbeEvidence{OpaqueLocation: location, Nonce: nonce, IssuedAt: now, Region: region, ServerRTT: 37 * time.Millisecond}, nonce, nil
}}
server := httptest.NewServer(service.Handler())
defer server.Close()
req, _ := http.NewRequest(http.MethodPost, server.URL+"/v1/probes/NA", strings.NewReader(`{"opaque_location":"b3BhcXVl","nonce":"bm9uY2U="}`))
req.Header.Set("Authorization", "Bearer "+session.SessionID+":"+token)
response, err := http.DefaultClient.Do(req)
if err != nil || response.StatusCode != http.StatusAccepted {
t.Fatalf("status=%v err=%v", response.StatusCode, err)
}
response.Body.Close()
if recorder.calls != 1 || recorder.last.player != "player-a" || recorder.last.region != "NA" || recorder.last.rtt != 37*time.Millisecond {
t.Fatalf("recorded probe=%+v calls=%d", recorder.last, recorder.calls)
}
recorder.err = errors.New("database unavailable")
req, _ = http.NewRequest(http.MethodPost, server.URL+"/v1/probes/NA", strings.NewReader(`{"opaque_location":"b3BhcXVl","nonce":"bm9uY2U="}`))
req.Header.Set("Authorization", "Bearer "+session.SessionID+":"+token)
response, err = http.DefaultClient.Do(req)
if err != nil || response.StatusCode != http.StatusServiceUnavailable {
t.Fatalf("persistence status=%v err=%v", response.StatusCode, err)
}
response.Body.Close()
}
func TestProposalRecoveryIsParticipantScopedAndExpiresAtReadBoundary(t *testing.T) { func TestProposalRecoveryIsParticipantScopedAndExpiresAtReadBoundary(t *testing.T) {
now := time.Unix(1000, 0).UTC() now := time.Unix(1000, 0).UTC()
sessions := domain.NewSessionStore() sessions := domain.NewSessionStore()
+28
View File
@@ -163,6 +163,34 @@ func (q *Queue) Expire(now time.Time) []QueueTicket {
return q.expireLocked(now) return q.expireLocked(now)
} }
// RecordProbe stores server-computed RTT metadata on the player's active
// ticket. It never accepts client-provided latency and refuses expired or
// non-queueable tickets.
func (q *Queue) RecordProbe(playerID, region string, rtt time.Duration, now time.Time) error {
if playerID == "" || (region != "EU" && region != "NA") || rtt < 0 || now.IsZero() {
return fmt.Errorf("invalid probe recording")
}
q.mu.Lock()
defer q.mu.Unlock()
ticketID, ok := q.byPlayer[playerID]
if !ok {
return ErrTicketNotFound
}
ticket, ok := q.tickets[ticketID]
if !ok || (ticket.State != Queued && ticket.State != Proposed) {
return ErrTicketNotFound
}
if !now.Before(ticket.ExpiresAt) {
return ErrTicketExpired
}
if ticket.Candidate.PredictedRTT == nil {
ticket.Candidate.PredictedRTT = make(map[string]float64)
}
ticket.Candidate.PredictedRTT[region] = float64(rtt) / float64(time.Millisecond)
q.tickets[ticketID] = ticket
return nil
}
func (q *Queue) expireLocked(now time.Time) []QueueTicket { func (q *Queue) expireLocked(now time.Time) []QueueTicket {
var expired []QueueTicket var expired []QueueTicket
for id, ticket := range q.tickets { for id, ticket := range q.tickets {
+21
View File
@@ -136,3 +136,24 @@ func TestQueueConcurrentCreateKeepsOneActiveTicketPerPlayer(t *testing.T) {
t.Fatalf("concurrent creates succeeded %d times", succeeded) t.Fatalf("concurrent creates succeeded %d times", succeeded)
} }
} }
func TestQueueRecordProbeBindsServerRTTToActivePlayerTicket(t *testing.T) {
now := time.Unix(1000, 0).UTC()
queue := NewQueue()
if _, err := queue.Create("player-a", "ticket-a", "create-key-123456", Candidate{PlayerID: "player-a", TicketID: "ticket-a", Playlist: Casual, EnqueuedAt: now}, now); err != nil {
t.Fatal(err)
}
if err := queue.RecordProbe("player-a", "EU", 42*time.Millisecond, now); err != nil {
t.Fatal(err)
}
ticket, err := queue.Get("player-a", "ticket-a", now)
if err != nil || ticket.Candidate.PredictedRTT["EU"] != 42 {
t.Fatalf("ticket=%+v err=%v", ticket, err)
}
if err := queue.RecordProbe("player-a", "NA", time.Millisecond, now.Add(QueueExpiryWindow)); err != ErrTicketExpired {
t.Fatalf("expired record err=%v", err)
}
if err := queue.RecordProbe("player-other", "EU", time.Millisecond, now); err != ErrTicketNotFound {
t.Fatalf("unknown player err=%v", err)
}
}
+22
View File
@@ -137,6 +137,28 @@ type queueTicketRecord struct {
type PostgresQueue struct{ DB *sql.DB } type PostgresQueue struct{ DB *sql.DB }
const QueueProbeRecordSQL = `UPDATE queue_tickets
SET predicted_rtt = jsonb_set(COALESCE(predicted_rtt, '{}'::jsonb), ARRAY[$2], to_jsonb($3::double precision), true)
WHERE player_id = $1 AND state IN ('QUEUED', 'PROPOSED') AND expires_at > $4`
func (q PostgresQueue) RecordProbe(ctx context.Context, playerID, region string, rtt time.Duration, now time.Time) error {
if q.DB == nil || playerID == "" || (region != "EU" && region != "NA") || rtt < 0 || now.IsZero() {
return fmt.Errorf("invalid probe recording")
}
result, err := q.DB.ExecContext(ctx, QueueProbeRecordSQL, playerID, region, float64(rtt)/float64(time.Millisecond), now)
if err != nil {
return err
}
changed, err := result.RowsAffected()
if err != nil {
return err
}
if changed == 0 {
return domain.ErrTicketNotFound
}
return nil
}
func (q PostgresQueue) Create(ctx context.Context, playerID, ticketID, idempotencyKey string, spec domain.QueueSpec, now time.Time) (domain.QueueTicket, error) { func (q PostgresQueue) Create(ctx context.Context, playerID, ticketID, idempotencyKey string, spec domain.QueueSpec, now time.Time) (domain.QueueTicket, error) {
return CreateQueueTicket(ctx, q.DB, ticketID, playerID, idempotencyKey, spec, now) return CreateQueueTicket(ctx, q.DB, ticketID, playerID, idempotencyKey, spec, now)
} }