feat: repair matcher candidates through redis projection

This commit is contained in:
Josh Creek
2026-09-01 10:11:54 +01:00
parent 7a3d520608
commit d882469c79
5 changed files with 82 additions and 3 deletions
+4 -2
View File
@@ -135,8 +135,10 @@ product policy are in [`docs/MATCHMAKING.md`](docs/MATCHMAKING.md).
- [ ] **IN PROGRESS:** Fence proposals/participants in a PostgreSQL serializable transaction; - [ ] **IN PROGRESS:** Fence proposals/participants in a PostgreSQL serializable transaction;
prove loss of an acknowledged Redis write cannot split players. The Go store prove loss of an acknowledged Redis write cannot split players. The Go store
adapter now performs proposal insertion, participant insertion, and every adapter now performs proposal insertion, participant insertion, and every
queue-ticket promotion in one rollback-safe SERIALIZABLE callback; live DB/ queue-ticket promotion in one rollback-safe SERIALIZABLE callback; the runnable
Redis failover testing remains. casual matcher can optionally use a Redis candidate projection and repairs an
empty/lost index from PostgreSQL before claiming durably; live DB/Redis failover
testing remains.
- [ ] Casual: target 3v3 humans, after 60 s allow >=2 humans (one/team) plus - [ ] Casual: target 3v3 humans, after 60 s allow >=2 humans (one/team) plus
bots, kickoff-only human backfill and no backfill loss/decline penalty. bots, kickoff-only human backfill and no backfill loss/decline penalty.
- [ ] **IN PROGRESS:** Ranked: exactly six humans, solo-only, no bots/backfill, - [ ] **IN PROGRESS:** Ranked: exactly six humans, solo-only, no bots/backfill,
+1 -1
View File
@@ -1196,7 +1196,7 @@ the local/CI/community transport, not a silent production fallback.
| 8.15 `[D:7.8,8.3]` | **IN PROGRESS.** Pure Go probe validation treats Steam location as opaque, requires nonce/freshness/region and server-computed RTT, and implements discrepancy quarantine/release; authenticated HTTP now accepts only opaque location/nonce input through a server-owned probe provider and records validated RTT into the active player's durable queue ticket; durable queue projections have a server-derived RTT JSON field for matcher reads | `server/domain/probes.go`, `server/migrations/0003_queue_probe_metadata.sql`, adversarial fixtures and `server/api/service.go`/`store/queue_sql.go` cover stale/wrong/forged evidence, the 25 ms/30% threshold, three-sample quarantine, five-clean release, authenticated provider arguments, rejection of client RTT fields, player/ticket/expiry binding, persistence failure, playlist-scoped candidate reads and bounded metadata decoding; Steam coordinator, regional probe adapters and multi-region probe population remain | | 8.15 `[D:7.8,8.3]` | **IN PROGRESS.** Pure Go probe validation treats Steam location as opaque, requires nonce/freshness/region and server-computed RTT, and implements discrepancy quarantine/release; authenticated HTTP now accepts only opaque location/nonce input through a server-owned probe provider and records validated RTT into the active player's durable queue ticket; durable queue projections have a server-derived RTT JSON field for matcher reads | `server/domain/probes.go`, `server/migrations/0003_queue_probe_metadata.sql`, adversarial fixtures and `server/api/service.go`/`store/queue_sql.go` cover stale/wrong/forged evidence, the 25 ms/30% threshold, three-sample quarantine, five-clean release, authenticated provider arguments, rejection of client RTT fields, player/ticket/expiry binding, persistence failure, playlist-scoped candidate reads and bounded metadata decoding; Steam coordinator, regional probe adapters and multi-region probe population remain |
| 8.16 `[D:8.14,8.15]` | **IN PROGRESS.** Pure Go candidate/team selection implements the <=100 ms ceiling, pairwise widening tolerance, anchor inclusion, deterministic set/region scoring and balanced team partitioning; queue-backed formation now consumes the server-owned projection, fences duplicate player identities and rejects playlist/build/protocol mixing; the matcher worker performs bounded formation and delegates the final claim to the durable proposal transaction | `server/domain/matcher.go`, `teams.go`, `server/matcher/worker.go` and adversarial fixtures cover no-common-region, tolerance boundaries, lexical ties, mean-rating balance, malformed candidates, duplicate identities, compatibility mismatches, incomplete batches, source failure, durable claim failure and queue-backed oldest-anchor formation; ranked provider and long-running worker integration remain | | 8.16 `[D:8.14,8.15]` | **IN PROGRESS.** Pure Go candidate/team selection implements the <=100 ms ceiling, pairwise widening tolerance, anchor inclusion, deterministic set/region scoring and balanced team partitioning; queue-backed formation now consumes the server-owned projection, fences duplicate player identities and rejects playlist/build/protocol mixing; the matcher worker performs bounded formation and delegates the final claim to the durable proposal transaction | `server/domain/matcher.go`, `teams.go`, `server/matcher/worker.go` and adversarial fixtures cover no-common-region, tolerance boundaries, lexical ties, mean-rating balance, malformed candidates, duplicate identities, compatibility mismatches, incomplete batches, source failure, durable claim failure and queue-backed oldest-anchor formation; ranked provider and long-running worker integration remain |
| 8.17 `[D:8.14,8.16]` | **IN PROGRESS.** Pure Go proposal policy sends a 10-second response window to every selected human, requires unanimous acceptance, applies exact decline/timeout cooldowns and ranked escalation; authenticated API exposes revisioned accept/decline mutations; formed matches now pass through a playlist-aware proposal boundary | `server/domain/proposal.go`, `formation.go` and `server/api/service.go` plus adversarial fixtures cover partial/unanimous response, expiry, replay/conflict, stale API revision, casual lineup preparation and ranked metadata validation; queue precedence and allocation integration remain | | 8.17 `[D:8.14,8.16]` | **IN PROGRESS.** Pure Go proposal policy sends a 10-second response window to every selected human, requires unanimous acceptance, applies exact decline/timeout cooldowns and ranked escalation; authenticated API exposes revisioned accept/decline mutations; formed matches now pass through a playlist-aware proposal boundary | `server/domain/proposal.go`, `formation.go` and `server/api/service.go` plus adversarial fixtures cover partial/unanimous response, expiry, replay/conflict, stale API revision, casual lineup preparation and ranked metadata validation; queue precedence and allocation integration remain |
| 8.18 `[D:8.5,8.14,8.17]` | **IN PROGRESS.** Go store layer defines PostgreSQL SERIALIZABLE whole-transaction retries and queue candidate/proposal claim SQL using `FOR UPDATE SKIP LOCKED` plus durable uniqueness/revision fences; proposal creation now inserts proposal/participants and promotes every ticket in one rollback-safe transaction with player- and playlist-bound claim predicates, queue creation has a durable idempotency/owner-read adapter, participant-scoped proposal recovery now expires OPEN proposals and pending participants transactionally at read time, and proposal accept/decline now uses participant/proposal locks, revision fencing and durable idempotency; response attempts also advance expired proposals and pending participants before returning closed; runnable casual matcher polling now reads an authoritative PostgreSQL candidate batch and delegates its final claim to this transaction | `server/store/serializable.go`, `queue_sql.go`, `proposal_sql.go`, `proposal_recovery_sql.go`, `server/matcher/worker.go` and tests cover retry classification, claim-boundary invariants, player/ticket/playlist mapping, durable queue replay/conflict, owner-scoped queue/proposal recovery, expiry at read and mutation boundaries, response replay/conflict, stale revisions, zero-row claim aborts, atomic statement ordering, incomplete matcher batches and source/claim failures; opt-in PostgreSQL execution now covers queue create/replay/fencing, assignment persistence, proposal claim/promotion, participant recovery, unanimous response and rollback of partial claims; ranked provider, Redis-backed worker repair, worker-failure and concurrent two-matcher integration tests remain | | 8.18 `[D:8.5,8.14,8.17]` | **IN PROGRESS.** Go store layer defines PostgreSQL SERIALIZABLE whole-transaction retries and queue candidate/proposal claim SQL using `FOR UPDATE SKIP LOCKED` plus durable uniqueness/revision fences; proposal creation now inserts proposal/participants and promotes every ticket in one rollback-safe transaction with player- and playlist-bound claim predicates, queue creation has a durable idempotency/owner-read adapter, participant-scoped proposal recovery now expires OPEN proposals and pending participants transactionally at read time, and proposal accept/decline now uses participant/proposal locks, revision fencing and durable idempotency; response attempts also advance expired proposals and pending participants before returning closed; runnable casual matcher polling now supports an optional Redis candidate projection that repairs empty/lost cache state from authoritative PostgreSQL before the durable final claim | `server/store/serializable.go`, `queue_sql.go`, `proposal_sql.go`, `proposal_recovery_sql.go`, `redis_candidates.go`, `server/matcher/worker.go` and tests cover retry classification, claim-boundary invariants, player/ticket/playlist mapping, durable queue replay/conflict, owner-scoped queue/proposal recovery, expiry at read and mutation boundaries, response replay/conflict, stale revisions, zero-row claim aborts, atomic statement ordering, incomplete matcher batches, source failures and empty-index Redis repair; opt-in PostgreSQL execution now covers queue create/replay/fencing, assignment persistence, proposal claim/promotion, participant recovery, unanimous response and rollback of partial claims; ranked provider, worker-failure and concurrent two-matcher integration tests remain |
| 8.19 `[D:8.18]` | **IN PROGRESS.** Pure Go casual lineup requires 26 humans with at least one per team, fills missing slots with explicit bots, permits kickoff-only bot-slot backfill and assigns no backfill penalty/rating update; proposal preparation now derives the lineup from formed teams | `server/domain/casual.go`, `formation.go` cover both-team minimum, bot shape, live-play rejection, zero-penalty backfill and casual proposal composition; queue candidate selection, opt-in 10 s backfill proposals, reconnect/leave penalties and live integration remain | | 8.19 `[D:8.18]` | **IN PROGRESS.** Pure Go casual lineup requires 26 humans with at least one per team, fills missing slots with explicit bots, permits kickoff-only bot-slot backfill and assigns no backfill penalty/rating update; proposal preparation now derives the lineup from formed teams | `server/domain/casual.go`, `formation.go` cover both-team minimum, bot shape, live-play rejection, zero-penalty backfill and casual proposal composition; queue candidate selection, opt-in 10 s backfill proposals, reconnect/leave penalties and live integration remain |
| 8.20 `[D:8.18]` | **IN PROGRESS.** Pure Go ranked admission requires six unique verified solo humans, rejects bots/backfill/parties, and allows only random-enabled non-elevated arenas; proposal preparation requires matching metadata for every formed player | `server/domain/ranked.go`, `formation.go` cover count, identity, party, bot/backfill, arena eligibility and formed-player metadata rejection; `ArenaRegistry` integration, allocation wiring and innocent-ticket restoration remain | | 8.20 `[D:8.18]` | **IN PROGRESS.** Pure Go ranked admission requires six unique verified solo humans, rejects bots/backfill/parties, and allows only random-enabled non-elevated arenas; proposal preparation requires matching metadata for every formed player | `server/domain/ranked.go`, `formation.go` cover count, identity, party, bot/backfill, arena eligibility and formed-player metadata rejection; `ArenaRegistry` integration, allocation wiring and innocent-ticket restoration remain |
| 8.21 `[D:8.5,8.20]` | **IN PROGRESS.** Pure Go rating core implements canonical Glicko-2, daily inactivity, ranked 1/3 and casual 1/N human-opponent weights, deterministic opponent ordering, and authoritative draw/overtime/abandon scoring; certified result completion now applies per-player updates inside the durable transaction with lexical row locks and revision increments | `server/domain/rating.go`, `server/store/result_sql.go` and tests cover canonical/inactivity/weight/invalid-input, draw/OT/abandon, ordered participant snapshots, lock/value re-read and rating update SQL; live PostgreSQL rating, seasons and concurrent result transaction tests remain | | 8.21 `[D:8.5,8.20]` | **IN PROGRESS.** Pure Go rating core implements canonical Glicko-2, daily inactivity, ranked 1/3 and casual 1/N human-opponent weights, deterministic opponent ordering, and authoritative draw/overtime/abandon scoring; certified result completion now applies per-player updates inside the durable transaction with lexical row locks and revision increments | `server/domain/rating.go`, `server/store/result_sql.go` and tests cover canonical/inactivity/weight/invalid-input, draw/OT/abandon, ordered participant snapshots, lock/value re-read and rating update SQL; live PostgreSQL rating, seasons and concurrent result transaction tests remain |
+43
View File
@@ -16,6 +16,7 @@ import (
"github.com/cosmic-clash/cosmic-clash/server/migrations" "github.com/cosmic-clash/cosmic-clash/server/migrations"
"github.com/cosmic-clash/cosmic-clash/server/store" "github.com/cosmic-clash/cosmic-clash/server/store"
_ "github.com/jackc/pgx/v5/stdlib" _ "github.com/jackc/pgx/v5/stdlib"
"github.com/redis/go-redis/v9"
) )
func main() { func main() {
@@ -24,6 +25,9 @@ func main() {
playlist := flag.String("playlist", string(domain.Casual), "playlist to match; ranked requires a provider-enabled role") playlist := flag.String("playlist", string(domain.Casual), "playlist to match; ranked requires a provider-enabled role")
size := flag.Int("size", 4, "players per match") size := flag.Int("size", 4, "players per match")
interval := flag.Duration("interval", time.Second, "poll interval") interval := flag.Duration("interval", time.Second, "poll interval")
redisAddr := flag.String("redis-addr", os.Getenv("COSMIC_CLASH_REDIS_ADDR"), "optional Redis candidate projection address")
redisPrefix := flag.String("redis-prefix", envOrDefault("COSMIC_CLASH_REDIS_PREFIX", "cosmic-clash"), "Redis key prefix")
redisTTL := flag.Duration("redis-ttl", 60*time.Second, "TTL for transient candidate projection entries")
flag.Parse() flag.Parse()
if *dsn == "" { if *dsn == "" {
fatalf("--dsn or COSMIC_CLASH_POSTGRES_DSN is required") fatalf("--dsn or COSMIC_CLASH_POSTGRES_DSN is required")
@@ -31,6 +35,9 @@ func main() {
if *playlist != string(domain.Casual) { if *playlist != string(domain.Casual) {
fatalf("unsupported playlist %q; only casual is currently enabled", *playlist) fatalf("unsupported playlist %q; only casual is currently enabled", *playlist)
} }
if *redisTTL <= 0 {
fatalf("--redis-ttl must be positive")
}
db, err := sql.Open("pgx", *dsn) db, err := sql.Open("pgx", *dsn)
if err != nil { if err != nil {
fatalf("open PostgreSQL: %v", err) fatalf("open PostgreSQL: %v", err)
@@ -45,8 +52,37 @@ func main() {
fatalf("apply migrations: %v", err) fatalf("apply migrations: %v", err)
} }
now := func() time.Time { return time.Now().UTC() } now := func() time.Time { return time.Now().UTC() }
var redisClient *redis.Client
var projection *store.CandidateProjection
if *redisAddr != "" {
redisClient = redis.NewClient(&redis.Options{Addr: *redisAddr})
defer redisClient.Close()
candidateProjection := store.CandidateProjection{
Index: store.RedisCandidateIndex{Client: redisClient, Prefix: *redisPrefix, TTL: *redisTTL},
Source: func(ctx context.Context, at time.Time) ([]domain.Candidate, error) {
return store.ListQueuedCandidates(ctx, db, domain.Casual, at, 1000)
},
}
projection = &candidateProjection
}
worker := matcher.Worker{ worker := matcher.Worker{
Source: func(ctx context.Context, at time.Time, playlist domain.Playlist, limit int) ([]domain.Candidate, error) { Source: func(ctx context.Context, at time.Time, playlist domain.Playlist, limit int) ([]domain.Candidate, error) {
if projection != nil {
candidates, err := projection.Snapshot(ctx, at)
if err != nil {
return nil, err
}
if len(candidates) > limit {
candidates = candidates[:limit]
}
filtered := make([]domain.Candidate, 0, len(candidates))
for _, candidate := range candidates {
if candidate.Playlist == playlist {
filtered = append(filtered, candidate)
}
}
return filtered, nil
}
return store.ListQueuedCandidates(ctx, db, playlist, at, limit) return store.ListQueuedCandidates(ctx, db, playlist, at, limit)
}, },
Creator: matcher.ProposalCreatorFunc(func(ctx context.Context, proposal domain.Proposal, ticketIDs map[string]string, at time.Time) error { Creator: matcher.ProposalCreatorFunc(func(ctx context.Context, proposal domain.Proposal, ticketIDs map[string]string, at time.Time) error {
@@ -65,6 +101,13 @@ func main() {
} }
} }
func envOrDefault(name, fallback string) string {
if value := os.Getenv(name); value != "" {
return value
}
return fallback
}
func fatalf(format string, args ...any) { func fatalf(format string, args ...any) {
log.Printf("matcher: "+format, args...) log.Printf("matcher: "+format, args...)
os.Exit(1) os.Exit(1)
+25
View File
@@ -46,3 +46,28 @@ func TestCandidateProjectionDoesNotReturnCacheWhenRepairSourceFails(t *testing.T
t.Fatal("cache projection succeeded without a usable Redis/index source") t.Fatal("cache projection succeeded without a usable Redis/index source")
} }
} }
func TestCandidateProjectionRepairsEmptyIndexFromDurableSource(t *testing.T) {
mini, err := miniredis.Run()
if err != nil {
t.Fatal(err)
}
defer mini.Close()
client := redis.NewClient(&redis.Options{Addr: mini.Addr()})
defer client.Close()
now := time.Unix(1000, 0).UTC()
candidate := domain.Candidate{TicketID: "miss-ticket", PlayerID: "miss-player", EnqueuedAt: now}
projection := CandidateProjection{
Index: RedisCandidateIndex{Client: client, Prefix: "miss", TTL: time.Minute},
Source: func(context.Context, time.Time) ([]domain.Candidate, error) {
return []domain.Candidate{candidate}, nil
},
}
got, err := projection.Snapshot(context.Background(), now)
if err != nil {
t.Fatal(err)
}
if len(got) != 1 || got[0].TicketID != candidate.TicketID {
t.Fatalf("empty-index repair = %+v", got)
}
}
+9
View File
@@ -49,6 +49,15 @@ func (p CandidateProjection) Snapshot(ctx context.Context, now time.Time) ([]dom
} }
candidates, err := p.Index.Snapshot(ctx, now) candidates, err := p.Index.Snapshot(ctx, now)
if err == nil { if err == nil {
// An empty index is indistinguishable from a Redis restart or a lost
// keyspace. Rebuild from PostgreSQL before returning so queued players
// are not hidden until the next enqueue mutation.
if len(candidates) == 0 {
if err := p.Repair(ctx, now); err != nil {
return nil, err
}
return p.Index.Snapshot(ctx, now)
}
return candidates, nil return candidates, nil
} }
if err := p.Repair(ctx, now); err != nil { if err := p.Repair(ctx, now); err != nil {