From bdc89303b404c58a4ffa5022887b16678702534b Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Tue, 1 Sep 2026 09:30:28 +0100 Subject: [PATCH] feat: add durable matcher worker orchestration --- multiplayer-next.md | 4 +- multiplayer-todo.md | 2 +- server/matcher/worker.go | 88 +++++++++++++++++++++++++++++++++++ server/matcher/worker_test.go | 77 ++++++++++++++++++++++++++++++ 4 files changed, 169 insertions(+), 2 deletions(-) create mode 100644 server/matcher/worker.go create mode 100644 server/matcher/worker_test.go diff --git a/multiplayer-next.md b/multiplayer-next.md index 596b0074..0175f5ba 100644 --- a/multiplayer-next.md +++ b/multiplayer-next.md @@ -45,7 +45,9 @@ product policy are in [`docs/MATCHMAKING.md`](docs/MATCHMAKING.md). application, and a TTL-bound Redis candidate index now supports atomic rebuild, snapshot and removal with durable-source repair on partial/malformed cache state; proposal/result transactions and live Redis restart/failover gates - remain. + remain. The matcher package now performs bounded candidate formation and + delegates the final proposal claim to the durable transaction boundary; + ranked metadata/provider wiring and a long-running worker role remain. - [ ] **IN PROGRESS:** Run the Go control plane against PostgreSQL/Redis with independently runnable API, matcher, allocator and maintenance roles. The `cmd/control-plane` API role now opens PostgreSQL, applies migrations, wires diff --git a/multiplayer-todo.md b/multiplayer-todo.md index 6d6ad130..1bde30e7 100644 --- a/multiplayer-todo.md +++ b/multiplayer-todo.md @@ -1194,7 +1194,7 @@ the local/CI/community transport, not a silent production fallback. |---|---|---| | 8.14 `[D:8.4,8.5,8.8]` | **IN PROGRESS.** Pure Go queue domain enforces one active ticket per verified player under concurrent mutation, 10 s heartbeat/30 s expiry, retry-safe create/heartbeat/cancel, owner-only recovery reads and deterministic candidate projection; store layer adds a rebuildable candidate-cache boundary and authenticated HTTP queue adapter with playlist/build/protocol compatibility metadata | `server/domain/queue.go`, `server/store/candidates.go`, `server/store/queue_sql.go`, `server/store/redis_candidates.go` and `server/api/service.go` cover ownership/expiry/idempotency, concurrent create fencing, candidate/player ownership binding, owner/revision-scoped SQL heartbeat/cancel/recovery, injectable PostgreSQL queue backend selected by the HTTP service, authoritative queue-to-cache rebuild, expired recovery as a terminal error, owner-scoped SQL recovery with authoritative expiry handling, server-owned candidate resolution, strict compatibility metadata, TTL-bound Redis upsert/remove/snapshot, API create/heartbeat/cancel projection hooks, optional control-plane Redis configuration and atomic durable-source repair on partial/malformed cache state; opt-in real PostgreSQL execution now covers create/replay/active-player fencing, owner recovery, revision-fenced heartbeat/cancel and expiry, while miniredis covers Redis behavior and repair-source failure; live Redis restart/failover and worker integration remain | | 8.15 `[D:7.8,8.3]` | **IN PROGRESS.** Pure Go probe validation treats Steam location as opaque, requires nonce/freshness/region and server-computed RTT, and implements discrepancy quarantine/release; authenticated HTTP now accepts only opaque location/nonce input through a server-owned probe provider | `server/domain/probes.go`, adversarial fixtures and `server/api/service.go` cover stale/wrong/forged evidence, the 25 ms/30% threshold, three-sample quarantine, five-clean release, authenticated provider arguments and rejection of client RTT fields; Steam coordinator and regional probe adapters remain | -| 8.16 `[D:8.14,8.15]` | **IN PROGRESS.** Pure Go candidate/team selection implements the <=100 ms ceiling, pairwise widening tolerance, anchor inclusion, deterministic set/region scoring and balanced team partitioning; queue-backed formation now consumes the server-owned projection, fences duplicate player identities and rejects playlist/build/protocol mixing | `server/domain/matcher.go`, `teams.go` and adversarial fixtures cover no-common-region, tolerance boundaries, lexical ties, mean-rating balance, malformed candidates, duplicate identities, compatibility mismatches and queue-backed oldest-anchor formation; full population fixtures and durable matcher claim integration remain | +| 8.16 `[D:8.14,8.15]` | **IN PROGRESS.** Pure Go candidate/team selection implements the <=100 ms ceiling, pairwise widening tolerance, anchor inclusion, deterministic set/region scoring and balanced team partitioning; queue-backed formation now consumes the server-owned projection, fences duplicate player identities and rejects playlist/build/protocol mixing; the matcher worker performs bounded formation and delegates the final claim to the durable proposal transaction | `server/domain/matcher.go`, `teams.go`, `server/matcher/worker.go` and adversarial fixtures cover no-common-region, tolerance boundaries, lexical ties, mean-rating balance, malformed candidates, duplicate identities, compatibility mismatches, incomplete batches, source failure, durable claim failure and queue-backed oldest-anchor formation; ranked provider and long-running worker integration remain | | 8.17 `[D:8.14,8.16]` | **IN PROGRESS.** Pure Go proposal policy sends a 10-second response window to every selected human, requires unanimous acceptance, applies exact decline/timeout cooldowns and ranked escalation; authenticated API exposes revisioned accept/decline mutations; formed matches now pass through a playlist-aware proposal boundary | `server/domain/proposal.go`, `formation.go` and `server/api/service.go` plus adversarial fixtures cover partial/unanimous response, expiry, replay/conflict, stale API revision, casual lineup preparation and ranked metadata validation; queue precedence and allocation integration remain | | 8.18 `[D:8.5,8.14,8.17]` | **IN PROGRESS.** Go store layer defines PostgreSQL SERIALIZABLE whole-transaction retries and queue candidate/proposal claim SQL using `FOR UPDATE SKIP LOCKED` plus durable uniqueness/revision fences; proposal creation now inserts proposal/participants and promotes every ticket in one rollback-safe transaction with player- and playlist-bound claim predicates, queue creation has a durable idempotency/owner-read adapter, participant-scoped proposal recovery now expires OPEN proposals and pending participants transactionally at read time, and proposal accept/decline now uses participant/proposal locks, revision fencing and durable idempotency; response attempts also advance expired proposals and pending participants before returning closed | `server/store/serializable.go`, `queue_sql.go`, `proposal_sql.go`, `proposal_recovery_sql.go` and tests cover retry classification, claim-boundary invariants, player/ticket/playlist mapping, durable queue replay/conflict, owner-scoped queue/proposal recovery, expiry at read and mutation boundaries, response replay/conflict, stale revisions, zero-row claim aborts and atomic statement ordering; opt-in PostgreSQL execution now covers queue create/replay/fencing, assignment persistence, proposal claim/promotion, participant recovery, unanimous response and rollback of partial claims; Redis candidate index/repair, worker-failure and concurrent two-matcher integration tests remain | | 8.19 `[D:8.18]` | **IN PROGRESS.** Pure Go casual lineup requires 2–6 humans with at least one per team, fills missing slots with explicit bots, permits kickoff-only bot-slot backfill and assigns no backfill penalty/rating update; proposal preparation now derives the lineup from formed teams | `server/domain/casual.go`, `formation.go` cover both-team minimum, bot shape, live-play rejection, zero-penalty backfill and casual proposal composition; queue candidate selection, opt-in 10 s backfill proposals, reconnect/leave penalties and live integration remain | diff --git a/server/matcher/worker.go b/server/matcher/worker.go new file mode 100644 index 00000000..5831880f --- /dev/null +++ b/server/matcher/worker.go @@ -0,0 +1,88 @@ +// Package matcher contains the provider-neutral orchestration around the +// durable proposal claim transaction. +package matcher + +import ( + "context" + "fmt" + "time" + + "github.com/cosmic-clash/cosmic-clash/server/domain" +) + +type CandidateSource func(context.Context, time.Time, int) ([]domain.Candidate, error) + +type ProposalCreator interface { + CreateProposal(context.Context, domain.Proposal, map[string]string, time.Time) error +} + +type ProposalCreatorFunc func(context.Context, domain.Proposal, map[string]string, time.Time) error + +func (f ProposalCreatorFunc) CreateProposal(ctx context.Context, proposal domain.Proposal, ticketIDs map[string]string, now time.Time) error { + return f(ctx, proposal, ticketIDs, now) +} + +type PrepareFunc func(string, domain.Playlist, domain.MatchFormation, time.Time) (domain.PreparedProposal, error) + +type Worker struct { + Source CandidateSource + Creator ProposalCreator + Playlist domain.Playlist + Size int + Now func() time.Time + NextID func() string + Prepare PrepareFunc +} + +// RunOnce performs one bounded matchmaking attempt. The source may be Redis +// backed, but the creator must be the durable transaction that claims tickets; +// a stale cache therefore fails safely and can be retried on the next pass. +func (w Worker) RunOnce(ctx context.Context) (bool, error) { + if w.Source == nil || w.Creator == nil || w.Now == nil || w.NextID == nil || w.Prepare == nil { + return false, fmt.Errorf("matcher worker is not configured") + } + if w.Playlist != domain.Casual && w.Playlist != domain.Ranked { + return false, fmt.Errorf("unsupported matcher playlist") + } + if w.Size < 2 || w.Size > 6 { + return false, fmt.Errorf("invalid matcher size") + } + now := w.Now() + candidates, err := w.Source(ctx, now, w.Size) + if err != nil { + return false, err + } + if len(candidates) < w.Size { + return false, nil + } + queue := domain.NewQueue() + for _, candidate := range candidates { + if _, err := queue.Create(candidate.PlayerID, candidate.TicketID, "matcher-"+candidate.TicketID, candidate, now); err != nil { + return false, err + } + } + formation, err := domain.FormFromQueue(queue, w.Size, now) + if err != nil { + return false, err + } + prepared, err := w.Prepare(w.NextID(), w.Playlist, formation, now) + if err != nil { + return false, err + } + ticketIDs := make(map[string]string, len(prepared.Proposal.Participants)) + for _, participant := range prepared.Proposal.Participants { + for _, candidate := range formation.Selection.Players { + if candidate.PlayerID == participant.PlayerID { + ticketIDs[participant.PlayerID] = candidate.TicketID + break + } + } + } + if len(ticketIDs) != len(prepared.Proposal.Participants) { + return false, fmt.Errorf("proposal participant is not in formed selection") + } + if err := w.Creator.CreateProposal(ctx, prepared.Proposal, ticketIDs, now); err != nil { + return false, err + } + return true, nil +} diff --git a/server/matcher/worker_test.go b/server/matcher/worker_test.go new file mode 100644 index 00000000..a4f6b432 --- /dev/null +++ b/server/matcher/worker_test.go @@ -0,0 +1,77 @@ +package matcher + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/cosmic-clash/cosmic-clash/server/domain" +) + +type creatorSpy struct { + calls int + err error + last domain.Proposal + ids map[string]string +} + +func (c *creatorSpy) CreateProposal(_ context.Context, proposal domain.Proposal, ids map[string]string, _ time.Time) error { + c.calls++ + c.last = proposal + c.ids = ids + return c.err +} + +func candidates() []domain.Candidate { + now := time.Unix(1000, 0).UTC() + result := make([]domain.Candidate, 4) + for i := range result { + result[i] = domain.Candidate{TicketID: "ticket-" + string(rune('1'+i)), PlayerID: "player-" + string(rune('1'+i)), Playlist: domain.Casual, EnqueuedAt: now.Add(time.Duration(i) * time.Second), PredictedRTT: map[string]float64{"EU": 20}} + } + return result +} + +func workerFor(source CandidateSource, creator ProposalCreator) Worker { + return Worker{Source: source, Creator: creator, Playlist: domain.Casual, Size: 4, Now: func() time.Time { return time.Unix(1000, 0).UTC() }, NextID: func() string { return "proposal-1234567890123456" }, Prepare: func(id string, playlist domain.Playlist, formation domain.MatchFormation, now time.Time) (domain.PreparedProposal, error) { + return domain.PrepareProposal(id, playlist, formation, nil, domain.RankedArena{}, now) + }} +} + +func TestRunOnceDelegatesFinalClaimAndBindsTickets(t *testing.T) { + creator := &creatorSpy{} + worker := workerFor(func(context.Context, time.Time, int) ([]domain.Candidate, error) { return candidates(), nil }, creator) + formed, err := worker.RunOnce(context.Background()) + if err != nil || !formed || creator.calls != 1 { + t.Fatalf("formed=%v err=%v calls=%d", formed, err, creator.calls) + } + if len(creator.ids) != 4 || creator.ids["player-1"] != "ticket-1" { + t.Fatalf("ticket bindings=%v", creator.ids) + } +} + +func TestRunOnceFailsClosedOnSourceOrDurableClaimFailure(t *testing.T) { + creator := &creatorSpy{err: errors.New("serialization conflict")} + worker := workerFor(func(context.Context, time.Time, int) ([]domain.Candidate, error) { + return nil, errors.New("redis unavailable") + }, creator) + if _, err := worker.RunOnce(context.Background()); err == nil { + t.Fatal("source failure was swallowed") + } + worker.Source = func(context.Context, time.Time, int) ([]domain.Candidate, error) { return candidates(), nil } + if _, err := worker.RunOnce(context.Background()); err == nil { + t.Fatal("durable claim failure was swallowed") + } + if creator.calls != 1 { + t.Fatalf("creator calls=%d", creator.calls) + } +} + +func TestRunOnceDoesNotClaimAnIncompleteBatch(t *testing.T) { + creator := &creatorSpy{} + worker := workerFor(func(context.Context, time.Time, int) ([]domain.Candidate, error) { return candidates()[:3], nil }, creator) + formed, err := worker.RunOnce(context.Background()) + if err != nil || formed || creator.calls != 0 { + t.Fatalf("formed=%v err=%v calls=%d", formed, err, creator.calls) + } +}