From c4c2ada1f6dd1d16a347833513aead8ce197f43c Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:23:22 +0100 Subject: [PATCH] test(multiplayer): add api load gate --- .github/workflows/multiplayer-load.yml | 22 +++++ Makefile | 5 +- multiplayer-next.md | 2 +- server/api/load_test.go | 119 +++++++++++++++++++++++++ 4 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/multiplayer-load.yml create mode 100644 server/api/load_test.go diff --git a/.github/workflows/multiplayer-load.yml b/.github/workflows/multiplayer-load.yml new file mode 100644 index 00000000..ffc6eca4 --- /dev/null +++ b/.github/workflows/multiplayer-load.yml @@ -0,0 +1,22 @@ +name: Multiplayer API Load + +on: + workflow_dispatch: + pull_request: + paths: + - server/api/** + - server/domain/** + - Makefile + - .github/workflows/multiplayer-load.yml + +permissions: + contents: read + +jobs: + api-load: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - name: Verify 10,000-client API load boundary + run: make verify-multiplayer-load diff --git a/Makefile b/Makefile index 79adc44b..022a5ed8 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,11 @@ -.PHONY: verify-phase6 verify-enet-integration verify-steam-templates verify-supply-chain verify-kind-agones verify-allocated-compose verify-multiplayer-local +.PHONY: verify-phase6 verify-enet-integration verify-steam-templates verify-supply-chain verify-kind-agones verify-allocated-compose verify-multiplayer-local verify-multiplayer-load verify-multiplayer-local: bash scripts/verify_multiplayer_local.sh +verify-multiplayer-load: + (cd server && go test -tags load ./api -run TestQueueCreateHTTPLoad -count=1) + verify-phase6: bash scripts/verify_phase6.sh diff --git a/multiplayer-next.md b/multiplayer-next.md index 6fb6d632..02dbdc78 100644 --- a/multiplayer-next.md +++ b/multiplayer-next.md @@ -1250,7 +1250,7 @@ the local/CI/community transport, not a silent production fallback. | 8.48 `[D:8.10,8.14,8.17,8.18,8.27,8.31,8.35,8.47]` | **IN PROGRESS.** Offline testkit exercises verified queue projection → ranked six-player proposal → ENet allocation → assignment-ready manifest → certified durable result receipt; `compose.allocated-smoke.yml` independently runs the real testkit API, matcher, allocator, Agones-shaped provider, PostgreSQL, and game-server supervisor with a generated signed roster, verifying queue/proposal/allocation binding, authenticated result, idempotent retry, shutdown acknowledgment, durable receipt/audit rows, and SIGTERM-driven game-process drain | `.github/workflows/allocated-compose.yml` runs `make verify-allocated-compose`. Live Docker evidence from this workspace and legacy fixture non-regression remain open | | 8.49 `[D:8.25,8.26,8.28,8.29,8.30,8.31,8.35,8.36]` | **IN PROGRESS.** `scripts/verify_kind_agones.sh` creates a disposable kind cluster, installs pinned Agones, loads the real `game-server` image, applies the Fleet in an explicitly separate Agones-only supervisor/UDP readiness mode, and verifies readiness plus allocation of a dynamic UDP endpoint; `.github/workflows/agones-integration.yml` runs it for infrastructure changes and on demand | The cloud-free runner is committed and fails clearly when Docker/kind/Helm are unavailable. CI/live evidence for production control-plane registration, roster/no-show, both readiness stages, races, multi-match node, result-pending reconciliation, drain, and rollback remains open | | 8.50 `[D:8.25,8.37,8.43,8.49]` | Network/chaos suite: 100 ms RTT, jitter/loss, client/API/matcher restart, game-pod death, node drain, Redis failover and control-plane loss | System recovers to a defined state; infrastructure-caused cases cannot penalise affected players | -| 8.51 `[D:8.17,8.18,8.30,8.31,8.45]` | Load test >=10,000 queued clients, >=100 proposals/s and forecast launch concurrency x2 | API p95 <=250 ms, durable matcher fence holds, both readiness/allocation SLOs are met and replicas scale without duplicate claims | +| 8.51 `[D:8.17,8.18,8.30,8.31,8.45]` | **IN PROGRESS.** The opt-in `make verify-multiplayer-load` gate drives 10,000 real HTTP queue-create requests through the service with 256 in flight and records p95/p99; the handler and in-process ownership boundary are exercised without weakening normal tests | Local API load passes at p95 <250 ms in normal and race runs; PostgreSQL saturation, >=100 proposals/s, durable matcher fencing under load, forecast launch concurrency x2, and replica scaling remain live infrastructure gates | | 8.52 `[D:8.32,8.34,8.45,8.51]` | Per-region cost model from measured density, warm capacity, bandwidth, DB/Redis and telemetry; add budgets and allocation quotas | Cost per completed match and forecast monthly bands are recorded; a denial-of-wallet test triggers limits/alerts before budget breach | | 8.53 `[D:7.8,8.13,8.38,8.45,8.46,8.48,8.49,8.50,8.51,8.52]` | Progressive release: development → internal → casual canary → casual → provisional ranked → ranked | Each promotion requires SLO/security/cost gates, rollback rehearsal, EU+NA playtests and unchanged legacy gates; rollback criteria and owner are explicit | diff --git a/server/api/load_test.go b/server/api/load_test.go new file mode 100644 index 00000000..a2b736a2 --- /dev/null +++ b/server/api/load_test.go @@ -0,0 +1,119 @@ +//go:build load + +package api + +import ( + "bytes" + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "sort" + "strconv" + "sync" + "testing" + "time" + + "github.com/cosmic-clash/cosmic-clash/server/domain" +) + +// TestQueueCreateHTTPLoad is the bounded, repeatable API portion of §8.51. +// It deliberately uses the real HTTP handler and in-process queue boundary; +// database/replica capacity and matcher throughput remain separate gates. +func TestQueueCreateHTTPLoad(t *testing.T) { + clients := loadInt(t, "COSMIC_CLASH_LOAD_CLIENTS", 10000) + concurrency := loadInt(t, "COSMIC_CLASH_LOAD_CONCURRENCY", 256) + p95Limit := time.Duration(loadInt(t, "COSMIC_CLASH_LOAD_P95_MS", 250)) * time.Millisecond + if clients < 1 || clients > 100000 || concurrency < 1 || concurrency > clients || p95Limit <= 0 || p95Limit > 10*time.Second { + t.Fatalf("invalid load configuration clients=%d concurrency=%d p95=%s", clients, concurrency, p95Limit) + } + now := time.Unix(1_000_000, 0).UTC() + sessions := domain.NewSessionStore() + queue := domain.NewQueue() + service := &Service{ + Sessions: sessions, + Queue: queue, + Now: func() time.Time { return now }, + CandidateV2: func(playerID, ticketID string, spec domain.QueueSpec) (domain.Candidate, error) { + return domain.Candidate{ + PlayerID: playerID, TicketID: ticketID, Playlist: spec.Playlist, + ClientBuild: spec.ClientBuild, ProtocolVersion: spec.ProtocolVersion, + EnqueuedAt: now, PredictedRTT: map[string]float64{"EU": 20}, + }, nil + }, + } + httpServer := httptest.NewServer(service.Handler()) + defer httpServer.Close() + + tokens := make([]string, clients) + for i := range tokens { + session, token, err := sessions.Issue(fmt.Sprintf("load-player-%d", i), time.Hour, now) + if err != nil { + t.Fatalf("issue session %d: %v", i, err) + } + tokens[i] = session.SessionID + ":" + token + } + client := &http.Client{Transport: &http.Transport{MaxIdleConns: clients, MaxIdleConnsPerHost: clients}} + start := make(chan struct{}) + jobs := make(chan int) + durations := make([]time.Duration, clients) + statuses := make([]int, clients) + var wg sync.WaitGroup + for worker := 0; worker < concurrency; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + for index := range jobs { + started := time.Now() + body := fmt.Sprintf(`{"ticket_id":"load-ticket-%08d","playlist":"casual","client_build":"build-1","protocol_version":1}`, index) + request, err := http.NewRequestWithContext(context.Background(), http.MethodPost, httpServer.URL+"/v1/queue", bytes.NewBufferString(body)) + if err != nil { + continue + } + request.Header.Set("Authorization", "Bearer "+tokens[index]) + request.Header.Set("Idempotency-Key", fmt.Sprintf("load-create-key-%08d", index)) + request.Header.Set("Content-Type", "application/json") + response, err := client.Do(request) + if err == nil { + statuses[index] = response.StatusCode + response.Body.Close() + } + durations[index] = time.Since(started) + } + }() + } + close(start) + for i := 0; i < clients; i++ { + jobs <- i + } + close(jobs) + wg.Wait() + + ordered := append([]time.Duration(nil), durations...) + sort.Slice(ordered, func(i, j int) bool { return ordered[i] < ordered[j] }) + p95 := ordered[(len(ordered)*95+99)/100-1] + for i, status := range statuses { + if status != http.StatusCreated { + t.Fatalf("client %d returned HTTP %d; the load request must create a queue ticket", i, status) + } + } + if p95 > p95Limit { + t.Fatalf("queue-create HTTP p95=%s exceeds %s for %d clients at %d in-flight", p95, p95Limit, clients, concurrency) + } + t.Logf("queue-create load: clients=%d concurrency=%d p95=%s p99=%s", clients, concurrency, p95, ordered[(len(ordered)*99+99)/100-1]) +} + +func loadInt(t *testing.T, name string, fallback int) int { + t.Helper() + value := fallback + if raw := os.Getenv(name); raw != "" { + parsed, err := strconv.Atoi(raw) + if err != nil { + t.Fatalf("%s=%q is not an integer", name, raw) + } + value = parsed + } + return value +}