mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
feat: add durable allocator claim boundary
This commit is contained in:
Josh Creek
@@ -53,6 +53,9 @@ product policy are in [`docs/MATCHMAKING.md`](docs/MATCHMAKING.md).
|
||||
server-computed RTT values into the active player's durable queue ticket and
|
||||
fails closed when that write is unavailable; Steam/coordinator evidence
|
||||
acquisition and multi-region probe population remain.
|
||||
- [ ] **IN PROGRESS:** Durable allocator registry now records READY GameServer
|
||||
projections and atomically claims compatible capacity with replay/conflict
|
||||
fencing; provider allocation and assignment publication remain.
|
||||
- [ ] **IN PROGRESS:** Run the Go control plane against PostgreSQL/Redis with
|
||||
independently runnable API, matcher, allocator and maintenance roles. The
|
||||
`cmd/control-plane` API role now opens PostgreSQL, applies migrations, wires
|
||||
|
||||
+2
-2
@@ -1173,7 +1173,7 @@ the local/CI/community transport, not a silent production fallback.
|
||||
| 8.2 `[D:8.1]` | **DONE.** Encode the launch SLOs from `docs/MATCHMAKING.md`: RTT, allocation/connect latency, 99.9% allocation/result success, API latency and tick health | [`docs/MATCHMAKING-SLOs.md`](docs/MATCHMAKING-SLOs.md) defines each metric, denominator, percentile/window, owner, alert threshold and release evidence |
|
||||
| 8.3 `[D:8.1]` | **DONE.** Publish versioned OpenAPI + WebSocket contracts for Steam login/session, profile/rating, queue create/heartbeat/cancel/resume, proposal accept/decline, assignment/status, server registration/roster/result/shutdown | [`server/contracts/v1/`](server/contracts/v1/) contains machine-readable REST/events contracts and dependency-free structural tests; REST resync is specified by the contract; `server/api/service.go` also exposes the documented `/api/v1` route names (including server-assigned idempotent queue ticket IDs and DELETE cancellation) alongside the existing client `/v1` routes, covered by `TestDocumentedContractRoutesAdaptToServiceAPI` |
|
||||
| 8.4 `[D:8.3]` | **DONE.** Define opaque IDs, legal queue/match state transitions, revisions and idempotency keys | [`server/contracts/v1/state-transitions.json`](server/contracts/v1/state-transitions.json) locks terminal states, legal edges, stale-revision handling and same-key replay/conflict behavior; contract tests cover the invariants |
|
||||
| 8.5 `[D:8.4]` | **IN PROGRESS.** Initial PostgreSQL migration now defines durable idempotency keys, queue ownership/active-participation fencing, identities, sessions/revocations, ranked seasons, ratings/events, matches/participants, penalties, results, audits and outbox; follow-up migration persists server-derived queue probe RTT metadata | `server/migrations/0001_initial.sql`, `0003_queue_probe_metadata.sql`, `migrations/runner.go`, `cmd/migrate` and static checks cover the durable tables, uniqueness/check constraints, Redis-as-cache boundary and serialized forward migration recording; opt-in `scripts/run_postgres_integration.sh` now runs the runner and real queue/assignment ownership, idempotency, revision and expiry checks through pgx; rollback/down migration, the remaining serializable adapters and cache-loss repair remain |
|
||||
| 8.5 `[D:8.4]` | **IN PROGRESS.** Initial PostgreSQL migration now defines durable idempotency keys, queue ownership/active-participation fencing, identities, sessions/revocations, ranked seasons, ratings/events, matches/participants, penalties, results, audits and outbox; follow-up migrations persist server-derived queue probe RTT metadata and the allocator GameServer/allocation registry | `server/migrations/0001_initial.sql`, `0003_queue_probe_metadata.sql`, `0004_allocator_registry.sql`, `migrations/runner.go`, `cmd/migrate` and static checks cover the durable tables, uniqueness/check constraints, Redis-as-cache boundary and serialized forward migration recording; opt-in `scripts/run_postgres_integration.sh` now runs the runner and real queue/assignment ownership, idempotency, revision and expiry checks through pgx; rollback/down migration, the remaining serializable adapters and cache-loss repair remain |
|
||||
| 8.6 `[D:8.3,8.4]` | **IN PROGRESS.** Add allocated-mode `ServerConfig` compatibility fields as opt-in defaults | `ServerConfig` now validates allocation mode, match/server IDs, playlist version, client build, future assignment expiry, image digest, transport and EU/NA region; `server_boot.gd` fails closed for the not-yet-wired Steam SDR transport, constrains allocated processes to one match, and emits allocation identity/transport in `server_started`; signed-authorisation admission, dynamic endpoint wiring and full manifest/runtime tests remain |
|
||||
|
||||
#### 8B — Authentication and secure control plane
|
||||
@@ -1213,7 +1213,7 @@ the local/CI/community transport, not a silent production fallback.
|
||||
| 8.27 `[D:8.26]` | **IN PROGRESS.** Go supervisor package provides local-safe Agones REST discovery, validates assigned address/port data, injects dynamic `SDR_LISTEN_PORT`/`SDR_IP`, performs explicit process-ready probing and Ready transition; direct mode bypasses Agones | `server/supervisor/` covers allocated/direct startup, invalid endpoint rejection, dynamic endpoint/Ready ordering and authenticated drain; allocated Godot now supplies a loopback readiness/drain control surface and `agones_sdk.gd` supplies sidecar Health/Ready/Shutdown/annotation REST operations; metadata watch, real Agones annotation/shutdown confirmation and emulator integration remain |
|
||||
| 8.28 `[D:8.6,8.27]` | **IN PROGRESS.** Supervisor separates explicit process-ready from Agones Ready and never scrapes stdout; allocated mode refuses to mark Ready without a configured readiness probe | `server/supervisor/` tests prove Ready follows the probe and direct mode remains functional; `server_control.gd`, `agones_sdk.gd` and process-level smokes prove loopback `/ready`, `/health`, bearer-protected `/drain`, sidecar-shaped Health/Ready calls and drain admission fencing; detached-container and Health-reclaim integration remain |
|
||||
| 8.29 `[D:8.26,8.27]` | **IN PROGRESS.** Supervisor discovers and validates the Agones endpoint, propagates the actual dynamic `--port`, and exports `SDR_LISTEN_PORT`/`SDR_IP` only for Hosted-SDR while preserving an isolated ENet path | `server/supervisor/` tests cover invalid address/port rejection, dynamic port argument/env propagation and SDR-vs-ENet separation; real Agones dynamic/passthrough mapping, POP/cert/firewall/NAT and multi-match fixture remain |
|
||||
| 8.30 `[D:8.18,8.26,8.28,8.29]` | **IN PROGRESS.** Pure Go allocator filters Ready servers by region/build/protocol/transport, atomically claims one with idempotent allocation replay, and now owns the assignment-publication boundary | `server/domain/allocator.go` covers deterministic compatible selection, exhaustion, conflicting/identical allocation replay, unknown allocations, and assignment replay/conflict; Agones `GameServerAllocation`, signed roster metadata, bounded cross-replica retry and live integration remain |
|
||||
| 8.30 `[D:8.18,8.26,8.28,8.29]` | **IN PROGRESS.** Pure Go allocator filters Ready servers by region/build/protocol/transport, atomically claims one with idempotent allocation replay, and now owns the assignment-publication boundary; PostgreSQL adds durable GameServer registration and compatible `SKIP LOCKED` claims with request-digest fencing | `server/domain/allocator.go`, `server/store/allocator_sql.go`, `server/migrations/0004_allocator_registry.sql` and tests cover deterministic compatible selection, exhaustion, conflicting/identical allocation replay, unknown allocations, SQL claim ordering, invalid input and assignment replay/conflict; Agones `GameServerAllocation`, signed roster metadata, bounded cross-replica retry and live integration remain |
|
||||
| 8.31 `[D:8.9,8.30]` | **IN PROGRESS.** Pure Go assignment gate requires Allocated state, exact allocation ID/match/server/region/build/protocol/transport compatibility, non-empty hosted endpoint and verified manifest signature before exposure; allocator publication cannot expose Ready state | `server/domain/assignment.go` and `allocator.go` plus adversarial fixtures cover early-connect, tampered signature/manifest, wrong compatibility, empty endpoint, unknown allocation and post-publication mutation rejection; Agones metadata watch, hosted-address registration, production signer and client-ticket publication remain |
|
||||
| 8.32 `[D:8.2,8.26,8.30]` | **IN PROGRESS.** Provider-neutral FleetAutoscaler baseline preserves a two-process Ready buffer, caps warm capacity, and leaves Allocated scale-down independent of the Ready floor; Fleet image references remain digest-pinned for current/rollback pre-pull | `deploy/k8s/base/fleet-autoscaler.yaml` and manifest tests cover Fleet ownership, Buffer policy and floor/cap invariants; regional on-demand node pools/failure domains, pre-pull rollout, warm-allocation p95/p99 and N+1 certification remain |
|
||||
| 8.33 `[D:8.26,8.32]` | **IN PROGRESS.** Fleet scheduling now requires on-demand capacity and spreads Ready processes across zones with skew 1; the autoscaler preserves the two-process Ready floor | `deploy/k8s/base/fleet.yaml` and manifest tests reject interruptible placement and single-zone concentration structurally; regional node pools, forced node-loss testing and measured N+1 headroom remain |
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
-- Durable allocator registry. Agones remains the provider-facing lifecycle
|
||||
-- authority; these rows are the control-plane's auditable claim projection.
|
||||
CREATE TABLE game_servers (
|
||||
server_id TEXT PRIMARY KEY,
|
||||
region TEXT NOT NULL CHECK (region IN ('EU', 'NA')),
|
||||
build TEXT NOT NULL,
|
||||
protocol_version INTEGER NOT NULL CHECK (protocol_version > 0),
|
||||
transport TEXT NOT NULL CHECK (transport IN ('enet', 'steam_sdr')),
|
||||
state TEXT NOT NULL CHECK (state IN ('READY', 'ALLOCATED')),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE allocations (
|
||||
allocation_id TEXT PRIMARY KEY,
|
||||
match_id TEXT NOT NULL UNIQUE,
|
||||
server_id TEXT NOT NULL REFERENCES game_servers(server_id),
|
||||
region TEXT NOT NULL CHECK (region IN ('EU', 'NA')),
|
||||
build TEXT NOT NULL,
|
||||
protocol_version INTEGER NOT NULL CHECK (protocol_version > 0),
|
||||
transport TEXT NOT NULL CHECK (transport IN ('enet', 'steam_sdr')),
|
||||
request_digest BYTEA NOT NULL,
|
||||
state TEXT NOT NULL CHECK (state = 'ALLOCATED'),
|
||||
allocated_at TIMESTAMPTZ NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX game_servers_ready_compatibility
|
||||
ON game_servers (region, build, protocol_version, transport, server_id)
|
||||
WHERE state = 'READY';
|
||||
@@ -0,0 +1,85 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/cosmic-clash/cosmic-clash/server/domain"
|
||||
)
|
||||
|
||||
const RegisterReadyServerSQL = `INSERT INTO game_servers
|
||||
(server_id, region, build, protocol_version, transport, state, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, 'READY', $6)
|
||||
ON CONFLICT (server_id) DO UPDATE SET region = EXCLUDED.region,
|
||||
build = EXCLUDED.build, protocol_version = EXCLUDED.protocol_version,
|
||||
transport = EXCLUDED.transport, state = 'READY', updated_at = EXCLUDED.updated_at`
|
||||
|
||||
const ClaimReadyServerSQL = `UPDATE game_servers SET state = 'ALLOCATED', updated_at = $5
|
||||
WHERE server_id = (
|
||||
SELECT server_id FROM game_servers
|
||||
WHERE state = 'READY' AND region = $1 AND build = $2
|
||||
AND protocol_version = $3 AND transport = $4
|
||||
ORDER BY server_id
|
||||
LIMIT 1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
RETURNING server_id`
|
||||
|
||||
const InsertAllocationSQL = `INSERT INTO allocations
|
||||
(allocation_id, match_id, server_id, region, build, protocol_version, transport, request_digest, state, allocated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'ALLOCATED', $9)`
|
||||
|
||||
const SelectAllocationSQL = `SELECT allocation_id, match_id, server_id, region, build,
|
||||
protocol_version, transport, allocated_at, request_digest
|
||||
FROM allocations WHERE allocation_id = $1`
|
||||
|
||||
func RegisterReadyServer(ctx context.Context, db *sql.DB, server domain.ReadyServer, now time.Time) error {
|
||||
if db == nil || server.ServerID == "" || (server.Region != "EU" && server.Region != "NA") || server.Build == "" || server.Protocol <= 0 || (server.Transport != "enet" && server.Transport != "steam_sdr") || server.State != domain.ServerReady || now.IsZero() {
|
||||
return fmt.Errorf("invalid ready server registration")
|
||||
}
|
||||
_, err := db.ExecContext(ctx, RegisterReadyServerSQL, server.ServerID, server.Region, server.Build, server.Protocol, server.Transport, now)
|
||||
return err
|
||||
}
|
||||
|
||||
func ClaimAllocation(ctx context.Context, db *sql.DB, request domain.AllocationRequest, now time.Time) (domain.Allocation, error) {
|
||||
if db == nil || request.AllocationID == "" || request.MatchID == "" || (request.Region != "EU" && request.Region != "NA") || request.Build == "" || request.Protocol <= 0 || (request.Transport != "enet" && request.Transport != "steam_sdr") || now.IsZero() {
|
||||
return domain.Allocation{}, domain.ErrAllocationInput
|
||||
}
|
||||
digest := allocationRequestDigest(request)
|
||||
var allocation domain.Allocation
|
||||
err := RunSerializable(ctx, db, DefaultSerializableAttempts, func(ctx context.Context, tx *sql.Tx) error {
|
||||
var prior domain.Allocation
|
||||
var priorDigest []byte
|
||||
err := tx.QueryRowContext(ctx, SelectAllocationSQL, request.AllocationID).Scan(&prior.AllocationID, &prior.MatchID, &prior.ServerID, &prior.Region, &prior.Build, &prior.Protocol, &prior.Transport, &prior.AllocatedAt, &priorDigest)
|
||||
if err == nil {
|
||||
if !bytes.Equal(priorDigest, digest[:]) {
|
||||
return domain.ErrConflict
|
||||
}
|
||||
allocation = prior
|
||||
allocation.State = domain.ServerAllocated
|
||||
return nil
|
||||
}
|
||||
if err != sql.ErrNoRows {
|
||||
return err
|
||||
}
|
||||
var serverID string
|
||||
if err := tx.QueryRowContext(ctx, ClaimReadyServerSQL, request.Region, request.Build, request.Protocol, request.Transport, now).Scan(&serverID); err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return domain.ErrNoCapacity
|
||||
}
|
||||
return err
|
||||
}
|
||||
allocation = domain.Allocation{AllocationID: request.AllocationID, MatchID: request.MatchID, ServerID: serverID, Region: request.Region, Build: request.Build, Protocol: request.Protocol, Transport: request.Transport, State: domain.ServerAllocated, AllocatedAt: now}
|
||||
_, err = tx.ExecContext(ctx, InsertAllocationSQL, request.AllocationID, request.MatchID, serverID, request.Region, request.Build, request.Protocol, request.Transport, digest[:], now)
|
||||
return err
|
||||
})
|
||||
return allocation, err
|
||||
}
|
||||
|
||||
func allocationRequestDigest(request domain.AllocationRequest) [32]byte {
|
||||
return sha256.Sum256([]byte(fmt.Sprintf("%s\x00%s\x00%s\x00%s\x00%d\x00%s", request.AllocationID, request.MatchID, request.Region, request.Build, request.Protocol, request.Transport)))
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/cosmic-clash/cosmic-clash/server/domain"
|
||||
)
|
||||
|
||||
func TestAllocatorSQLClaimsAndAuditsCompatibleReadyServers(t *testing.T) {
|
||||
for query, fragments := range map[string][]string{
|
||||
RegisterReadyServerSQL: {"game_servers", "ON CONFLICT", "state = 'READY'"},
|
||||
ClaimReadyServerSQL: {"state = 'READY'", "region = $1", "protocol_version = $3", "FOR UPDATE SKIP LOCKED", "ORDER BY server_id"},
|
||||
InsertAllocationSQL: {"allocations", "request_digest", "state", "ALLOCATED"},
|
||||
} {
|
||||
for _, fragment := range fragments {
|
||||
if !contains(query, fragment) {
|
||||
t.Fatalf("query missing %q", fragment)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaimAllocationRejectsInvalidRequestsWithoutDatabase(t *testing.T) {
|
||||
_, err := ClaimAllocation(nil, nil, domain.AllocationRequest{AllocationID: "a", MatchID: "m", Region: "EU", Build: "b", Protocol: 1, Transport: "enet"}, time.Unix(1000, 0))
|
||||
if err == nil {
|
||||
t.Fatal("nil database accepted")
|
||||
}
|
||||
if _, err := ClaimAllocation(nil, nil, domain.AllocationRequest{AllocationID: "a", MatchID: "m", Region: "EU", Build: "b", Protocol: 0, Transport: "enet"}, time.Unix(1000, 0)); err != domain.ErrAllocationInput {
|
||||
t.Fatalf("invalid request err=%v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user