mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
feat(multiplayer): deliver allocated server rosters
This commit is contained in:
+1
-1
@@ -1235,7 +1235,7 @@ the local/CI/community transport, not a silent production fallback.
|
||||
|---|---|---|
|
||||
| 8.39 `[D:8.3,8.14,8.17]` | **IN PROGRESS.** `MatchmakingState` now projects queue → proposal → allocation/process-ready/assignment-ready/connect/live plus terminal failure states; autoload `ControlPlaneClient` provides authenticated queue create/recovery/heartbeat/cancel and proposal response requests with idempotency/revision headers; `matchmaking.tscn`/`matchmaking.gd` expose the state and authoritative actions from the main menu; authenticated proposal recovery now reconciles missed proposal events and expires them at read time; the Go API publishes targeted authenticated revisioned queue/proposal/assignment events and the Godot client consumes state/proposal/assignment events, fetching the authoritative assignment after assignment readiness; the API can now use the durable participant-scoped proposal provider for both recovery and accept/decline mutations | `test_matchmaking_state.gd`, `test_control_plane_client.gd`, `test_matchmaking_ui.gd`, `TestProposalRecoveryIsParticipantScopedAndExpiresAtReadBoundary`, `TestAuthenticatedWebSocketDeliversOnlyTargetedRevisionedEvents`, `TestStateChangingAPIActionsPublishTargetedEvents`, `TestProposalRecoveryUsesDurableBackendAndRemainsParticipantScoped` and store proposal SQL tests reject stale/gapped/conflicting updates, validate endpoint/token/payload normalization, preserve idempotent duplicates, and guarantee visible phase/terminal copy; wiring durable allocation events into the outbox, wait/latency explanations and Godot runtime verification remain |
|
||||
| 8.40 `[D:8.3,8.14]` | **IN PROGRESS.** Pure Go revisioned replica reducer rejects gaps for REST resync, makes duplicate/out-of-order events idempotent, and resumes from the authoritative snapshot revision; authenticated queue-ticket recovery now has an owner-checked REST read; Godot client projection persists non-secret ticket/proposal state, forces authoritative recovery after restart, and can replay a lost queue-create response with the original ticket/idempotency key; the Go API now exposes an authenticated `/v1/events` WebSocket with bounded per-player queues, strict upgrade/vocabulary validation and REST-resync-safe slow-client failure; Godot `ControlPlaneClient` can connect to the stream using Godot 4.7's handshake-header API, consume validated events, automatically reconnect with bounded backoff, and trigger REST recovery on projection gaps and stream return, including a deferred recovery when an event arrives during an in-flight HTTP mutation; `OutboxDispatcher` now provides ordered at-least-once delivery after durable commit; assignment recovery events carry the durable assignment revision without leaking it into REST JSON; prediction startup now distinguishes the sequence-0 warm-up acknowledgement from genuine missing/evicted history, preventing a false hard-snap during a live match; proposal creation writes one durable targeted `proposal_changed` outbox event per proposal aggregate revision, while result completion writes `match_completed` and production `cmd/control-plane` plus the test-only API harness dispatch both event types through separate filtered consumers | `server/domain/sync.go`, `server/api/events.go`, `server/api/outbox.go`, `server/api/service.go`, `server/store/outbox.go`, `server/store/proposal_sql.go`, `server/cmd/control-plane`, `server/cmd/testkit-api`, `service_test.go`, `outbox_test.go`, `matchmaking_state.gd`, `control_plane_client.gd`, `local_prediction_history.gd`, `net_ship_predictor.gd` and tests cover gap, snapshot, replay, same-revision conflict, owner-only ticket recovery, expired-ticket terminal handling, malformed restart snapshots, API-level duplicate-create replay/conflict, authenticated handshake/key rejection, targeted event delivery, exactly-once slow-subscriber closure, invalid-event rejection, delivery-before-ack failure ordering, assignment event/response revision separation, prediction warm-up/hard-resync separation, deferred proposal recovery and proposal/result outbox filtering and delivery failures; Godot 4.7.1 headless project parse and 150-test unit harness pass with compatibility rendering, and a two-process ENet authoritative match smoke passes spawn, movement, prediction, 0% snapshot loss and 0 hard snaps; `cmd/control-plane` provides a signal-bound API role, `cmd/matcher` now supports casual and explicitly enabled ranked roles with durable identity lookup, and `cmd/maintenance` provides bounded season maintenance. **Live multi-process control-plane/game verification now exists**: `scripts/verify_control_plane_integration.sh` runs a real `postgres:17-alpine`, the real `api.Service` (via the new test-only `server/cmd/testkit-api`, wired identically to `cmd/control-plane` except for a fake Steam login — see §8.7), and a real headless Godot client (`control_plane_smoke.gd`) round-tripping login → `fetch_ranked_profile` (expect 404, §8.22) → `queue_create` → heartbeat → `cancel_queue` over an actual network connection — the first time this boundary was exercised end to end rather than against a mock on either side. **The two-player proposal extension now also passes**: `scripts/verify_control_plane_proposal_integration.sh` starts isolated PostgreSQL, the real matcher and testkit API, launches two headless Godot clients, seeds deterministic RTT projections, observes the real WebSocket `OPEN` proposal, accepts from both clients, and verifies both durable tickets plus the proposal reach `ACCEPTED` (including authoritative recovery when the concurrent revision advances). `scripts/run_result_fanout_integration.sh` additionally verifies a real PostgreSQL-backed authenticated WebSocket receives a completed-match event; allocator and Redis fan-out live verification remain |
|
||||
| 8.41 `[D:7.8,8.9,8.31,8.40]` | **IN PROGRESS.** Authenticated `GET /v1/assignments/{matchId}` now exposes only a validated, player-scoped assignment view including the hosted endpoint; Godot `AssignmentState`/`ControlPlaneClient.fetch_assignment()` bind the response to the authenticated player, recheck expiry, validate and retain the endpoint, and `connect_to_assignment()` now starts only the validated ENet/Steam transport after assignment readiness; the opaque join authorisation is carried in the MatchNet hello payload rather than the endpoint URL; allocated Godot servers now fail closed unless an operator-mounted JSON roster of control-plane signed envelopes and an HMAC-SHA256 key are present, and MatchNet verifies the canonical Go claim bytes, checks exact token membership plus match/server/protocol/slot/expiry claims before admitting a peer, rejects concurrent reuse of an active token, tracks server-owned reconnect generations across the 60-second reclaim window, fences expired reclaims and invalidates old peers; migration 0002 and the Go store adapter now persist/recover the complete player-scoped assignment projection with conflict-safe identical replay; `AssignmentProviderFromStore` wires that durable projection into the API injection point; `SaveAssignments` publishes a complete signed roster atomically instead of allowing partial player visibility; `SaveVerifiedAssignmentRoster` rechecks signed claims before deriving player rows; live authenticated Godot/PostgreSQL assignment verification now exists via `scripts/verify_assignment_integration.sh`, which seeds a durable player-scoped `ASSIGNMENT_READY` row behind the real testkit API and verifies the returned endpoint/join authorisation over HTTP/JSON (including the live numeric-decoding boundary); `scripts/run_supervisor_integration.sh` additionally verifies the allocated-server process-ready → assignment-ready registration path against real PostgreSQL and the real API, with the workload token recovered from an Agones-shaped GameServer annotation | `server/api/service.go`, `store_adapters.go`, `service_test.go`, `assignment_state.gd`, `control_plane_client.gd`, `match_net.gd`, `server_boot.gd`, `server_config.gd`, `test_assignment_state.gd`, `test_control_plane_client.gd`, `test_match_net.gd`, `server/contracts/v1/openapi.json`, `server/migrations/0002_assignments.sql`, `server/allocator/allocator_integration_test.go`, `server/supervisor/supervisor_integration_test.go` and `server/store/assignment_sql.go` cover participant/identity/expiry/shape/transport/endpoint boundaries, player-scoped schema keys, expiry-filtered reads, assignment upsert conflict handling, atomic batch validation, signed-claim binding, strict endpoint splitting, allowlisted claim rejection, canonical HMAC interoperability, forged-signature rejection, duplicate active-token rejection, reclaim generation/expiry behavior, real allocator reconciliation, real supervisor registration and 150-test Godot compatibility coverage; the three-process disconnect/reclaim smoke passes ship retention, replacement drive and old-peer invalidation; SDR relay-ticket installation and live Agones cluster integration remain |
|
||||
| 8.41 `[D:7.8,8.9,8.31,8.40]` | **IN PROGRESS.** Authenticated `GET /v1/assignments/{matchId}` now exposes only a validated, player-scoped assignment view including the hosted endpoint; Godot `AssignmentState`/`ControlPlaneClient.fetch_assignment()` bind the response to the authenticated player, recheck expiry, validate and retain the endpoint, and `connect_to_assignment()` now starts only the validated ENet/Steam transport after assignment readiness; the opaque join authorisation is carried in the MatchNet hello payload rather than the endpoint URL; allocated Godot servers now fail closed unless an operator-mounted JSON roster of control-plane signed envelopes and an HMAC-SHA256 key are present, and MatchNet verifies the canonical Go claim bytes, checks exact token membership plus match/server/protocol/slot/expiry claims before admitting a peer, rejects concurrent reuse of an active token, tracks server-owned reconnect generations across the 60-second reclaim window, fences expired reclaims and invalidates old peers; migration 0002 and the Go store adapter now persist/recover the complete player-scoped assignment projection with conflict-safe identical replay; `AssignmentProviderFromStore` wires that durable projection into the API injection point; `SaveAssignments` publishes a complete signed roster atomically instead of allowing partial player visibility; `SaveVerifiedAssignmentRoster` rechecks signed claims before deriving player rows; a workload-authenticated `GET /v1/servers/{serverId}/roster` now returns the complete signed envelope set only for the bound allocation, and the allocated supervisor atomically materializes it before launching Godot; live authenticated Godot/PostgreSQL assignment verification now exists via `scripts/verify_assignment_integration.sh`, which seeds a durable player-scoped `ASSIGNMENT_READY` row behind the real testkit API and verifies the returned endpoint/join authorisation over HTTP/JSON (including the live numeric-decoding boundary); `scripts/run_supervisor_integration.sh` additionally verifies the allocated-server process-ready → roster fetch/materialization → assignment-ready registration path against real PostgreSQL and the real API, with the workload token recovered from an Agones-shaped GameServer annotation | `server/api/service.go`, `store_adapters.go`, `service_test.go`, `assignment_state.gd`, `control_plane_client.gd`, `match_net.gd`, `server_boot.gd`, `server_config.gd`, `test_assignment_state.gd`, `test_control_plane_client.gd`, `test_match_net.gd`, `server/contracts/v1/openapi.json`, `server/migrations/0002_assignments.sql`, `server/allocator/allocator_integration_test.go`, `server/supervisor/supervisor_integration_test.go`, `server/supervisor/supervisor.go` and `server/store/assignment_sql.go` cover participant/identity/expiry/shape/transport/endpoint boundaries, player-scoped schema keys, expiry-filtered reads, assignment upsert conflict handling, atomic batch validation, signed-claim binding, strict endpoint splitting, allowlisted claim rejection, canonical HMAC interoperability, forged-signature rejection, duplicate active-token rejection, reclaim generation/expiry behavior, workload-authenticated roster delivery, atomic file installation, real allocator reconciliation, real supervisor registration and 150-test Godot compatibility coverage; the three-process disconnect/reclaim smoke passes ship retention, replacement drive and old-peer invalidation; dynamic per-match launch flags, SDR relay-ticket installation and live Agones cluster integration remain |
|
||||
| 8.42 `[D:8.22,8.23,8.24,8.40]` | **IN PROGRESS.** `RankedProfileState` and `ControlPlaneClient.fetch_ranked_profile()` expose the backend-authoritative rating/RD/volatility/games/tier/provisional/season view; matchmaking UI displays provisional/tier status without client-side rating math; runnable API binaries now wire the backend-owned default tier policy; live authenticated profile verification now exists via `scripts/verify_ranked_profile_integration.sh`, which seeds a durable rating and verifies the populated profile over real PostgreSQL, Go HTTP, and Godot JSON | `test_control_plane_client.gd` validates profile shape, numeric safety and provisional display; the real populated profile path now covers durable rating/games/tier/provisional decoding and caught the missing runtime tier-policy wiring; committed revision after reconnect, abandon status and season countdown remain dependent on live auth/backend events and Godot runtime verification |
|
||||
| 8.43 `[D:8.39,8.40,8.41]` | **IN PROGRESS.** Matchmaking client now distinguishes expired queue recovery, session expiry, missing records and retryable control-plane outages; a 401 clears the in-memory token, emits `session_expired` and disables retry until a new session is configured; terminal messages remain visible and active searches are not falsely failed on transient errors; stale proposal accept/decline conflicts now automatically schedule an authoritative proposal recovery instead of requiring each UI caller to implement 409 handling | `MatchmakingState` and `ControlPlaneClient` tests cover explicit expiry and the existing terminal/retry-safe state paths; the real two-player proposal smoke now exercises the client-owned conflict-recovery path; decline, version mismatch, regional outage retry UI, failed reconnect, duplicate-action recovery beyond proposals and broader live Godot verification remain |
|
||||
|
||||
|
||||
+28
-10
@@ -96,6 +96,7 @@ type AssignmentView struct {
|
||||
}
|
||||
|
||||
type AssignmentProvider func(context.Context, string, string, time.Time) (AssignmentView, error)
|
||||
type RosterProvider func(context.Context, domain.WorkloadBinding, time.Time) ([][]byte, error)
|
||||
|
||||
type Service struct {
|
||||
Sessions *domain.SessionStore
|
||||
@@ -113,6 +114,7 @@ type Service struct {
|
||||
ResultSubmitter ResultSubmitter
|
||||
ServerRegistrar ServerRegistrar
|
||||
Assignment AssignmentProvider
|
||||
Roster RosterProvider
|
||||
Now func() time.Time
|
||||
Proposals map[string]*domain.Proposal
|
||||
ProposalBackend ProposalBackend
|
||||
@@ -461,22 +463,17 @@ type serverRegistrationRequest struct {
|
||||
}
|
||||
|
||||
func (s *Service) serverMutation(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
writeError(w, http.StatusMethodNotAllowed, "method_not_allowed")
|
||||
return
|
||||
}
|
||||
parts := strings.Split(strings.TrimPrefix(r.URL.Path, "/v1/servers/"), "/")
|
||||
if len(parts) != 2 || parts[0] == "" || (parts[1] != "result" && parts[1] != "register") {
|
||||
if len(parts) != 2 || parts[0] == "" || (parts[1] != "result" && parts[1] != "register" && parts[1] != "roster") {
|
||||
writeError(w, http.StatusNotFound, "not_found")
|
||||
return
|
||||
}
|
||||
if s.WorkloadVerify == nil || (parts[1] == "result" && s.ResultSubmitter == nil) || (parts[1] == "register" && s.ServerRegistrar == nil) {
|
||||
writeError(w, http.StatusServiceUnavailable, "server_unavailable")
|
||||
if parts[1] == "roster" && r.Method != http.MethodGet || parts[1] != "roster" && r.Method != http.MethodPost {
|
||||
writeError(w, http.StatusMethodNotAllowed, "method_not_allowed")
|
||||
return
|
||||
}
|
||||
key := r.Header.Get("Idempotency-Key")
|
||||
if len(key) < 16 || len(key) > 128 {
|
||||
writeError(w, http.StatusBadRequest, "invalid_idempotency_key")
|
||||
if s.WorkloadVerify == nil || (parts[1] == "result" && s.ResultSubmitter == nil) || (parts[1] == "register" && s.ServerRegistrar == nil) || (parts[1] == "roster" && s.Roster == nil) {
|
||||
writeError(w, http.StatusServiceUnavailable, "server_unavailable")
|
||||
return
|
||||
}
|
||||
partsAuth := strings.Fields(r.Header.Get("Authorization"))
|
||||
@@ -491,6 +488,27 @@ func (s *Service) serverMutation(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusUnauthorized, "unauthorized")
|
||||
return
|
||||
}
|
||||
if parts[1] == "roster" {
|
||||
roster, err := s.Roster(r.Context(), binding, now)
|
||||
if err != nil || len(roster) == 0 {
|
||||
writeError(w, http.StatusUnprocessableEntity, "roster_unavailable")
|
||||
return
|
||||
}
|
||||
encodedRoster := make([]json.RawMessage, 0, len(roster))
|
||||
for _, envelope := range roster {
|
||||
encodedRoster = append(encodedRoster, json.RawMessage(envelope))
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(encodedRoster); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
key := r.Header.Get("Idempotency-Key")
|
||||
if len(key) < 16 || len(key) > 128 {
|
||||
writeError(w, http.StatusBadRequest, "invalid_idempotency_key")
|
||||
return
|
||||
}
|
||||
if parts[1] == "register" {
|
||||
var input serverRegistrationRequest
|
||||
if !decodeBody(w, r, &input) {
|
||||
|
||||
@@ -329,6 +329,44 @@ func TestAuthenticatedWebSocketDeliversOnlyTargetedRevisionedEvents(t *testing.T
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerRosterRequiresWorkloadBindingAndReturnsRawSignedEnvelopes(t *testing.T) {
|
||||
now := time.Unix(1000, 0).UTC()
|
||||
service := &Service{
|
||||
Now: func() time.Time { return now },
|
||||
WorkloadVerify: func(token string, at time.Time) (domain.WorkloadBinding, error) {
|
||||
if token != "workload-token" || !at.Equal(now) {
|
||||
t.Fatal("unexpected workload verification input")
|
||||
}
|
||||
return domain.WorkloadBinding{ServerID: "server-1", MatchID: "match-1", AllocationID: "allocation-1"}, nil
|
||||
},
|
||||
Roster: func(_ context.Context, binding domain.WorkloadBinding, at time.Time) ([][]byte, error) {
|
||||
if binding.ServerID != "server-1" || binding.MatchID != "match-1" || !at.Equal(now) {
|
||||
t.Fatal("unexpected roster binding")
|
||||
}
|
||||
return [][]byte{[]byte(`{"authorisation":{"player_id":"player-1"},"signature":"sig"}`)}, nil
|
||||
},
|
||||
}
|
||||
server := httptest.NewServer(service.Handler())
|
||||
defer server.Close()
|
||||
request, _ := http.NewRequest(http.MethodGet, server.URL+"/v1/servers/server-1/roster", nil)
|
||||
request.Header.Set("Authorization", "Bearer workload-token")
|
||||
response, err := server.Client().Do(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
t.Fatalf("roster status=%d", response.StatusCode)
|
||||
}
|
||||
var roster []json.RawMessage
|
||||
if err := json.NewDecoder(response.Body).Decode(&roster); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(roster) != 1 || !bytes.Contains(roster[0], []byte(`"player_id":"player-1"`)) {
|
||||
t.Fatalf("roster=%s", roster[0])
|
||||
}
|
||||
}
|
||||
|
||||
func readServerWebSocketFrame(reader *bufio.Reader) ([]byte, error) {
|
||||
first, err := reader.ReadByte()
|
||||
if err != nil {
|
||||
|
||||
@@ -104,11 +104,14 @@ func newAPIService(db *sql.DB, workloadSecret string, indexes ...api.CandidateIn
|
||||
RankedProfileProvider: store.PostgresRankedProfiles{DB: db},
|
||||
TierPolicy: domain.DefaultTierPolicy(),
|
||||
Assignment: api.AssignmentProviderFromStore(db),
|
||||
CandidateIndex: candidateIndex,
|
||||
ProbeRecorder: store.PostgresQueue{DB: db},
|
||||
WorkloadVerify: api.WorkloadVerifierFromSignedToken([]byte(workloadSecret), db),
|
||||
Now: func() time.Time { return time.Now().UTC() },
|
||||
Log: logEvent,
|
||||
Roster: func(ctx context.Context, binding domain.WorkloadBinding, now time.Time) ([][]byte, error) {
|
||||
return store.GetAssignmentRoster(ctx, db, binding.MatchID, binding.ServerID, now)
|
||||
},
|
||||
CandidateIndex: candidateIndex,
|
||||
ProbeRecorder: store.PostgresQueue{DB: db},
|
||||
WorkloadVerify: api.WorkloadVerifierFromSignedToken([]byte(workloadSecret), db),
|
||||
Now: func() time.Time { return time.Now().UTC() },
|
||||
Log: logEvent,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -49,6 +49,7 @@ func main() {
|
||||
imageDigestEnv := options.String("image-digest-env", "COSMIC_CLASH_IMAGE_DIGEST", "environment variable containing this build's sha256 image digest")
|
||||
assignmentReadyAttempts := options.Int("assignment-ready-attempts", 5, "retry attempts for assignment-ready registration after process-ready succeeds (a slow-to-propagate signed roster is not fatal)")
|
||||
assignmentReadyBackoff := options.Duration("assignment-ready-backoff", 2*time.Second, "delay between assignment-ready retry attempts")
|
||||
rosterPath := options.String("roster-path", "", "writable path for the workload-authenticated signed join roster; fetched before the child starts")
|
||||
if err := options.Parse(args[:separator]); err != nil {
|
||||
os.Exit(2)
|
||||
}
|
||||
@@ -75,6 +76,7 @@ func main() {
|
||||
|
||||
AssignmentReadyAttempts: *assignmentReadyAttempts,
|
||||
AssignmentReadyBackoff: *assignmentReadyBackoff,
|
||||
RosterPath: *rosterPath,
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "game-server-supervisor: %v\n", err)
|
||||
|
||||
@@ -66,9 +66,12 @@ func main() {
|
||||
RankedProfileProvider: store.PostgresRankedProfiles{DB: db},
|
||||
TierPolicy: domain.DefaultTierPolicy(),
|
||||
Assignment: api.AssignmentProviderFromStore(db),
|
||||
ProbeRecorder: store.PostgresQueue{DB: db},
|
||||
WorkloadVerify: api.WorkloadVerifierFromSignedToken([]byte(*workloadSecret), db),
|
||||
Now: func() time.Time { return time.Now().UTC() },
|
||||
Roster: func(ctx context.Context, binding domain.WorkloadBinding, now time.Time) ([][]byte, error) {
|
||||
return store.GetAssignmentRoster(ctx, db, binding.MatchID, binding.ServerID, now)
|
||||
},
|
||||
ProbeRecorder: store.PostgresQueue{DB: db},
|
||||
WorkloadVerify: api.WorkloadVerifierFromSignedToken([]byte(*workloadSecret), db),
|
||||
Now: func() time.Time { return time.Now().UTC() },
|
||||
}
|
||||
handler := service.Handler()
|
||||
listener, err := net.Listen("tcp", *listen)
|
||||
|
||||
@@ -69,6 +69,11 @@ const AssignmentSelectSQL = `SELECT match_id, player_id, allocation_id, server_i
|
||||
FROM assignments
|
||||
WHERE match_id = $1 AND player_id = $2 AND expires_at > $3`
|
||||
|
||||
const AssignmentRosterSelectSQL = `SELECT allocation_id, server_id, join_authorisation
|
||||
FROM assignments
|
||||
WHERE match_id = $1 AND server_id = $2 AND expires_at > $3
|
||||
ORDER BY slot, player_id`
|
||||
|
||||
func validateDurableAssignment(assignment DurableAssignment) error {
|
||||
if assignment.MatchID == "" || assignment.PlayerID == "" || assignment.AllocationID == "" || assignment.ServerID == "" || assignment.Slot < 0 || assignment.Slot > 5 || (assignment.Region != "EU" && assignment.Region != "NA") || assignment.ClientBuild == "" || assignment.ProtocolVersion < 1 || (assignment.Transport != "enet" && assignment.Transport != "steam_sdr") || assignment.Endpoint == "" || assignment.JoinAuthorisation == "" || len(assignment.ManifestDigest) == 0 || assignment.ExpiresAt.IsZero() || assignment.Revision < 0 {
|
||||
return fmt.Errorf("invalid durable assignment")
|
||||
@@ -187,3 +192,43 @@ func GetAssignment(ctx context.Context, db *sql.DB, playerID, matchID string, no
|
||||
}
|
||||
return assignment, nil
|
||||
}
|
||||
|
||||
// GetAssignmentRoster returns the complete signed roster for an allocated
|
||||
// server. It is intentionally server-scoped rather than player-scoped and is
|
||||
// called only after workload authentication at the API boundary. All rows
|
||||
// must belong to one allocation; a partial or mixed allocation is unsafe to
|
||||
// hand to the game process.
|
||||
func GetAssignmentRoster(ctx context.Context, db *sql.DB, matchID, serverID string, now time.Time) ([][]byte, error) {
|
||||
if db == nil || matchID == "" || serverID == "" || now.IsZero() {
|
||||
return nil, fmt.Errorf("invalid assignment roster arguments")
|
||||
}
|
||||
rows, err := db.QueryContext(ctx, AssignmentRosterSelectSQL, matchID, serverID, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var allocationID string
|
||||
var roster [][]byte
|
||||
for rows.Next() {
|
||||
var rowAllocation, rowServer, encoded string
|
||||
if err := rows.Scan(&rowAllocation, &rowServer, &encoded); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if rowServer != serverID || rowAllocation == "" || (allocationID != "" && allocationID != rowAllocation) {
|
||||
return nil, fmt.Errorf("assignment roster contains mixed allocation")
|
||||
}
|
||||
decoded, err := base64.RawURLEncoding.DecodeString(encoded)
|
||||
if err != nil || len(decoded) == 0 {
|
||||
return nil, fmt.Errorf("assignment roster contains invalid envelope")
|
||||
}
|
||||
allocationID = rowAllocation
|
||||
roster = append(roster, decoded)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(roster) == 0 {
|
||||
return nil, sql.ErrNoRows
|
||||
}
|
||||
return roster, nil
|
||||
}
|
||||
|
||||
@@ -8,11 +8,13 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -89,6 +91,11 @@ type Config struct {
|
||||
// listening and usable either way. Default 5 attempts, 2s apart.
|
||||
AssignmentReadyAttempts int
|
||||
AssignmentReadyBackoff time.Duration
|
||||
// RosterPath is an operator-mounted writable path where the supervisor
|
||||
// materializes the workload-authenticated signed roster before starting
|
||||
// Godot. It is deliberately separate from WorkloadTokenPath: the former
|
||||
// contains match join envelopes, the latter contains a bearer credential.
|
||||
RosterPath string
|
||||
}
|
||||
|
||||
type Supervisor struct {
|
||||
@@ -136,6 +143,9 @@ func New(config Config) (*Supervisor, error) {
|
||||
if config.ControlPlaneURL != "" && (config.ServerID == "" || config.ProtocolVersion < 1 || config.ImageDigest == "") {
|
||||
return nil, fmt.Errorf("control-plane registration requires a server ID, protocol version and image digest")
|
||||
}
|
||||
if config.RosterPath != "" && config.ControlPlaneURL == "" {
|
||||
return nil, fmt.Errorf("roster path requires control-plane URL")
|
||||
}
|
||||
// Neither MatchID nor WorkloadTokenPath is required here: both can
|
||||
// instead be resolved at Start time from the allocated GameServer's own
|
||||
// annotations (see registerControlPlane/workloadToken/matchID). They are
|
||||
@@ -172,6 +182,9 @@ func (s *Supervisor) Start(ctx context.Context) error {
|
||||
if s.config.Transport == "steam_sdr" {
|
||||
env = append(env, "SDR_LISTEN_PORT="+strconv.Itoa(port), "SDR_IP="+address+":"+strconv.Itoa(port))
|
||||
}
|
||||
if err := s.fetchRoster(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
command := withPort(s.config.Command, port)
|
||||
s.cmd = exec.CommandContext(ctx, command[0], command[1:]...)
|
||||
} else {
|
||||
@@ -205,6 +218,70 @@ func (s *Supervisor) Start(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Supervisor) fetchRoster(ctx context.Context) error {
|
||||
if s.config.RosterPath == "" {
|
||||
return nil
|
||||
}
|
||||
matchID := s.matchID()
|
||||
if matchID == "" {
|
||||
return fmt.Errorf("roster fetch has no match ID")
|
||||
}
|
||||
token, err := s.workloadToken()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rosterURL := strings.TrimRight(s.config.ControlPlaneURL, "/") + "/v1/servers/" + url.PathEscape(s.config.ServerID) + "/roster"
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, rosterURL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request.Header.Set("Authorization", "Bearer "+token)
|
||||
response, err := s.client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode/100 != 2 {
|
||||
return fmt.Errorf("control-plane roster returned %s", response.Status)
|
||||
}
|
||||
var roster []json.RawMessage
|
||||
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&roster); err != nil || len(roster) == 0 {
|
||||
if err == nil {
|
||||
err = fmt.Errorf("empty roster")
|
||||
}
|
||||
return fmt.Errorf("decode control-plane roster: %w", err)
|
||||
}
|
||||
for _, envelope := range roster {
|
||||
if len(envelope) == 0 || string(envelope) == "null" {
|
||||
return fmt.Errorf("control-plane roster contains an invalid envelope")
|
||||
}
|
||||
}
|
||||
contents, err := json.Marshal(roster)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode roster: %w", err)
|
||||
}
|
||||
directory := filepath.Dir(s.config.RosterPath)
|
||||
temporary, err := os.CreateTemp(directory, ".cosmic-clash-roster-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create roster file: %w", err)
|
||||
}
|
||||
temporaryName := temporary.Name()
|
||||
defer os.Remove(temporaryName)
|
||||
if err := temporary.Chmod(0600); err == nil {
|
||||
_, err = temporary.Write(contents)
|
||||
}
|
||||
if closeErr := temporary.Close(); err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("write roster file: %w", err)
|
||||
}
|
||||
if err := os.Rename(temporaryName, s.config.RosterPath); err != nil {
|
||||
return fmt.Errorf("install roster file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// reportAssignmentReady is best-effort: process-ready has already succeeded,
|
||||
// so the process is legitimately usable either way. A persistent failure is
|
||||
// written to stderr rather than returned, since treating it as fatal would
|
||||
|
||||
@@ -5,11 +5,13 @@ package supervisor
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -78,7 +80,7 @@ func TestRealSupervisorRegistersAllocatedServerThroughControlPlane(t *testing.T)
|
||||
if err := store.SaveAssignment(ctx, db, store.DurableAssignment{
|
||||
MatchID: "supervisor-live-match", PlayerID: player, AllocationID: request.AllocationID, ServerID: "supervisor-live-server", Slot: index,
|
||||
Region: "EU", ClientBuild: "build-1", ProtocolVersion: 1, Transport: "enet", Endpoint: "127.0.0.1:7777",
|
||||
JoinAuthorisation: "join-" + player, ManifestDigest: []byte{0, 1, 2, 3}, ExpiresAt: now.Add(time.Hour), Revision: 1,
|
||||
JoinAuthorisation: base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf(`{"authorisation":{"match_id":"supervisor-live-match","server_id":"supervisor-live-server","player_id":%q},"signature":"sig"}`, player))), ManifestDigest: []byte{0, 1, 2, 3}, ExpiresAt: now.Add(time.Hour), Revision: 1,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -99,13 +101,16 @@ func TestRealSupervisorRegistersAllocatedServerThroughControlPlane(t *testing.T)
|
||||
}
|
||||
}))
|
||||
defer sdk.Close()
|
||||
service := &api.Service{ServerRegistrar: api.ServerRegistrarFromStore(db), WorkloadVerify: api.WorkloadVerifierFromSignedToken(secret, db), Now: func() time.Time { return now }}
|
||||
rosterPath := filepath.Join(t.TempDir(), "join-roster.json")
|
||||
service := &api.Service{ServerRegistrar: api.ServerRegistrarFromStore(db), WorkloadVerify: api.WorkloadVerifierFromSignedToken(secret, db), Roster: func(ctx context.Context, binding domain.WorkloadBinding, at time.Time) ([][]byte, error) {
|
||||
return store.GetAssignmentRoster(ctx, db, binding.MatchID, binding.ServerID, at)
|
||||
}, Now: func() time.Time { return now }}
|
||||
control := httptest.NewServer(service.Handler())
|
||||
defer control.Close()
|
||||
supervisor, err := New(Config{
|
||||
Command: []string{"/bin/sh", "-c", "true"}, SDKBaseURL: sdk.URL, ReadyURL: sdk.URL + "/ready-probe", ControlPlaneURL: control.URL,
|
||||
ServerID: "supervisor-live-server", ProtocolVersion: 1, ImageDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
ReadyTimeout: time.Second, PollInterval: time.Millisecond, AssignmentReadyAttempts: 1,
|
||||
ReadyTimeout: time.Second, PollInterval: time.Millisecond, AssignmentReadyAttempts: 1, RosterPath: rosterPath,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -116,6 +121,10 @@ func TestRealSupervisorRegistersAllocatedServerThroughControlPlane(t *testing.T)
|
||||
if err := supervisor.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
roster, err := os.ReadFile(rosterPath)
|
||||
if err != nil || !strings.Contains(string(roster), "supervisor-live-a") || !strings.Contains(string(roster), "supervisor-live-b") {
|
||||
t.Fatalf("materialized live roster=%q err=%v", roster, err)
|
||||
}
|
||||
var matchState, ticketState string
|
||||
if err := db.QueryRowContext(ctx, `SELECT state FROM matches WHERE match_id = 'supervisor-live-match'`).Scan(&matchState); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -68,6 +68,56 @@ func TestAllocatedStartInjectsDynamicEndpointAndCallsReadyAfterProbe(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllocatedStartMaterializesWorkloadAuthenticatedRosterBeforeChild(t *testing.T) {
|
||||
rosterPath := filepath.Join(t.TempDir(), "join-roster.json")
|
||||
sdk := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/gameserver":
|
||||
_, _ = w.Write([]byte(`{"object_meta":{"annotations":{"cosmic-clash.io/match-id":"match-1","cosmic-clash.io/workload-token":"workload-token"}},"status":{"address":"127.0.0.1","ports":[{"name":"game","port":31001}]}}`))
|
||||
case "/ready-probe", "/ready":
|
||||
w.WriteHeader(http.StatusOK)
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer sdk.Close()
|
||||
controlPlane := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/v1/servers/server-1/roster" {
|
||||
if r.Method != http.MethodGet || r.Header.Get("Authorization") != "Bearer workload-token" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`[{"authorisation":{"player_id":"player-1"},"signature":"sig"}]`))
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/v1/servers/server-1/register" {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer controlPlane.Close()
|
||||
command := []string{"/bin/sh", "-c", "test -s '" + rosterPath + "'"}
|
||||
s, err := New(Config{
|
||||
Command: command, SDKBaseURL: sdk.URL, ReadyURL: sdk.URL + "/ready-probe", ControlPlaneURL: controlPlane.URL,
|
||||
ServerID: "server-1", ProtocolVersion: 1, ImageDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
RosterPath: rosterPath, ReadyTimeout: time.Second, PollInterval: time.Millisecond,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Start(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
contents, err := os.ReadFile(rosterPath)
|
||||
if err != nil || !strings.Contains(string(contents), "player-1") {
|
||||
t.Fatalf("materialized roster=%q err=%v", contents, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestControlPlaneRegistrationRejectsIncompleteConfig(t *testing.T) {
|
||||
base := Config{Command: []string{"/bin/true"}, ControlPlaneURL: "https://control-plane.invalid"}
|
||||
if _, err := New(base); err == nil {
|
||||
|
||||
Reference in New Issue
Block a user