feat(multiplayer): deliver allocated server rosters

This commit is contained in:
Josh Creek
2026-09-01 15:54:07 +01:00
parent 863cf61f1a
commit 68e76b5feb
10 changed files with 267 additions and 22 deletions
+77
View File
@@ -8,11 +8,13 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
@@ -89,6 +91,11 @@ type Config struct {
// listening and usable either way. Default 5 attempts, 2s apart.
AssignmentReadyAttempts int
AssignmentReadyBackoff time.Duration
// RosterPath is an operator-mounted writable path where the supervisor
// materializes the workload-authenticated signed roster before starting
// Godot. It is deliberately separate from WorkloadTokenPath: the former
// contains match join envelopes, the latter contains a bearer credential.
RosterPath string
}
type Supervisor struct {
@@ -136,6 +143,9 @@ func New(config Config) (*Supervisor, error) {
if config.ControlPlaneURL != "" && (config.ServerID == "" || config.ProtocolVersion < 1 || config.ImageDigest == "") {
return nil, fmt.Errorf("control-plane registration requires a server ID, protocol version and image digest")
}
if config.RosterPath != "" && config.ControlPlaneURL == "" {
return nil, fmt.Errorf("roster path requires control-plane URL")
}
// Neither MatchID nor WorkloadTokenPath is required here: both can
// instead be resolved at Start time from the allocated GameServer's own
// annotations (see registerControlPlane/workloadToken/matchID). They are
@@ -172,6 +182,9 @@ func (s *Supervisor) Start(ctx context.Context) error {
if s.config.Transport == "steam_sdr" {
env = append(env, "SDR_LISTEN_PORT="+strconv.Itoa(port), "SDR_IP="+address+":"+strconv.Itoa(port))
}
if err := s.fetchRoster(ctx); err != nil {
return err
}
command := withPort(s.config.Command, port)
s.cmd = exec.CommandContext(ctx, command[0], command[1:]...)
} else {
@@ -205,6 +218,70 @@ func (s *Supervisor) Start(ctx context.Context) error {
return nil
}
func (s *Supervisor) fetchRoster(ctx context.Context) error {
if s.config.RosterPath == "" {
return nil
}
matchID := s.matchID()
if matchID == "" {
return fmt.Errorf("roster fetch has no match ID")
}
token, err := s.workloadToken()
if err != nil {
return err
}
rosterURL := strings.TrimRight(s.config.ControlPlaneURL, "/") + "/v1/servers/" + url.PathEscape(s.config.ServerID) + "/roster"
request, err := http.NewRequestWithContext(ctx, http.MethodGet, rosterURL, nil)
if err != nil {
return err
}
request.Header.Set("Authorization", "Bearer "+token)
response, err := s.client.Do(request)
if err != nil {
return err
}
defer response.Body.Close()
if response.StatusCode/100 != 2 {
return fmt.Errorf("control-plane roster returned %s", response.Status)
}
var roster []json.RawMessage
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&roster); err != nil || len(roster) == 0 {
if err == nil {
err = fmt.Errorf("empty roster")
}
return fmt.Errorf("decode control-plane roster: %w", err)
}
for _, envelope := range roster {
if len(envelope) == 0 || string(envelope) == "null" {
return fmt.Errorf("control-plane roster contains an invalid envelope")
}
}
contents, err := json.Marshal(roster)
if err != nil {
return fmt.Errorf("encode roster: %w", err)
}
directory := filepath.Dir(s.config.RosterPath)
temporary, err := os.CreateTemp(directory, ".cosmic-clash-roster-*")
if err != nil {
return fmt.Errorf("create roster file: %w", err)
}
temporaryName := temporary.Name()
defer os.Remove(temporaryName)
if err := temporary.Chmod(0600); err == nil {
_, err = temporary.Write(contents)
}
if closeErr := temporary.Close(); err == nil {
err = closeErr
}
if err != nil {
return fmt.Errorf("write roster file: %w", err)
}
if err := os.Rename(temporaryName, s.config.RosterPath); err != nil {
return fmt.Errorf("install roster file: %w", err)
}
return nil
}
// reportAssignmentReady is best-effort: process-ready has already succeeded,
// so the process is legitimately usable either way. A persistent failure is
// written to stderr rather than returned, since treating it as fatal would
@@ -5,11 +5,13 @@ package supervisor
import (
"context"
"database/sql"
"encoding/base64"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -78,7 +80,7 @@ func TestRealSupervisorRegistersAllocatedServerThroughControlPlane(t *testing.T)
if err := store.SaveAssignment(ctx, db, store.DurableAssignment{
MatchID: "supervisor-live-match", PlayerID: player, AllocationID: request.AllocationID, ServerID: "supervisor-live-server", Slot: index,
Region: "EU", ClientBuild: "build-1", ProtocolVersion: 1, Transport: "enet", Endpoint: "127.0.0.1:7777",
JoinAuthorisation: "join-" + player, ManifestDigest: []byte{0, 1, 2, 3}, ExpiresAt: now.Add(time.Hour), Revision: 1,
JoinAuthorisation: base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf(`{"authorisation":{"match_id":"supervisor-live-match","server_id":"supervisor-live-server","player_id":%q},"signature":"sig"}`, player))), ManifestDigest: []byte{0, 1, 2, 3}, ExpiresAt: now.Add(time.Hour), Revision: 1,
}); err != nil {
t.Fatal(err)
}
@@ -99,13 +101,16 @@ func TestRealSupervisorRegistersAllocatedServerThroughControlPlane(t *testing.T)
}
}))
defer sdk.Close()
service := &api.Service{ServerRegistrar: api.ServerRegistrarFromStore(db), WorkloadVerify: api.WorkloadVerifierFromSignedToken(secret, db), Now: func() time.Time { return now }}
rosterPath := filepath.Join(t.TempDir(), "join-roster.json")
service := &api.Service{ServerRegistrar: api.ServerRegistrarFromStore(db), WorkloadVerify: api.WorkloadVerifierFromSignedToken(secret, db), Roster: func(ctx context.Context, binding domain.WorkloadBinding, at time.Time) ([][]byte, error) {
return store.GetAssignmentRoster(ctx, db, binding.MatchID, binding.ServerID, at)
}, Now: func() time.Time { return now }}
control := httptest.NewServer(service.Handler())
defer control.Close()
supervisor, err := New(Config{
Command: []string{"/bin/sh", "-c", "true"}, SDKBaseURL: sdk.URL, ReadyURL: sdk.URL + "/ready-probe", ControlPlaneURL: control.URL,
ServerID: "supervisor-live-server", ProtocolVersion: 1, ImageDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
ReadyTimeout: time.Second, PollInterval: time.Millisecond, AssignmentReadyAttempts: 1,
ReadyTimeout: time.Second, PollInterval: time.Millisecond, AssignmentReadyAttempts: 1, RosterPath: rosterPath,
})
if err != nil {
t.Fatal(err)
@@ -116,6 +121,10 @@ func TestRealSupervisorRegistersAllocatedServerThroughControlPlane(t *testing.T)
if err := supervisor.Wait(); err != nil {
t.Fatal(err)
}
roster, err := os.ReadFile(rosterPath)
if err != nil || !strings.Contains(string(roster), "supervisor-live-a") || !strings.Contains(string(roster), "supervisor-live-b") {
t.Fatalf("materialized live roster=%q err=%v", roster, err)
}
var matchState, ticketState string
if err := db.QueryRowContext(ctx, `SELECT state FROM matches WHERE match_id = 'supervisor-live-match'`).Scan(&matchState); err != nil {
t.Fatal(err)
+50
View File
@@ -68,6 +68,56 @@ func TestAllocatedStartInjectsDynamicEndpointAndCallsReadyAfterProbe(t *testing.
}
}
func TestAllocatedStartMaterializesWorkloadAuthenticatedRosterBeforeChild(t *testing.T) {
rosterPath := filepath.Join(t.TempDir(), "join-roster.json")
sdk := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/gameserver":
_, _ = w.Write([]byte(`{"object_meta":{"annotations":{"cosmic-clash.io/match-id":"match-1","cosmic-clash.io/workload-token":"workload-token"}},"status":{"address":"127.0.0.1","ports":[{"name":"game","port":31001}]}}`))
case "/ready-probe", "/ready":
w.WriteHeader(http.StatusOK)
default:
w.WriteHeader(http.StatusNotFound)
}
}))
defer sdk.Close()
controlPlane := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/servers/server-1/roster" {
if r.Method != http.MethodGet || r.Header.Get("Authorization") != "Bearer workload-token" {
w.WriteHeader(http.StatusUnauthorized)
return
}
_, _ = w.Write([]byte(`[{"authorisation":{"player_id":"player-1"},"signature":"sig"}]`))
return
}
if r.URL.Path == "/v1/servers/server-1/register" {
w.WriteHeader(http.StatusNoContent)
return
}
w.WriteHeader(http.StatusNotFound)
}))
defer controlPlane.Close()
command := []string{"/bin/sh", "-c", "test -s '" + rosterPath + "'"}
s, err := New(Config{
Command: command, SDKBaseURL: sdk.URL, ReadyURL: sdk.URL + "/ready-probe", ControlPlaneURL: controlPlane.URL,
ServerID: "server-1", ProtocolVersion: 1, ImageDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
RosterPath: rosterPath, ReadyTimeout: time.Second, PollInterval: time.Millisecond,
})
if err != nil {
t.Fatal(err)
}
if err := s.Start(context.Background()); err != nil {
t.Fatal(err)
}
if err := s.Wait(); err != nil {
t.Fatal(err)
}
contents, err := os.ReadFile(rosterPath)
if err != nil || !strings.Contains(string(contents), "player-1") {
t.Fatalf("materialized roster=%q err=%v", contents, err)
}
}
func TestControlPlaneRegistrationRejectsIncompleteConfig(t *testing.T) {
base := Config{Command: []string{"/bin/true"}, ControlPlaneURL: "https://control-plane.invalid"}
if _, err := New(base); err == nil {