fix(multiplayer): reconcile authoritative initial connections

2026-09-11 00:14:00 +00:00 · 2026-09-03 00:02:04 +01:00
parent 781cbc35aa
commit aa446cfbfe
40 changed files with 809 additions and 87 deletions
@@ -51,6 +51,7 @@ type Config struct {
 	ReadyURL     string
 	Transport    string
 	DrainURL     string
+	AdmissionURL string
 	DrainToken   string
 	ReadyTimeout time.Duration
 	PollInterval time.Duration
@@ -87,9 +88,8 @@ type Config struct {
 	// holding a live, unexpired assignment -- see
 	// AdvanceServerRegistrationSQL) may not be satisfied on the very first
 	// attempt if the signed roster is still propagating, and that is
-	// expected, not fatal: unlike a process-ready registration failure, this
-	// does not kill the child, since the process is already legitimately
-	// listening and usable either way. Default 5 attempts, 2s apart.
+	// expected during the bounded retries. Exhaustion is fatal because player
+	// assignments remain hidden until this transition. Default 5 attempts, 2s apart.
 	AssignmentReadyAttempts int
 	AssignmentReadyBackoff  time.Duration
 	// RosterPath is an operator-mounted writable path where the supervisor
@@ -106,6 +106,12 @@ type Supervisor struct {
 	lastGameServer GameServer
 }

+const (
+	ChildControlPlaneURLEnv = "COSMIC_CLASH_CONTROL_PLANE_URL"
+	ChildWorkloadTokenEnv   = "COSMIC_CLASH_WORKLOAD_TOKEN"
+	ChildAdmissionSignalEnv = "COSMIC_CLASH_INITIAL_CONNECT_SIGNAL_REQUIRED"
+)
+
 const (
 	DefaultDrainGrace  = 285 * time.Second
 	DefaultHTTPTimeout = 10 * time.Second
@@ -144,9 +150,23 @@ func New(config Config) (*Supervisor, error) {
 			return nil, err
 		}
 	}
+	if config.AdmissionURL != "" {
+		if config.DrainToken == "" {
+			return nil, fmt.Errorf("initial-connect admission URL requires a control token")
+		}
+		if err := validateLocalDrainURL(config.AdmissionURL); err != nil {
+			return nil, fmt.Errorf("invalid initial-connect admission URL: %w", err)
+		}
+	}
 	if config.ControlPlaneURL != "" && (config.ServerID == "" || config.ProtocolVersion < 1 || config.ImageDigest == "") {
 		return nil, fmt.Errorf("control-plane registration requires a server ID, protocol version and image digest")
 	}
+	if config.ControlPlaneURL != "" {
+		parsed, err := url.Parse(config.ControlPlaneURL)
+		if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || parsed.Path != "" {
+			return nil, fmt.Errorf("control-plane URL must be an HTTP(S) origin")
+		}
+	}
 	if config.RosterPath != "" && config.ControlPlaneURL == "" {
 		return nil, fmt.Errorf("roster path requires control-plane URL")
 	}
@@ -190,6 +210,11 @@ func (s *Supervisor) Start(ctx context.Context) error {
 		if err != nil {
 			return err
 		}
+		childControlPlaneEnv, err := s.controlPlaneChildEnvironment()
+		if err != nil {
+			return err
+		}
+		env = append(env, childControlPlaneEnv...)
 		command := withAllocatedConfig(s.config.Command, s.matchID(), s.config.ServerID, s.config.ImageDigest, rosterExpiry)
 		command, err = withAllocatedCompatibility(command, s.lastGameServer)
 		if err != nil {
@@ -224,10 +249,61 @@ func (s *Supervisor) Start(ctx context.Context) error {
 		_ = s.cmd.Process.Kill()
 		return err
 	}
-	s.reportAssignmentReady(ctx)
+	if err := s.reportAssignmentReady(ctx); err != nil {
+		// Player assignments remain deliberately hidden until this durable
+		// transition succeeds. Do not leave an Agones-Ready process accepting
+		// connections for a match the control plane cannot expose.
+		_ = s.cmd.Process.Kill()
+		return err
+	}
+	if err := s.signalInitialConnectReady(ctx); err != nil {
+		_ = s.cmd.Process.Kill()
+		return err
+	}
 	return nil
 }

+func (s *Supervisor) signalInitialConnectReady(ctx context.Context) error {
+	if s.config.AdmissionURL == "" {
+		return nil
+	}
+	request, err := http.NewRequestWithContext(ctx, http.MethodPost, s.config.AdmissionURL, nil)
+	if err != nil {
+		return err
+	}
+	request.Header.Set("Authorization", "Bearer "+s.config.DrainToken)
+	response, err := s.client.Do(request)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	if response.StatusCode/100 != 2 {
+		return fmt.Errorf("initial-connect admission control returned %s", response.Status)
+	}
+	return nil
+}
+
+func (s *Supervisor) controlPlaneChildEnvironment() ([]string, error) {
+	if s.config.ControlPlaneURL == "" {
+		return nil, nil
+	}
+	token, err := s.workloadToken()
+	if err != nil {
+		return nil, err
+	}
+	if strings.ContainsRune(token, '\x00') {
+		return nil, fmt.Errorf("workload token contains an invalid environment byte")
+	}
+	environment := []string{
+		ChildControlPlaneURLEnv + "=" + strings.TrimRight(s.config.ControlPlaneURL, "/"),
+		ChildWorkloadTokenEnv + "=" + token,
+	}
+	if s.config.AdmissionURL != "" {
+		environment = append(environment, ChildAdmissionSignalEnv+"=1")
+	}
+	return environment, nil
+}
+
 func (s *Supervisor) fetchRoster(ctx context.Context) (time.Time, error) {
 	if s.config.RosterPath == "" {
 		return time.Time{}, nil
@@ -387,29 +463,27 @@ func withAllocatedValues(command []string, values map[string]string) []string {
 	return result
 }

-// reportAssignmentReady is best-effort: process-ready has already succeeded,
-// so the process is legitimately usable either way. A persistent failure is
-// written to stderr rather than returned, since treating it as fatal would
-// kill a perfectly healthy process over what is usually just the signed
-// roster's durable rows not having propagated yet.
-func (s *Supervisor) reportAssignmentReady(ctx context.Context) {
+// reportAssignmentReady retries the durable gate that makes player
+// assignments visible. A process without this transition is not usable even
+// when Agones and the local readiness probe consider it healthy.
+func (s *Supervisor) reportAssignmentReady(ctx context.Context) error {
 	if s.config.ControlPlaneURL == "" {
-		return
+		return nil
 	}
 	var lastErr error
 	for attempt := 0; attempt < s.config.AssignmentReadyAttempts; attempt++ {
 		if attempt > 0 {
 			select {
 			case <-ctx.Done():
-				return
+				return ctx.Err()
 			case <-time.After(s.config.AssignmentReadyBackoff):
 			}
 		}
 		if lastErr = s.registerControlPlane(ctx, true); lastErr == nil {
-			return
+			return nil
 		}
 	}
-	fmt.Fprintf(os.Stderr, "game-server-supervisor: assignment-ready registration did not succeed after %d attempts: %v\n", s.config.AssignmentReadyAttempts, lastErr)
+	return fmt.Errorf("assignment-ready registration did not succeed after %d attempts: %w", s.config.AssignmentReadyAttempts, lastErr)
 }

 // registerControlPlane reports the allocated process's readiness to the
@@ -23,6 +23,41 @@ func TestSupervisorDefaultHTTPClientHasRequestDeadline(t *testing.T) {
 	}
 }

+func TestAllocatedChildReceivesConnectionReportingEnvironmentWithoutCommandSecrets(t *testing.T) {
+	s, err := New(Config{
+		Command: []string{"game-server"}, ControlPlaneURL: "https://control.example",
+		ServerID: "server-1", ProtocolVersion: 1, ImageDigest: "sha256:" + strings.Repeat("a", 64),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	s.lastGameServer.ObjectMeta.Annotations = map[string]string{"cosmic-clash.io/workload-token": "signed-workload-token"}
+	environment, err := s.controlPlaneChildEnvironment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	joined := strings.Join(environment, "\n")
+	if !strings.Contains(joined, ChildControlPlaneURLEnv+"=https://control.example") || !strings.Contains(joined, ChildWorkloadTokenEnv+"=signed-workload-token") || strings.Contains(joined, ChildAdmissionSignalEnv) {
+		t.Fatalf("child connection-reporting environment = %v", environment)
+	}
+	if strings.Contains(strings.Join(s.config.Command, " "), "signed-workload-token") {
+		t.Fatal("workload token leaked into child command arguments")
+	}
+	s.config.AdmissionURL = "http://127.0.0.1:7780/initial-connect-ready"
+	environment, err = s.controlPlaneChildEnvironment()
+	if err != nil || !strings.Contains(strings.Join(environment, "\n"), ChildAdmissionSignalEnv+"=1") {
+		t.Fatalf("child admission signal environment = %v err=%v", environment, err)
+	}
+}
+
+func TestSupervisorRejectsUnsafeControlPlaneOrigins(t *testing.T) {
+	for _, raw := range []string{"control.example", "https://user:secret@control.example", "https://control.example/path", "https://control.example?token=secret"} {
+		if _, err := New(Config{Command: []string{"game-server"}, ControlPlaneURL: raw, ServerID: "server-1", ProtocolVersion: 1, ImageDigest: "sha256:" + strings.Repeat("a", 64)}); err == nil {
+			t.Fatalf("unsafe control-plane URL accepted: %q", raw)
+		}
+	}
+}
+
 func TestWithAllocatedConfigOverridesAuthoritativeChildFlags(t *testing.T) {
 	command := []string{
 		"game-server", "--", "--allocated-mode", "--match-id=stale-match",
@@ -265,6 +300,7 @@ func TestControlPlaneRegistrationReportsProcessReadyThenAssignmentReady(t *testi
 func TestAssignmentReadyRegistrationRetriesUntilTheControlPlaneCatchesUp(t *testing.T) {
 	var mu sync.Mutex
 	assignmentReadyAttempts := 0
+	admissionCalled := false
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		switch {
 		case r.URL.Path == "/gameserver":
@@ -288,6 +324,15 @@ func TestAssignmentReadyRegistrationRetriesUntilTheControlPlaneCatchesUp(t *test
 				return
 			}
 			w.WriteHeader(http.StatusNoContent)
+		case r.URL.Path == "/initial-connect-ready":
+			mu.Lock()
+			defer mu.Unlock()
+			if assignmentReadyAttempts != 3 || r.Header.Get("Authorization") != "Bearer control-token-123456" {
+				w.WriteHeader(http.StatusConflict)
+				return
+			}
+			admissionCalled = true
+			w.WriteHeader(http.StatusAccepted)
 		default:
 			w.WriteHeader(http.StatusNotFound)
 		}
@@ -301,13 +346,14 @@ func TestAssignmentReadyRegistrationRetriesUntilTheControlPlaneCatchesUp(t *test
 	s, err := New(Config{
 		Command: []string{"/bin/sh", "-c", "true"}, SDKBaseURL: server.URL, ReadyURL: server.URL + "/ready-probe", ReadyTimeout: time.Second, PollInterval: time.Millisecond,
 		ControlPlaneURL: server.URL, WorkloadTokenPath: tokenPath, ServerID: "server-1", MatchID: "match-1", ProtocolVersion: 1, ImageDigest: "sha256:aa",
+		DrainURL: server.URL + "/drain", AdmissionURL: server.URL + "/initial-connect-ready", DrainToken: "control-token-123456",
 		AssignmentReadyAttempts: 5, AssignmentReadyBackoff: time.Millisecond,
 	})
 	if err != nil {
 		t.Fatal(err)
 	}
-	// Start must still succeed -- a slow-to-propagate assignment-ready must
-	// never be treated as a Start() failure (which would kill the child).
+	// A transient conflict is retried inside Start; the assignment only becomes
+	// visible after the durable transition eventually succeeds.
 	if err := s.Start(context.Background()); err != nil {
 		t.Fatalf("Start failed despite assignment-ready eventually succeeding: %v", err)
 	}
@@ -319,6 +365,42 @@ func TestAssignmentReadyRegistrationRetriesUntilTheControlPlaneCatchesUp(t *test
 	if assignmentReadyAttempts != 3 {
 		t.Fatalf("assignment-ready attempts = %d, want exactly 3 (2 conflicts then success)", assignmentReadyAttempts)
 	}
+	if !admissionCalled {
+		t.Fatal("initial-connect clock was not armed after durable assignment readiness")
+	}
+}
+
+func TestPersistentAssignmentReadyFailureFailsClosed(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/gameserver":
+			_, _ = w.Write([]byte(`{"object_meta":{"annotations":{"cosmic-clash.io/match-id":"match-1","cosmic-clash.io/workload-token":"workload-token"}},"status":{"address":"127.0.0.1","ports":[{"name":"game","port":31001}]}}`))
+		case "/ready-probe", "/ready":
+			w.WriteHeader(http.StatusOK)
+		case "/v1/servers/server-1/register":
+			body, _ := io.ReadAll(r.Body)
+			if strings.Contains(string(body), `"assignment_ready":true`) {
+				w.WriteHeader(http.StatusConflict)
+				return
+			}
+			w.WriteHeader(http.StatusNoContent)
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer server.Close()
+	s, err := New(Config{
+		Command: []string{"/bin/sh", "-c", "sleep 30"}, SDKBaseURL: server.URL, ReadyURL: server.URL + "/ready-probe",
+		ControlPlaneURL: server.URL, ServerID: "server-1", ProtocolVersion: 1, ImageDigest: "sha256:aa",
+		ReadyTimeout: time.Second, PollInterval: time.Millisecond, AssignmentReadyAttempts: 2, AssignmentReadyBackoff: time.Millisecond,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := s.Start(context.Background()); err == nil || !strings.Contains(err.Error(), "assignment-ready registration did not succeed") {
+		t.Fatalf("persistent assignment-ready failure did not fail closed: %v", err)
+	}
+	_ = s.Wait()
 }

 func TestControlPlaneRegistrationFallsBackToGameServerAnnotationForMatchID(t *testing.T) {