fix(multiplayer): reconcile authoritative initial connections

This commit is contained in:
Josh Creek
2026-09-03 00:02:04 +01:00
parent 781cbc35aa
commit aa446cfbfe
40 changed files with 809 additions and 87 deletions
+88 -14
View File
@@ -51,6 +51,7 @@ type Config struct {
ReadyURL string
Transport string
DrainURL string
AdmissionURL string
DrainToken string
ReadyTimeout time.Duration
PollInterval time.Duration
@@ -87,9 +88,8 @@ type Config struct {
// holding a live, unexpired assignment -- see
// AdvanceServerRegistrationSQL) may not be satisfied on the very first
// attempt if the signed roster is still propagating, and that is
// expected, not fatal: unlike a process-ready registration failure, this
// does not kill the child, since the process is already legitimately
// listening and usable either way. Default 5 attempts, 2s apart.
// expected during the bounded retries. Exhaustion is fatal because player
// assignments remain hidden until this transition. Default 5 attempts, 2s apart.
AssignmentReadyAttempts int
AssignmentReadyBackoff time.Duration
// RosterPath is an operator-mounted writable path where the supervisor
@@ -106,6 +106,12 @@ type Supervisor struct {
lastGameServer GameServer
}
const (
ChildControlPlaneURLEnv = "COSMIC_CLASH_CONTROL_PLANE_URL"
ChildWorkloadTokenEnv = "COSMIC_CLASH_WORKLOAD_TOKEN"
ChildAdmissionSignalEnv = "COSMIC_CLASH_INITIAL_CONNECT_SIGNAL_REQUIRED"
)
const (
DefaultDrainGrace = 285 * time.Second
DefaultHTTPTimeout = 10 * time.Second
@@ -144,9 +150,23 @@ func New(config Config) (*Supervisor, error) {
return nil, err
}
}
if config.AdmissionURL != "" {
if config.DrainToken == "" {
return nil, fmt.Errorf("initial-connect admission URL requires a control token")
}
if err := validateLocalDrainURL(config.AdmissionURL); err != nil {
return nil, fmt.Errorf("invalid initial-connect admission URL: %w", err)
}
}
if config.ControlPlaneURL != "" && (config.ServerID == "" || config.ProtocolVersion < 1 || config.ImageDigest == "") {
return nil, fmt.Errorf("control-plane registration requires a server ID, protocol version and image digest")
}
if config.ControlPlaneURL != "" {
parsed, err := url.Parse(config.ControlPlaneURL)
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || parsed.Path != "" {
return nil, fmt.Errorf("control-plane URL must be an HTTP(S) origin")
}
}
if config.RosterPath != "" && config.ControlPlaneURL == "" {
return nil, fmt.Errorf("roster path requires control-plane URL")
}
@@ -190,6 +210,11 @@ func (s *Supervisor) Start(ctx context.Context) error {
if err != nil {
return err
}
childControlPlaneEnv, err := s.controlPlaneChildEnvironment()
if err != nil {
return err
}
env = append(env, childControlPlaneEnv...)
command := withAllocatedConfig(s.config.Command, s.matchID(), s.config.ServerID, s.config.ImageDigest, rosterExpiry)
command, err = withAllocatedCompatibility(command, s.lastGameServer)
if err != nil {
@@ -224,10 +249,61 @@ func (s *Supervisor) Start(ctx context.Context) error {
_ = s.cmd.Process.Kill()
return err
}
s.reportAssignmentReady(ctx)
if err := s.reportAssignmentReady(ctx); err != nil {
// Player assignments remain deliberately hidden until this durable
// transition succeeds. Do not leave an Agones-Ready process accepting
// connections for a match the control plane cannot expose.
_ = s.cmd.Process.Kill()
return err
}
if err := s.signalInitialConnectReady(ctx); err != nil {
_ = s.cmd.Process.Kill()
return err
}
return nil
}
func (s *Supervisor) signalInitialConnectReady(ctx context.Context) error {
if s.config.AdmissionURL == "" {
return nil
}
request, err := http.NewRequestWithContext(ctx, http.MethodPost, s.config.AdmissionURL, nil)
if err != nil {
return err
}
request.Header.Set("Authorization", "Bearer "+s.config.DrainToken)
response, err := s.client.Do(request)
if err != nil {
return err
}
defer response.Body.Close()
if response.StatusCode/100 != 2 {
return fmt.Errorf("initial-connect admission control returned %s", response.Status)
}
return nil
}
func (s *Supervisor) controlPlaneChildEnvironment() ([]string, error) {
if s.config.ControlPlaneURL == "" {
return nil, nil
}
token, err := s.workloadToken()
if err != nil {
return nil, err
}
if strings.ContainsRune(token, '\x00') {
return nil, fmt.Errorf("workload token contains an invalid environment byte")
}
environment := []string{
ChildControlPlaneURLEnv + "=" + strings.TrimRight(s.config.ControlPlaneURL, "/"),
ChildWorkloadTokenEnv + "=" + token,
}
if s.config.AdmissionURL != "" {
environment = append(environment, ChildAdmissionSignalEnv+"=1")
}
return environment, nil
}
func (s *Supervisor) fetchRoster(ctx context.Context) (time.Time, error) {
if s.config.RosterPath == "" {
return time.Time{}, nil
@@ -387,29 +463,27 @@ func withAllocatedValues(command []string, values map[string]string) []string {
return result
}
// reportAssignmentReady is best-effort: process-ready has already succeeded,
// so the process is legitimately usable either way. A persistent failure is
// written to stderr rather than returned, since treating it as fatal would
// kill a perfectly healthy process over what is usually just the signed
// roster's durable rows not having propagated yet.
func (s *Supervisor) reportAssignmentReady(ctx context.Context) {
// reportAssignmentReady retries the durable gate that makes player
// assignments visible. A process without this transition is not usable even
// when Agones and the local readiness probe consider it healthy.
func (s *Supervisor) reportAssignmentReady(ctx context.Context) error {
if s.config.ControlPlaneURL == "" {
return
return nil
}
var lastErr error
for attempt := 0; attempt < s.config.AssignmentReadyAttempts; attempt++ {
if attempt > 0 {
select {
case <-ctx.Done():
return
return ctx.Err()
case <-time.After(s.config.AssignmentReadyBackoff):
}
}
if lastErr = s.registerControlPlane(ctx, true); lastErr == nil {
return
return nil
}
}
fmt.Fprintf(os.Stderr, "game-server-supervisor: assignment-ready registration did not succeed after %d attempts: %v\n", s.config.AssignmentReadyAttempts, lastErr)
return fmt.Errorf("assignment-ready registration did not succeed after %d attempts: %w", s.config.AssignmentReadyAttempts, lastErr)
}
// registerControlPlane reports the allocated process's readiness to the