diff --git a/multiplayer-next.md b/multiplayer-next.md index 9c14d854..e7f5f043 100644 --- a/multiplayer-next.md +++ b/multiplayer-next.md @@ -150,8 +150,9 @@ product policy are in [`docs/MATCHMAKING.md`](docs/MATCHMAKING.md). requests/limits and node density from measurements plus 30% headroom. - [ ] Add 30 s no-show handling, Go PID-1 TERM/drain supervision, PDB/Fleet drain, signed result annotation/retry, RPO <=5 m and RTO <=30 m. The Go - drain boundary is now authenticated and loopback-only, and the base PDB - protects the two-Ready floor; lifecycle/PDB/Fleet integration remains. + supervisor now owns bounded drain-before-kill orchestration, the drain + boundary is authenticated and loopback-only, and the base PDB protects the + two-Ready floor; lifecycle/PDB/Fleet integration remains. - [ ] Rehearse migration only after the second provider's EU/NA locations have Valve approval, POP/certs, public UDP/firewall and coordinator trust. diff --git a/multiplayer-todo.md b/multiplayer-todo.md index 398410d8..95fe158a 100644 --- a/multiplayer-todo.md +++ b/multiplayer-todo.md @@ -1219,7 +1219,7 @@ the local/CI/community transport, not a silent production fallback. | 8.33 `[D:8.26,8.32]` | **IN PROGRESS.** Fleet scheduling now requires on-demand capacity and spreads Ready processes across zones with skew 1; the autoscaler preserves the two-process Ready floor | `deploy/k8s/base/fleet.yaml` and manifest tests reject interruptible placement and single-zone concentration structurally; regional node pools, forced node-loss testing and measured N+1 headroom remain | | 8.34 `[D:8.28,8.29]` | Native x86_64 benchmark of boot-to-process-ready and assignment-ready, p99 CPU/RSS/network and 60 Hz ticks; limits/node cap with 30% headroom | Measurements replace old estimates and certify density with no tick backlog | | 8.35 `[D:8.17,8.19,8.20,8.30,8.31]` | **IN PROGRESS.** Pure Go initial-connect policy decides ranked 30 s no-show cancellation with abandon ladder and casual 60 s bot start only when each team has a human; empty-team casual allocations cancel | `server/domain/noshow.go` covers wait/deadline boundaries, deterministic no-show/innocent ordering, ranked cooldown history and no pre-live rating action; persistent ticket restoration, allocation shutdown, bot spawn and live integration remain | -| 8.36 `[D:8.10,8.25,8.28,8.30]` | **IN PROGRESS.** Supervisor exposes an authenticated loopback-only drain request boundary that never places the token in command arguments/logs and rejects remote/partial/query-bearing configurations; the base now includes a two-Ready PodDisruptionBudget | `server/supervisor/`, `server_control.gd` and `deploy/k8s/base/game-server-pdb.yaml` cover bearer-token enforcement, loopback URL validation, secret-safe configuration, missing drain credentials, readiness transitions and Ready-floor disruption protection; TERM signal handling, 300 s/285 s lifecycle, live PDB/Fleet drain and infrastructure-abort classification remain | +| 8.36 `[D:8.10,8.25,8.28,8.30]` | **IN PROGRESS.** Supervisor exposes an authenticated loopback-only drain request boundary that never places the token in command arguments/logs and rejects remote/partial/query-bearing configurations; the base now includes a two-Ready PodDisruptionBudget; `Supervisor.Run` now orchestrates drain-before-kill with a bounded grace deadline | `server/supervisor/`, `server_control.gd` and `deploy/k8s/base/game-server-pdb.yaml` cover bearer-token enforcement, loopback URL validation, secret-safe configuration, missing drain credentials, readiness transitions, Ready-floor disruption protection, graceful child exit after drain and force-kill of an unresponsive child; signal wiring in an executable supervisor, 300 s/285 s production lifecycle, live PDB/Fleet drain and infrastructure-abort classification remain | | 8.37 `[D:8.5,8.10,8.25,8.26,8.31]` | Horizontally scaled primary control plane + warm standby, EU/NA fleets, RPO <=5 m/RTO <=30 m; signed result annotation/reconciliation preserves delivery outages | Restore/failover meet targets; live simulation continues; valid delayed result commits exactly once after recovery | | 8.38 `[D:7.7,8.26,8.36,8.37]` | Provider migration after Valve approves both providers' EU/NA POPs/certs and public UDP: restore, validate coordinator trust, switch allocations, drain old | Both geographies complete Hosted-SDR matches on new provider and no old-provider live match is terminated | diff --git a/server/supervisor/supervisor.go b/server/supervisor/supervisor.go index bddce6bb..d0c22848 100644 --- a/server/supervisor/supervisor.go +++ b/server/supervisor/supervisor.go @@ -46,6 +46,8 @@ type Supervisor struct { cmd *exec.Cmd } +const DefaultDrainGrace = 285 * time.Second + func New(config Config) (*Supervisor, error) { if len(config.Command) == 0 || config.Command[0] == "" { return nil, fmt.Errorf("supervisor command is required") @@ -142,6 +144,58 @@ func (s *Supervisor) Wait() error { return s.cmd.Wait() } +// Run owns the PID-1 termination sequence. The child gets its own context so +// cancellation of the supervisor does not kill it before the authenticated +// drain request has had a chance to stop new admissions. A non-responsive +// child is force-killed after drainGrace; a drain failure is recorded only by +// the returned error if the child exits cleanly, while the deadline still +// prevents a stuck process from hanging termination forever. +func (s *Supervisor) Run(ctx context.Context, drainGrace time.Duration) error { + if s == nil || ctx == nil { + return fmt.Errorf("supervisor context is required") + } + if drainGrace <= 0 { + drainGrace = DefaultDrainGrace + } + processCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + if err := s.Start(processCtx); err != nil { + return err + } + wait := make(chan error, 1) + go func() { wait <- s.Wait() }() + select { + case err := <-wait: + return err + case <-ctx.Done(): + } + + var drainErr error + if s.config.DrainURL != "" { + drainCtx, drainCancel := context.WithTimeout(context.Background(), 5*time.Second) + drainErr = s.Drain(drainCtx) + drainCancel() + } + timer := time.NewTimer(drainGrace) + defer timer.Stop() + select { + case err := <-wait: + if drainErr != nil { + return fmt.Errorf("child exited after drain failure: %w", drainErr) + } + return err + case <-timer.C: + if s.cmd != nil && s.cmd.Process != nil { + _ = s.cmd.Process.Kill() + } + <-wait + if drainErr != nil { + return fmt.Errorf("drain failed and child was force-killed: %w", drainErr) + } + return fmt.Errorf("child force-killed after drain deadline") + } +} + // Drain asks the allocated Godot process to stop accepting new work. The // token is sent only over the configured localhost control endpoint and is // never placed in command arguments or logs. diff --git a/server/supervisor/supervisor_test.go b/server/supervisor/supervisor_test.go index dfe76872..eee9afc0 100644 --- a/server/supervisor/supervisor_test.go +++ b/server/supervisor/supervisor_test.go @@ -180,3 +180,53 @@ func TestSupervisorRejectsRemoteOrPartialDrainConfiguration(t *testing.T) { } } } + +func TestRunDrainsBeforeChildExit(t *testing.T) { + marker := filepath.Join(t.TempDir(), "drained") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/drain" { + w.WriteHeader(http.StatusNotFound) + return + } + if r.Header.Get("Authorization") != "Bearer run-secret" { + w.WriteHeader(http.StatusUnauthorized) + return + } + if err := os.WriteFile(marker, []byte("drained"), 0600); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusAccepted) + })) + defer server.Close() + command := []string{"/bin/sh", "-c", "while [ ! -f '" + marker + "' ]; do sleep 0.01; done"} + s, err := New(Config{Command: command, DrainURL: server.URL + "/drain", DrainToken: "run-secret"}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + time.AfterFunc(30*time.Millisecond, cancel) + if err := s.Run(ctx, time.Second); err != nil { + t.Fatalf("graceful run: %v", err) + } + if _, err := os.Stat(marker); err != nil { + t.Fatalf("drain endpoint was not called: %v", err) + } +} + +func TestRunForceKillsUnresponsiveChildAtDeadline(t *testing.T) { + s, err := New(Config{Command: []string{"/bin/sh", "-c", "trap '' TERM; sleep 5"}}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + time.AfterFunc(30*time.Millisecond, cancel) + started := time.Now() + err = s.Run(ctx, 50*time.Millisecond) + if err == nil || !strings.Contains(err.Error(), "force-killed") { + t.Fatalf("unresponsive child result = %v", err) + } + if elapsed := time.Since(started); elapsed > time.Second { + t.Fatalf("force-kill exceeded bounded deadline: %s", elapsed) + } +}