feat(multiplayer): acknowledge supervisor shutdown

multiplayer-next.md

@@ -1432,3 +1432,5 @@ Clients now consume planned shutdowns: the reason is retained for presentation,
 An adversarial UI review found the lobby’s generic disconnect handler still replaced that message with the main menu immediately afterward. Planned disconnects are now fenced in the lobby, and a lobby reached from an active match restores the retained reason on startup; unplanned disconnects keep the existing main-menu behavior.
 
 The workload-authenticated `POST /servers/{serverId}/shutdown` contract is now exposed for allocated servers. It validates the bound credential and reason, records an idempotent `SERVER_SHUTDOWN` audit event under a serializable transaction, and returns a stable acknowledgment on retry; match-state transitions remain owned by the no-show/result transactions. API/store tests cover authorization, validation, idempotency SQL, and audit wiring; live PostgreSQL delivery remains an integration gate.
+
+The allocated supervisor now calls that shutdown acknowledgment during signal-bound controlled drain, using the same workload credential and a deterministic idempotency key after the local drain request succeeds. The lifecycle test verifies the drain-before-ack ordering, credential separation, and bounded graceful child exit; live pod termination and control-plane outage behavior remain deployment gates.

server/cmd/game-server-supervisor/main.go

@@ -16,7 +16,8 @@ const usageText = `Usage: game-server-supervisor [options] -- <game-server-comma
 
 The child command is started only after an allocated Agones endpoint has been
 validated and, when configured, an explicit process-ready probe succeeds.
-SIGTERM/SIGINT requests authenticated drain before the bounded grace deadline.
+SIGTERM/SIGINT requests authenticated drain, acknowledges planned shutdown to
+the control plane when configured, and then enforces the bounded grace deadline.
 `
 
 func main() {

server/supervisor/supervisor.go

@@ -499,6 +499,12 @@ func (s *Supervisor) Run(ctx context.Context, drainGrace time.Duration) error {
 		drainErr = s.Drain(drainCtx)
 		drainCancel()
 	}
+	var shutdownErr error
+	if s.config.ControlPlaneURL != "" && drainErr == nil {
+		shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
+		shutdownErr = s.acknowledgeShutdown(shutdownCtx, "server_draining")
+		shutdownCancel()
+	}
 	timer := time.NewTimer(drainGrace)
 	defer timer.Stop()
 	select {
@@ -506,6 +512,9 @@ func (s *Supervisor) Run(ctx context.Context, drainGrace time.Duration) error {
 		if drainErr != nil {
 			return fmt.Errorf("child exited after drain failure: %w", drainErr)
 		}
+		if shutdownErr != nil {
+			return fmt.Errorf("child exited after shutdown acknowledgement failure: %w", shutdownErr)
+		}
 		return err
 	case <-timer.C:
 		if s.cmd != nil && s.cmd.Process != nil {
@@ -515,6 +524,9 @@ func (s *Supervisor) Run(ctx context.Context, drainGrace time.Duration) error {
 		if drainErr != nil {
 			return fmt.Errorf("drain failed and child was force-killed: %w", drainErr)
 		}
+		if shutdownErr != nil {
+			return fmt.Errorf("shutdown acknowledgement failed and child was force-killed: %w", shutdownErr)
+		}
 		return fmt.Errorf("child force-killed after drain deadline")
 	}
 }
@@ -542,6 +554,50 @@ func (s *Supervisor) Drain(ctx context.Context) error {
 	return nil
 }
 
+// acknowledgeShutdown records the supervisor's planned termination after the
+// local game process has been told to drain. It uses the same bound workload
+// credential as registration. The idempotency key makes a repeated call safe.
+func (s *Supervisor) acknowledgeShutdown(ctx context.Context, reason string) error {
+	if s.config.ControlPlaneURL == "" {
+		return nil
+	}
+	matchID := s.matchID()
+	if matchID == "" {
+		return fmt.Errorf("shutdown acknowledgement has no match ID")
+	}
+	token, err := s.workloadToken()
+	if err != nil {
+		return err
+	}
+	body, err := json.Marshal(struct {
+		Reason string `json:"reason"`
+	}{reason})
+	if err != nil {
+		return err
+	}
+	endpoint := strings.TrimRight(s.config.ControlPlaneURL, "/") + "/v1/servers/" + url.PathEscape(s.config.ServerID) + "/shutdown"
+	request, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))
+	if err != nil {
+		return err
+	}
+	request.Header.Set("Content-Type", "application/json")
+	request.Header.Set("Authorization", "Bearer "+token)
+	key := "supervisor-shutdown-" + s.config.ServerID + "-" + matchID + "-" + reason
+	if len(key) > 128 {
+		key = key[:128]
+	}
+	request.Header.Set("Idempotency-Key", key)
+	response, err := s.client.Do(request)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	if response.StatusCode/100 != 2 {
+		return fmt.Errorf("control-plane shutdown returned %s", response.Status)
+	}
+	return nil
+}
+
 func (s *Supervisor) assignedEndpoint(ctx context.Context) (int, string, error) {
 	var server GameServer
 	if err := s.sdkGet(ctx, "/gameserver", &server); err != nil {

server/supervisor/supervisor_test.go

@@ -629,6 +629,82 @@ func TestRunDrainsBeforeChildExit(t *testing.T) {
 	}
 }
 
+func TestRunAcknowledgesControlledShutdownWithWorkloadCredential(t *testing.T) {
+	marker := filepath.Join(t.TempDir(), "drained")
+	tokenPath := filepath.Join(t.TempDir(), "token")
+	if err := os.WriteFile(tokenPath, []byte("workload-secret"), 0600); err != nil {
+		t.Fatal(err)
+	}
+	var shutdownCalls int
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/drain":
+			if r.Header.Get("Authorization") != "Bearer run-secret" {
+				w.WriteHeader(http.StatusUnauthorized)
+				return
+			}
+			if err := os.WriteFile(marker, []byte("drained"), 0600); err != nil {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
+			w.WriteHeader(http.StatusAccepted)
+		case "/v1/servers/server-1/shutdown":
+			if r.Header.Get("Authorization") != "Bearer workload-secret" || r.Header.Get("Idempotency-Key") == "" {
+				w.WriteHeader(http.StatusUnauthorized)
+				return
+			}
+			shutdownCalls++
+			w.WriteHeader(http.StatusNoContent)
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer server.Close()
+	s, err := New(Config{
+		Command:  []string{"/bin/sh", "-c", "while [ ! -f '" + marker + "' ]; do sleep 0.01; done"},
+		DrainURL: server.URL + "/drain", DrainToken: "run-secret", ControlPlaneURL: server.URL,
+		WorkloadTokenPath: tokenPath, ServerID: "server-1", MatchID: "match-1", ProtocolVersion: 1,
+		ImageDigest: "sha256:aa",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	time.AfterFunc(30*time.Millisecond, cancel)
+	if err := s.Run(ctx, time.Second); err != nil {
+		t.Fatalf("graceful run: %v", err)
+	}
+	if shutdownCalls != 1 {
+		t.Fatalf("shutdown calls = %d, want 1", shutdownCalls)
+	}
+}
+
+func TestRunDoesNotAcknowledgeWhenLocalDrainFails(t *testing.T) {
+	var shutdownCalls int
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/v1/servers/server-1/shutdown" {
+			shutdownCalls++
+		}
+		w.WriteHeader(http.StatusInternalServerError)
+	}))
+	defer server.Close()
+	s, err := New(Config{
+		Command:  []string{"/bin/sh", "-c", "sleep 5"},
+		DrainURL: server.URL + "/drain", DrainToken: "run-secret", ControlPlaneURL: server.URL,
+		WorkloadTokenPath: filepath.Join(t.TempDir(), "missing-token"), ServerID: "server-1", MatchID: "match-1",
+		ProtocolVersion: 1, ImageDigest: "sha256:aa",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	time.AfterFunc(30*time.Millisecond, cancel)
+	err = s.Run(ctx, 50*time.Millisecond)
+	if err == nil || shutdownCalls != 0 {
+		t.Fatalf("failed drain result=%v shutdown calls=%d", err, shutdownCalls)
+	}
+}
+
 func TestRunForceKillsUnresponsiveChildAtDeadline(t *testing.T) {
 	s, err := New(Config{Command: []string{"/bin/sh", "-c", "trap '' TERM; sleep 5"}})
 	if err != nil {