mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
feat(multiplayer): acknowledge supervisor shutdown
This commit is contained in:
@@ -499,6 +499,12 @@ func (s *Supervisor) Run(ctx context.Context, drainGrace time.Duration) error {
|
||||
drainErr = s.Drain(drainCtx)
|
||||
drainCancel()
|
||||
}
|
||||
var shutdownErr error
|
||||
if s.config.ControlPlaneURL != "" && drainErr == nil {
|
||||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
shutdownErr = s.acknowledgeShutdown(shutdownCtx, "server_draining")
|
||||
shutdownCancel()
|
||||
}
|
||||
timer := time.NewTimer(drainGrace)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
@@ -506,6 +512,9 @@ func (s *Supervisor) Run(ctx context.Context, drainGrace time.Duration) error {
|
||||
if drainErr != nil {
|
||||
return fmt.Errorf("child exited after drain failure: %w", drainErr)
|
||||
}
|
||||
if shutdownErr != nil {
|
||||
return fmt.Errorf("child exited after shutdown acknowledgement failure: %w", shutdownErr)
|
||||
}
|
||||
return err
|
||||
case <-timer.C:
|
||||
if s.cmd != nil && s.cmd.Process != nil {
|
||||
@@ -515,6 +524,9 @@ func (s *Supervisor) Run(ctx context.Context, drainGrace time.Duration) error {
|
||||
if drainErr != nil {
|
||||
return fmt.Errorf("drain failed and child was force-killed: %w", drainErr)
|
||||
}
|
||||
if shutdownErr != nil {
|
||||
return fmt.Errorf("shutdown acknowledgement failed and child was force-killed: %w", shutdownErr)
|
||||
}
|
||||
return fmt.Errorf("child force-killed after drain deadline")
|
||||
}
|
||||
}
|
||||
@@ -542,6 +554,50 @@ func (s *Supervisor) Drain(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// acknowledgeShutdown records the supervisor's planned termination after the
|
||||
// local game process has been told to drain. It uses the same bound workload
|
||||
// credential as registration. The idempotency key makes a repeated call safe.
|
||||
func (s *Supervisor) acknowledgeShutdown(ctx context.Context, reason string) error {
|
||||
if s.config.ControlPlaneURL == "" {
|
||||
return nil
|
||||
}
|
||||
matchID := s.matchID()
|
||||
if matchID == "" {
|
||||
return fmt.Errorf("shutdown acknowledgement has no match ID")
|
||||
}
|
||||
token, err := s.workloadToken()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := json.Marshal(struct {
|
||||
Reason string `json:"reason"`
|
||||
}{reason})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
endpoint := strings.TrimRight(s.config.ControlPlaneURL, "/") + "/v1/servers/" + url.PathEscape(s.config.ServerID) + "/shutdown"
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
request.Header.Set("Authorization", "Bearer "+token)
|
||||
key := "supervisor-shutdown-" + s.config.ServerID + "-" + matchID + "-" + reason
|
||||
if len(key) > 128 {
|
||||
key = key[:128]
|
||||
}
|
||||
request.Header.Set("Idempotency-Key", key)
|
||||
response, err := s.client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode/100 != 2 {
|
||||
return fmt.Errorf("control-plane shutdown returned %s", response.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Supervisor) assignedEndpoint(ctx context.Context) (int, string, error) {
|
||||
var server GameServer
|
||||
if err := s.sdkGet(ctx, "/gameserver", &server); err != nil {
|
||||
|
||||
@@ -629,6 +629,82 @@ func TestRunDrainsBeforeChildExit(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunAcknowledgesControlledShutdownWithWorkloadCredential(t *testing.T) {
|
||||
marker := filepath.Join(t.TempDir(), "drained")
|
||||
tokenPath := filepath.Join(t.TempDir(), "token")
|
||||
if err := os.WriteFile(tokenPath, []byte("workload-secret"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var shutdownCalls int
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/drain":
|
||||
if r.Header.Get("Authorization") != "Bearer run-secret" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
if err := os.WriteFile(marker, []byte("drained"), 0600); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
case "/v1/servers/server-1/shutdown":
|
||||
if r.Header.Get("Authorization") != "Bearer workload-secret" || r.Header.Get("Idempotency-Key") == "" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
shutdownCalls++
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
s, err := New(Config{
|
||||
Command: []string{"/bin/sh", "-c", "while [ ! -f '" + marker + "' ]; do sleep 0.01; done"},
|
||||
DrainURL: server.URL + "/drain", DrainToken: "run-secret", ControlPlaneURL: server.URL,
|
||||
WorkloadTokenPath: tokenPath, ServerID: "server-1", MatchID: "match-1", ProtocolVersion: 1,
|
||||
ImageDigest: "sha256:aa",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
time.AfterFunc(30*time.Millisecond, cancel)
|
||||
if err := s.Run(ctx, time.Second); err != nil {
|
||||
t.Fatalf("graceful run: %v", err)
|
||||
}
|
||||
if shutdownCalls != 1 {
|
||||
t.Fatalf("shutdown calls = %d, want 1", shutdownCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDoesNotAcknowledgeWhenLocalDrainFails(t *testing.T) {
|
||||
var shutdownCalls int
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/v1/servers/server-1/shutdown" {
|
||||
shutdownCalls++
|
||||
}
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer server.Close()
|
||||
s, err := New(Config{
|
||||
Command: []string{"/bin/sh", "-c", "sleep 5"},
|
||||
DrainURL: server.URL + "/drain", DrainToken: "run-secret", ControlPlaneURL: server.URL,
|
||||
WorkloadTokenPath: filepath.Join(t.TempDir(), "missing-token"), ServerID: "server-1", MatchID: "match-1",
|
||||
ProtocolVersion: 1, ImageDigest: "sha256:aa",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
time.AfterFunc(30*time.Millisecond, cancel)
|
||||
err = s.Run(ctx, 50*time.Millisecond)
|
||||
if err == nil || shutdownCalls != 0 {
|
||||
t.Fatalf("failed drain result=%v shutdown calls=%d", err, shutdownCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunForceKillsUnresponsiveChildAtDeadline(t *testing.T) {
|
||||
s, err := New(Config{Command: []string{"/bin/sh", "-c", "trap '' TERM; sleep 5"}})
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user