feat: bound supervisor drain termination

This commit is contained in:
Josh Creek
2026-09-01 09:18:19 +01:00
parent e7c835af52
commit 58508bd87c
4 changed files with 108 additions and 3 deletions
+54
View File
@@ -46,6 +46,8 @@ type Supervisor struct {
cmd *exec.Cmd
}
const DefaultDrainGrace = 285 * time.Second
func New(config Config) (*Supervisor, error) {
if len(config.Command) == 0 || config.Command[0] == "" {
return nil, fmt.Errorf("supervisor command is required")
@@ -142,6 +144,58 @@ func (s *Supervisor) Wait() error {
return s.cmd.Wait()
}
// Run owns the PID-1 termination sequence. The child gets its own context so
// cancellation of the supervisor does not kill it before the authenticated
// drain request has had a chance to stop new admissions. A non-responsive
// child is force-killed after drainGrace; a drain failure is recorded only by
// the returned error if the child exits cleanly, while the deadline still
// prevents a stuck process from hanging termination forever.
func (s *Supervisor) Run(ctx context.Context, drainGrace time.Duration) error {
if s == nil || ctx == nil {
return fmt.Errorf("supervisor context is required")
}
if drainGrace <= 0 {
drainGrace = DefaultDrainGrace
}
processCtx, cancel := context.WithCancel(context.Background())
defer cancel()
if err := s.Start(processCtx); err != nil {
return err
}
wait := make(chan error, 1)
go func() { wait <- s.Wait() }()
select {
case err := <-wait:
return err
case <-ctx.Done():
}
var drainErr error
if s.config.DrainURL != "" {
drainCtx, drainCancel := context.WithTimeout(context.Background(), 5*time.Second)
drainErr = s.Drain(drainCtx)
drainCancel()
}
timer := time.NewTimer(drainGrace)
defer timer.Stop()
select {
case err := <-wait:
if drainErr != nil {
return fmt.Errorf("child exited after drain failure: %w", drainErr)
}
return err
case <-timer.C:
if s.cmd != nil && s.cmd.Process != nil {
_ = s.cmd.Process.Kill()
}
<-wait
if drainErr != nil {
return fmt.Errorf("drain failed and child was force-killed: %w", drainErr)
}
return fmt.Errorf("child force-killed after drain deadline")
}
}
// Drain asks the allocated Godot process to stop accepting new work. The
// token is sent only over the configured localhost control endpoint and is
// never placed in command arguments or logs.