CosmicClash/server/cmd/maintenance/main.go

package main

import (
	"context"
	"database/sql"
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/cosmic-clash/cosmic-clash/server/migrations"
	"github.com/cosmic-clash/cosmic-clash/server/store"
	_ "github.com/jackc/pgx/v5/stdlib"
)

func main() {
	dsn := flag.String("dsn", os.Getenv("COSMIC_CLASH_POSTGRES_DSN"), "PostgreSQL connection string")
	migrationDir := flag.String("migrations", "migrations", "directory containing numbered SQL migrations")
	interval := flag.Duration("interval", time.Minute, "maintenance poll interval")
	initialConnectInterval := flag.Duration("initial-connect-interval", time.Second, "initial-connect reconciliation poll interval")
	batch := flag.Int("batch", 100, "maximum player rollovers per pass")
	stalledAllocationDeadline := flag.Duration("stalled-allocation-deadline", 2*time.Minute, "reclaim a match stuck in ALLOCATING/PROCESS_READY/ASSIGNMENT_READY (server crashed or was reclaimed before registering) after this long, requeuing every participant without penalty")
	stalledAllocationBatch := flag.Int("stalled-allocation-batch", 100, "maximum stalled matches reclaimed per pass")
	initialConnectBatch := flag.Int("initial-connect-batch", 100, "maximum pre-live matches evaluated per pass")
	liveAbandonmentBatch := flag.Int("live-abandonment-batch", 100, "maximum live ranked matches evaluated for expired reconnect leases per pass")
	retentionBatch := flag.Int("retention-batch", 500, "maximum rows deleted per table per retention pass")
	flag.Parse()
	if *dsn == "" {
		fatalf("--dsn or COSMIC_CLASH_POSTGRES_DSN is required")
	}
	if *interval <= 0 || *initialConnectInterval <= 0 || *batch < 1 || *batch > 1000 {
		fatalf("invalid interval or batch")
	}
	if *stalledAllocationDeadline <= 0 || *stalledAllocationBatch < 1 || *stalledAllocationBatch > 1000 {
		fatalf("invalid stalled-allocation deadline or batch")
	}
	if *initialConnectBatch < 1 || *initialConnectBatch > 1000 || *liveAbandonmentBatch < 1 || *liveAbandonmentBatch > 1000 {
		fatalf("invalid initial-connect or live-abandonment batch")
	}
	db, err := sql.Open("pgx", *dsn)
	if err != nil {
		fatalf("open PostgreSQL: %v", err)
	}
	defer db.Close()
	startupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	if err := db.PingContext(startupCtx); err != nil {
		fatalf("ping PostgreSQL: %v", err)
	}
	if err := migrations.Apply(startupCtx, db, *migrationDir); err != nil {
		fatalf("apply migrations: %v", err)
	}
	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()
	runGeneral := func(now time.Time) {
		count, err := store.RolloverDueSeasons(ctx, db, now, *batch)
		if err != nil {
			fatalf("season maintenance: %v", err)
		}
		if count > 0 {
			log.Printf("applied %d ranked season rollovers", count)
		}
		reclaimed, err := store.ExpireStalledAllocations(ctx, db, now, *stalledAllocationDeadline, *stalledAllocationBatch)
		if err != nil {
			fatalf("stalled-allocation maintenance: %v", err)
		}
		if reclaimed > 0 {
			log.Printf("reclaimed %d stalled allocations, requeuing their participants", reclaimed)
		}
		// Retention. Without this, idempotency keys alone grow by roughly one
		// row per queued player per heartbeat interval, forever.
		purged, err := store.PurgeExpiredRecords(ctx, db, now, *retentionBatch)
		if err != nil {
			fatalf("retention maintenance: %v", err)
		}
		if purged.Total() > 0 {
			log.Printf("purged %d expired records (idempotency=%d outbox=%d dead-lettered=%d sessions=%d)",
				purged.Total(), purged.IdempotencyKeys, purged.PublishedOutbox, purged.DeadLetteredOutbox, purged.ExpiredSessions)
		}
		// Deletion lag: a backlog that keeps climbing means the interval or
		// batch size is too small for current volume.
		backlog, err := store.RetentionBacklog(ctx, db, now)
		if err != nil {
			fatalf("retention backlog: %v", err)
		}
		if backlog > 0 {
			log.Printf("retention backlog is %d rows past their window", backlog)
		}
		staleProbes, err := store.PurgeExpiredProbeChallenges(ctx, db, now)
		if err != nil {
			fatalf("probe challenge maintenance: %v", err)
		}
		if staleProbes > 0 {
			log.Printf("purged %d unanswered probe challenges", staleProbes)
		}
		deadLettered, err := store.CountDeadLetteredOutboxEvents(ctx, db)
		if err != nil {
			fatalf("dead-letter count: %v", err)
		}
		if deadLettered > 0 {
			log.Printf("WARNING: %d outbox events were never delivered and are dead-lettered", deadLettered)
		}
	}
	runInitialConnect := func(now time.Time) {
		reconciled, err := store.ReconcileInitialConnect(ctx, db, now, *initialConnectBatch)
		if err != nil {
			fatalf("initial-connect maintenance: %v", err)
		}
		if reconciled > 0 {
			log.Printf("reconciled %d initial-connect outcomes", reconciled)
		}
		abandoned, err := store.ReconcileLiveAbandonments(ctx, db, now, *liveAbandonmentBatch)
		if err != nil {
			fatalf("live-abandonment maintenance: %v", err)
		}
		if abandoned > 0 {
			log.Printf("recorded expired reconnect leases in %d live matches", abandoned)
		}
	}

	runGeneral(time.Now().UTC())
	runInitialConnect(time.Now().UTC())
	generalTicker := time.NewTicker(*interval)
	initialConnectTicker := time.NewTicker(*initialConnectInterval)
	defer generalTicker.Stop()
	defer initialConnectTicker.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case now := <-generalTicker.C:
			runGeneral(now.UTC())
		case now := <-initialConnectTicker.C:
			runInitialConnect(now.UTC())
		}
	}
}

func fatalf(format string, args ...any) {
	fmt.Fprintf(os.Stderr, "maintenance: "+format+"\n", args...)
	os.Exit(1)
}