mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-11 20:33:44 +00:00
79ab0d1404
Closes part of the 'live Redis failover' gap in §8.46, found by reproducing a genuine Redis outage (not just an empty/partial cache) against CandidateProjection.Snapshot with a killed miniredis instance. CandidateProjection.Snapshot funnelled two different situations into the same code path: the index erroring outright (Redis unreachable) and the index coming back empty (ambiguous — a genuinely empty queue, or a lost keyspace). Both went through Repair, which itself calls Index.Rebuild — a second Redis round-trip that fails for exactly the same reason the first one did. The result: a real Redis outage, or the window during a failover, made Snapshot fail outright even though PostgreSQL — the documented authoritative source everywhere (RedisCandidateIndex's own comment, cmd/matcher, cmd/control-plane's --redis-addr help text all call it a rebuildable/optional acceleration layer) — was completely healthy. Matchmaking would stop entirely on a Redis outage despite the architecture explicitly not requiring that. Snapshot now falls back to serving Source (PostgreSQL) directly whenever the index errors OR comes back empty, and only best-effort attempts to repopulate Redis afterward — that attempt's outcome is deliberately ignored, since a caller must never be denied service just because the opportunistic rebuild also hit the same down Redis. Snapshot still fails when Source itself is unavailable; the fallback is not unconditional. Verified: reproduced the bug first (killed-miniredis Snapshot call failed even though Source was healthy), then fixed it. go build/vet clean; all pre-existing store-package tests pass unmodified, including the two live-redis:7-alpine-container tests (TestRealRedisCandidateIndexUpsertSnapshotRemove, TestRealRedisCandidateProjectionRepairsAfterFlush, run against a real container and torn down after). Two new tests cover the fallback directly (killed miniredis, Source still served, exactly one Source call) and that the fallback is not unconditional (both Redis and Source down still fails). Full go test ./... -race clean across every server package. Remaining: live matcher-worker-under-load-during-failover integration, i.e. running the actual matcher process against a real Redis that goes down mid-run under concurrent load, not just this unit-level reproduction.
138 lines
5.2 KiB
Go
138 lines
5.2 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/alicebob/miniredis/v2"
|
|
"github.com/cosmic-clash/cosmic-clash/server/domain"
|
|
"github.com/redis/go-redis/v9"
|
|
)
|
|
|
|
func TestCandidateProjectionRepairsPartialRedisStateFromDurableSource(t *testing.T) {
|
|
mini, err := miniredis.Run()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer mini.Close()
|
|
client := redis.NewClient(&redis.Options{Addr: mini.Addr()})
|
|
defer client.Close()
|
|
now := time.Unix(1000, 0).UTC()
|
|
candidate := domain.Candidate{TicketID: "repair-ticket", PlayerID: "repair-player", EnqueuedAt: now}
|
|
index := RedisCandidateIndex{Client: client, Prefix: "repair", TTL: time.Minute}
|
|
_, orderKey := index.keys()
|
|
if err := client.ZAdd(context.Background(), orderKey, redis.Z{Score: float64(now.UnixNano()), Member: candidate.TicketID}).Err(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
projection := CandidateProjection{Index: index, Source: func(context.Context, time.Time) ([]domain.Candidate, error) {
|
|
return []domain.Candidate{candidate}, nil
|
|
}}
|
|
got, err := projection.Snapshot(context.Background(), now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(got) != 1 || got[0].TicketID != candidate.TicketID {
|
|
t.Fatalf("repaired projection = %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestCandidateProjectionDoesNotReturnCacheWhenRepairSourceFails(t *testing.T) {
|
|
index := RedisCandidateIndex{TTL: time.Minute}
|
|
projection := CandidateProjection{Index: index, Source: func(context.Context, time.Time) ([]domain.Candidate, error) {
|
|
return nil, context.DeadlineExceeded
|
|
}}
|
|
if _, err := projection.Snapshot(context.Background(), time.Unix(1000, 0)); err == nil {
|
|
t.Fatal("cache projection succeeded without a usable Redis/index source")
|
|
}
|
|
}
|
|
|
|
// TestCandidateProjectionFallsBackToSourceWhenRedisIsEntirelyUnreachable
|
|
// covers the gap multiplayer-next.md §8.46 named "live Redis failover":
|
|
// Redis is documented everywhere (RedisCandidateIndex's own comment,
|
|
// cmd/matcher, cmd/control-plane) as an optional, rebuildable acceleration
|
|
// layer over PostgreSQL authority. Before this fix, Snapshot funnelled a
|
|
// genuine Redis connection failure into the same Repair path as an empty
|
|
// cache -- but Repair's own Index.Rebuild call also needs Redis, so it failed
|
|
// for the identical reason, and Snapshot returned an error even though the
|
|
// authoritative Source was perfectly healthy. A real Redis outage or
|
|
// mid-failover window would have taken matchmaking down completely.
|
|
func TestCandidateProjectionFallsBackToSourceWhenRedisIsEntirelyUnreachable(t *testing.T) {
|
|
mini, err := miniredis.Run()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
client := redis.NewClient(&redis.Options{Addr: mini.Addr()})
|
|
defer client.Close()
|
|
mini.Close() // Redis is now entirely unreachable, not merely empty or stale.
|
|
|
|
now := time.Unix(1000, 0).UTC()
|
|
candidate := domain.Candidate{TicketID: "down-ticket", PlayerID: "down-player", EnqueuedAt: now}
|
|
sourceCalls := 0
|
|
projection := CandidateProjection{
|
|
Index: RedisCandidateIndex{Client: client, Prefix: "down", TTL: time.Minute},
|
|
Source: func(context.Context, time.Time) ([]domain.Candidate, error) {
|
|
sourceCalls++
|
|
return []domain.Candidate{candidate}, nil
|
|
},
|
|
}
|
|
got, err := projection.Snapshot(context.Background(), now)
|
|
if err != nil {
|
|
t.Fatalf("Snapshot failed while Redis was down, even though Source (PostgreSQL) was healthy: %v", err)
|
|
}
|
|
if len(got) != 1 || got[0].TicketID != candidate.TicketID {
|
|
t.Fatalf("fallback snapshot = %+v, want the durable candidate served directly", got)
|
|
}
|
|
if sourceCalls != 1 {
|
|
t.Fatalf("Source calls = %d, want exactly 1", sourceCalls)
|
|
}
|
|
}
|
|
|
|
// TestCandidateProjectionStillFailsWhenBothRedisAndSourceAreDown proves the
|
|
// fallback isn't unconditional: if PostgreSQL itself is also unavailable,
|
|
// Snapshot must still fail rather than silently return an empty match pool.
|
|
func TestCandidateProjectionStillFailsWhenBothRedisAndSourceAreDown(t *testing.T) {
|
|
mini, err := miniredis.Run()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
client := redis.NewClient(&redis.Options{Addr: mini.Addr()})
|
|
defer client.Close()
|
|
mini.Close()
|
|
|
|
projection := CandidateProjection{
|
|
Index: RedisCandidateIndex{Client: client, Prefix: "down", TTL: time.Minute},
|
|
Source: func(context.Context, time.Time) ([]domain.Candidate, error) {
|
|
return nil, context.DeadlineExceeded
|
|
},
|
|
}
|
|
if _, err := projection.Snapshot(context.Background(), time.Unix(1000, 0)); err == nil {
|
|
t.Fatal("Snapshot succeeded with both Redis and the durable source unavailable")
|
|
}
|
|
}
|
|
|
|
func TestCandidateProjectionRepairsEmptyIndexFromDurableSource(t *testing.T) {
|
|
mini, err := miniredis.Run()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer mini.Close()
|
|
client := redis.NewClient(&redis.Options{Addr: mini.Addr()})
|
|
defer client.Close()
|
|
now := time.Unix(1000, 0).UTC()
|
|
candidate := domain.Candidate{TicketID: "miss-ticket", PlayerID: "miss-player", EnqueuedAt: now}
|
|
projection := CandidateProjection{
|
|
Index: RedisCandidateIndex{Client: client, Prefix: "miss", TTL: time.Minute},
|
|
Source: func(context.Context, time.Time) ([]domain.Candidate, error) {
|
|
return []domain.Candidate{candidate}, nil
|
|
},
|
|
}
|
|
got, err := projection.Snapshot(context.Background(), now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(got) != 1 || got[0].TicketID != candidate.TicketID {
|
|
t.Fatalf("empty-index repair = %+v", got)
|
|
}
|
|
}
|