jcreek/CosmicClash
1
0
Fork 0
mirror of https://github.com/jcreek/CosmicClash.git synced 2026-09-14 01:02:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(server): partition and bound the Redis candidate projection

Browse Source 
Both playlists shared one hash and sorted set, causing two independent
failures.

Starvation: Snapshot performed an unbounded ZRANGEBYSCORE and HMGET,
decoded the whole queue, and the matcher then truncated to its candidate
limit *before* filtering by playlist. A large casual prefix could
therefore leave the ranked worker with zero candidates indefinitely even
while ranked tickets were queued further down the set.

Mutual erasure: each matcher captured only its own playlist as the
durable source, but Rebuild replaced the shared keys, so a casual repair
wiped ranked projections and vice versa.

Namespace the keys per playlist, push the limit into Redis (LIMIT 0 N)
so reads no longer scale with total queue depth, and scope Rebuild to
one namespace. Rebuild now rejects a candidate whose playlist does not
match the namespace, which would reintroduce the starvation. Upsert
derives the namespace from the candidate; Remove takes the playlist,
since a ticket ID alone no longer identifies its namespace.

Add tests for a 300-deep casual backlog not starving ranked, for neither
playlist's rebuild erasing the other, and for the limit being applied
without losing enqueue ordering.
This commit is contained in:
Josh Creek
2026-09-05 10:23:52 +01:00
parent f6a87463c5
commit 320ec46ba2
7 changed files with 215 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/store/candidate_projection_test.go
+14 -14
View File
@@ -19,16 +19,16 @@ func TestCandidateProjectionRepairsPartialRedisStateFromDurableSource(t *testing
client := redis.NewClient(&redis.Options{Addr: mini.Addr()})
defer client.Close()
now := time.Unix(1000, 0).UTC()
candidate := domain.Candidate{TicketID: "repair-ticket", PlayerID: "repair-player", EnqueuedAt: now}
candidate := domain.Candidate{Playlist: domain.Casual, TicketID: "repair-ticket", PlayerID: "repair-player", EnqueuedAt: now}
index := RedisCandidateIndex{Client: client, Prefix: "repair", TTL: time.Minute}
_, orderKey := index.keys()
_, orderKey := index.keys(domain.Casual)
if err := client.ZAdd(context.Background(), orderKey, redis.Z{Score: float64(now.UnixNano()), Member: candidate.TicketID}).Err(); err != nil {
t.Fatal(err)
}
projection := CandidateProjection{Index: index, Source: func(context.Context, time.Time) ([]domain.Candidate, error) {
projection := CandidateProjection{Index: index, Source: func(context.Context, domain.Playlist, time.Time, int) ([]domain.Candidate, error) {
return []domain.Candidate{candidate}, nil
}}
got, err := projection.Snapshot(context.Background(), now)
got, err := projection.Snapshot(context.Background(), domain.Casual, now, 1000)
if err != nil {
t.Fatal(err)
}
@@ -39,10 +39,10 @@ func TestCandidateProjectionRepairsPartialRedisStateFromDurableSource(t *testing
func TestCandidateProjectionDoesNotReturnCacheWhenRepairSourceFails(t *testing.T) {
index := RedisCandidateIndex{TTL: time.Minute}
projection := CandidateProjection{Index: index, Source: func(context.Context, time.Time) ([]domain.Candidate, error) {
projection := CandidateProjection{Index: index, Source: func(context.Context, domain.Playlist, time.Time, int) ([]domain.Candidate, error) {
return nil, context.DeadlineExceeded
}}
if _, err := projection.Snapshot(context.Background(), time.Unix(1000, 0)); err == nil {
if _, err := projection.Snapshot(context.Background(), domain.Casual, time.Unix(1000, 0), 1000); err == nil {
t.Fatal("cache projection succeeded without a usable Redis/index source")
}
}
@@ -67,16 +67,16 @@ func TestCandidateProjectionFallsBackToSourceWhenRedisIsEntirelyUnreachable(t *t
mini.Close() // Redis is now entirely unreachable, not merely empty or stale.
now := time.Unix(1000, 0).UTC()
candidate := domain.Candidate{TicketID: "down-ticket", PlayerID: "down-player", EnqueuedAt: now}
candidate := domain.Candidate{Playlist: domain.Casual, TicketID: "down-ticket", PlayerID: "down-player", EnqueuedAt: now}
sourceCalls := 0
projection := CandidateProjection{
Index: RedisCandidateIndex{Client: client, Prefix: "down", TTL: time.Minute},
Source: func(context.Context, time.Time) ([]domain.Candidate, error) {
Source: func(context.Context, domain.Playlist, time.Time, int) ([]domain.Candidate, error) {
sourceCalls++
return []domain.Candidate{candidate}, nil
},
}
got, err := projection.Snapshot(context.Background(), now)
got, err := projection.Snapshot(context.Background(), domain.Casual, now, 1000)
if err != nil {
t.Fatalf("Snapshot failed while Redis was down, even though Source (PostgreSQL) was healthy: %v", err)
}
@@ -102,11 +102,11 @@ func TestCandidateProjectionStillFailsWhenBothRedisAndSourceAreDown(t *testing.T
projection := CandidateProjection{
Index: RedisCandidateIndex{Client: client, Prefix: "down", TTL: time.Minute},
Source: func(context.Context, time.Time) ([]domain.Candidate, error) {
Source: func(context.Context, domain.Playlist, time.Time, int) ([]domain.Candidate, error) {
return nil, context.DeadlineExceeded
},
}
if _, err := projection.Snapshot(context.Background(), time.Unix(1000, 0)); err == nil {
if _, err := projection.Snapshot(context.Background(), domain.Casual, time.Unix(1000, 0), 1000); err == nil {
t.Fatal("Snapshot succeeded with both Redis and the durable source unavailable")
}
}
@@ -120,14 +120,14 @@ func TestCandidateProjectionRepairsEmptyIndexFromDurableSource(t *testing.T) {
client := redis.NewClient(&redis.Options{Addr: mini.Addr()})
defer client.Close()
now := time.Unix(1000, 0).UTC()
candidate := domain.Candidate{TicketID: "miss-ticket", PlayerID: "miss-player", EnqueuedAt: now}
candidate := domain.Candidate{Playlist: domain.Casual, TicketID: "miss-ticket", PlayerID: "miss-player", EnqueuedAt: now}
projection := CandidateProjection{
Index: RedisCandidateIndex{Client: client, Prefix: "miss", TTL: time.Minute},
Source: func(context.Context, time.Time) ([]domain.Candidate, error) {
Source: func(context.Context, domain.Playlist, time.Time, int) ([]domain.Candidate, error) {
return []domain.Candidate{candidate}, nil
},
}
got, err := projection.Snapshot(context.Background(), now)
got, err := projection.Snapshot(context.Background(), domain.Casual, now, 1000)
if err != nil {
t.Fatal(err)
}