jcreek/CosmicClash
1
0
Fork 0
mirror of https://github.com/jcreek/CosmicClash.git synced 2026-09-14 01:52:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(server): partition and bound the Redis candidate projection

Browse Source 
Both playlists shared one hash and sorted set, causing two independent
failures.

Starvation: Snapshot performed an unbounded ZRANGEBYSCORE and HMGET,
decoded the whole queue, and the matcher then truncated to its candidate
limit *before* filtering by playlist. A large casual prefix could
therefore leave the ranked worker with zero candidates indefinitely even
while ranked tickets were queued further down the set.

Mutual erasure: each matcher captured only its own playlist as the
durable source, but Rebuild replaced the shared keys, so a casual repair
wiped ranked projections and vice versa.

Namespace the keys per playlist, push the limit into Redis (LIMIT 0 N)
so reads no longer scale with total queue depth, and scope Rebuild to
one namespace. Rebuild now rejects a candidate whose playlist does not
match the namespace, which would reintroduce the starvation. Upsert
derives the namespace from the candidate; Remove takes the playlist,
since a ticket ID alone no longer identifies its namespace.

Add tests for a 300-deep casual backlog not starving ranked, for neither
playlist's rebuild erasing the other, and for the limit being applied
without losing enqueue ordering.
This commit is contained in:
Josh Creek
2026-09-05 10:23:52 +01:00
parent f6a87463c5
commit 320ec46ba2
7 changed files with 215 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/store/redis_integration_test.go
+10 -10
View File
@@ -56,7 +56,7 @@ func TestRealRedisCandidateIndexUpsertSnapshotRemove(t *testing.T) {
if err := index.Upsert(ctx, b); err != nil {
t.Fatalf("upsert b: %v", err)
}
got, err := index.Snapshot(ctx, now.Add(time.Hour))
got, err := index.Snapshot(ctx, domain.Casual, now.Add(time.Hour), 1000)
if err != nil {
t.Fatalf("snapshot: %v", err)
}
@@ -64,10 +64,10 @@ func TestRealRedisCandidateIndexUpsertSnapshotRemove(t *testing.T) {
t.Fatalf("snapshot after upsert = %+v", got)
}
if err := index.Remove(ctx, "real-ticket-a"); err != nil {
if err := index.Remove(ctx, domain.Casual, "real-ticket-a"); err != nil {
t.Fatalf("remove: %v", err)
}
got, err = index.Snapshot(ctx, now.Add(time.Hour))
got, err = index.Snapshot(ctx, domain.Casual, now.Add(time.Hour), 1000)
if err != nil {
t.Fatalf("snapshot after remove: %v", err)
}
@@ -77,11 +77,11 @@ func TestRealRedisCandidateIndexUpsertSnapshotRemove(t *testing.T) {
// A real TTL, actually waited out, not miniredis's manual FastForward.
shortLived := RedisCandidateIndex{Client: client, Prefix: "integration-real-ttl", TTL: 1500 * time.Millisecond}
if err := shortLived.Upsert(ctx, domain.Candidate{TicketID: "real-ticket-ttl", PlayerID: "real-player-ttl", EnqueuedAt: now}); err != nil {
if err := shortLived.Upsert(ctx, domain.Candidate{Playlist: domain.Casual, TicketID: "real-ticket-ttl", PlayerID: "real-player-ttl", EnqueuedAt: now}); err != nil {
t.Fatalf("upsert ttl candidate: %v", err)
}
time.Sleep(2 * time.Second)
got, err = shortLived.Snapshot(ctx, now.Add(time.Hour))
got, err = shortLived.Snapshot(ctx, domain.Casual, now.Add(time.Hour), 1000)
if err != nil {
t.Fatalf("snapshot after real TTL expiry: %v", err)
}
@@ -100,11 +100,11 @@ func TestRealRedisCandidateProjectionRepairsAfterFlush(t *testing.T) {
now := time.Now().UTC().Truncate(time.Microsecond)
durable := []domain.Candidate{
{TicketID: "repair-ticket-a", PlayerID: "repair-player-a", EnqueuedAt: now},
{TicketID: "repair-ticket-b", PlayerID: "repair-player-b", EnqueuedAt: now.Add(time.Second)},
{Playlist: domain.Casual, TicketID: "repair-ticket-a", PlayerID: "repair-player-a", EnqueuedAt: now},
{Playlist: domain.Casual, TicketID: "repair-ticket-b", PlayerID: "repair-player-b", EnqueuedAt: now.Add(time.Second)},
}
sourceCalls := 0
projection := CandidateProjection{Index: index, Source: func(context.Context, time.Time) ([]domain.Candidate, error) {
projection := CandidateProjection{Index: index, Source: func(context.Context, domain.Playlist, time.Time, int) ([]domain.Candidate, error) {
sourceCalls++
return durable, nil
}}
@@ -120,7 +120,7 @@ func TestRealRedisCandidateProjectionRepairsAfterFlush(t *testing.T) {
t.Fatalf("flush: %v", err)
}
got, err := projection.Snapshot(ctx, now.Add(time.Hour))
got, err := projection.Snapshot(ctx, domain.Casual, now.Add(time.Hour), 1000)
if err != nil {
t.Fatalf("snapshot after flush: %v", err)
}
@@ -134,7 +134,7 @@ func TestRealRedisCandidateProjectionRepairsAfterFlush(t *testing.T) {
// The repair must actually have written back to Redis, not just returned
// the durable source's answer in memory -- confirm a second snapshot
// (Redis not flushed again) reads it back without a second Source call.
got, err = index.Snapshot(ctx, now.Add(time.Hour))
got, err = index.Snapshot(ctx, domain.Casual, now.Add(time.Hour), 1000)
if err != nil {
t.Fatalf("snapshot directly against Redis after repair: %v", err)
}