mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
fix(server): partition and bound the Redis candidate projection
Both playlists shared one hash and sorted set, causing two independent failures. Starvation: Snapshot performed an unbounded ZRANGEBYSCORE and HMGET, decoded the whole queue, and the matcher then truncated to its candidate limit *before* filtering by playlist. A large casual prefix could therefore leave the ranked worker with zero candidates indefinitely even while ranked tickets were queued further down the set. Mutual erasure: each matcher captured only its own playlist as the durable source, but Rebuild replaced the shared keys, so a casual repair wiped ranked projections and vice versa. Namespace the keys per playlist, push the limit into Redis (LIMIT 0 N) so reads no longer scale with total queue depth, and scope Rebuild to one namespace. Rebuild now rejects a candidate whose playlist does not match the namespace, which would reintroduce the starvation. Upsert derives the namespace from the candidate; Remove takes the playlist, since a ticket ID alone no longer identifies its namespace. Add tests for a 300-deep casual backlog not starving ranked, for neither playlist's rebuild erasing the other, and for the limit being applied without losing enqueue ordering.
This commit is contained in:
@@ -59,7 +59,9 @@ type QueueBackend interface {
|
||||
// failures must never change the result of an already successful mutation.
|
||||
type CandidateIndex interface {
|
||||
Upsert(context.Context, domain.Candidate) error
|
||||
Remove(context.Context, string) error
|
||||
// Remove is playlist-scoped because the projection is partitioned per
|
||||
// playlist; a ticket ID alone does not identify its namespace.
|
||||
Remove(context.Context, domain.Playlist, string) error
|
||||
}
|
||||
|
||||
type SessionBackend interface {
|
||||
@@ -486,9 +488,9 @@ func (s *Service) projectCandidate(ctx context.Context, ticket domain.QueueTicke
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) removeCandidate(ctx context.Context, ticketID string) {
|
||||
func (s *Service) removeCandidate(ctx context.Context, playlist domain.Playlist, ticketID string) {
|
||||
if s.CandidateIndex != nil {
|
||||
_ = s.CandidateIndex.Remove(ctx, ticketID)
|
||||
_ = s.CandidateIndex.Remove(ctx, playlist, ticketID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -894,7 +896,7 @@ func (s *Service) queueMutation(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
s.logQueueOutcome(eventName, ticketID, ticket, nil, now)
|
||||
if ticket.State == domain.Cancelled {
|
||||
s.removeCandidate(r.Context(), ticket.TicketID)
|
||||
s.removeCandidate(r.Context(), ticket.Playlist, ticket.TicketID)
|
||||
} else {
|
||||
s.projectCandidate(r.Context(), ticket)
|
||||
}
|
||||
|
||||
@@ -135,7 +135,7 @@ func (i *candidateIndexSpy) Upsert(_ context.Context, candidate domain.Candidate
|
||||
return i.upsertErr
|
||||
}
|
||||
|
||||
func (i *candidateIndexSpy) Remove(_ context.Context, _ string) error {
|
||||
func (i *candidateIndexSpy) Remove(_ context.Context, _ domain.Playlist, _ string) error {
|
||||
i.removeCalls++
|
||||
return i.removeErr
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user