Fix seen-key handling in pausable tape reduction
- Break on the first previously seen item in newest-first merges - Avoid cloning seen key sets unless new items are added - Add tape overhaul phase 1 notes
This commit is contained in:
parent
b88ef2b371
commit
f28c8e641f
2 changed files with 181 additions and 5 deletions
|
|
@ -409,13 +409,19 @@ export const reducePausableTapeData = <T extends SortableItem>(
|
|||
return current;
|
||||
}
|
||||
|
||||
const nextSeenKeys = new Set(current.seenKeys);
|
||||
const seenKeys = current.seenKeys;
|
||||
let nextSeenKeys: Set<string> | null = null;
|
||||
const unseen: T[] = [];
|
||||
|
||||
// Incoming items are maintained newest-first by mergeNewest.
|
||||
// Once we hit a previously seen key, the remainder is older history.
|
||||
for (const item of incoming) {
|
||||
const key = getTapeItemKey(item);
|
||||
if (nextSeenKeys.has(key)) {
|
||||
continue;
|
||||
if (seenKeys.has(key)) {
|
||||
break;
|
||||
}
|
||||
if (!nextSeenKeys) {
|
||||
nextSeenKeys = new Set(seenKeys);
|
||||
}
|
||||
nextSeenKeys.add(key);
|
||||
unseen.push(item);
|
||||
|
|
@ -431,7 +437,7 @@ export const reducePausableTapeData = <T extends SortableItem>(
|
|||
queued: mergeNewest(unseen, current.queued, retentionLimit, (evicted) =>
|
||||
incrementRetentionMetric("hotWindowEvictions", evicted)
|
||||
),
|
||||
seenKeys: nextSeenKeys,
|
||||
seenKeys: nextSeenKeys ?? seenKeys,
|
||||
dropped: current.dropped + unseen.length
|
||||
};
|
||||
}
|
||||
|
|
@ -442,7 +448,7 @@ export const reducePausableTapeData = <T extends SortableItem>(
|
|||
incrementRetentionMetric("hotWindowEvictions", evicted)
|
||||
),
|
||||
queued: [],
|
||||
seenKeys: nextSeenKeys,
|
||||
seenKeys: nextSeenKeys ?? seenKeys,
|
||||
dropped: 0
|
||||
};
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue