This repository has no description
fix pagination
author
damedotblog
date
(Apr 1, 2025, 10:36 PM -0400)
commit
4bf17fb7
4bf17fb727b2f72b99fcf04651286bcf720a2a4f
parent
3a52ac67
3a52ac67090f90a8d2a7dfe20288f2bfa6d289b0
+55
-7
+55
-7
src/components/CollectionsFeed/CollectionsFeed.js
+55
-7
src/components/CollectionsFeed/CollectionsFeed.js
···
221
221
let pageCount = 0;
222
222
let collectionRecords = [];
223
223
let reachedCutoff = false;
224
+
let totalRecordsForCollection = 0;
224
225
225
-
// For initial deep load, we paginate more to get historical data
226
+
// For initial deep load, we need to paginate as many times as needed to get all historical data
226
227
// For regular timeline browsing or load more, we just get one page
227
-
const maxPages = isInitialDeepLoad ? 20 : 1;
228
+
// Set a high limit for safety, but essentially allow unlimited pagination until we hit the cutoff date
229
+
const maxPages = isInitialDeepLoad ? 1000 : 1;
228
230
229
231
while (hasMoreRecords && pageCount < maxPages && !reachedCutoff) {
230
232
// Fetch up to 100 records per page
···
249
251
250
252
// Process records from this page
251
253
if (data.records && data.records.length > 0) {
254
+
console.log(`Received ${data.records.length} records for ${collection} page ${pageCount}`);
255
+
totalRecordsForCollection += data.records.length;
256
+
252
257
const processedRecords = data.records.map(record => {
253
258
const contentTimestamp = extractTimestamp(record);
254
259
const rkey = record.uri.split('/').pop();
···
274
279
return recordTime < oldest ? recordTime : oldest;
275
280
}, Date.now());
276
281
277
-
// If the oldest record on this page is older than our cutoff, we'll stop
278
-
if (oldestRecordTime < cutoffDate.getTime()) {
282
+
// For commonly used collections like likes, follows, etc.,
283
+
// we need to be more cautious about when to stop paginating
284
+
const isHighVolumeCollection = collection.includes('like') ||
285
+
collection.includes('follow') ||
286
+
collection.includes('repost');
287
+
288
+
// If the oldest record on this page is older than our cutoff, and
289
+
// 1. It's not a high volume collection, OR
290
+
// 2. It's a high volume collection but we've already gone through several pages
291
+
if (oldestRecordTime < cutoffDate.getTime() &&
292
+
(!isHighVolumeCollection || pageCount > 5)) {
279
293
reachedCutoff = true;
280
-
console.log(`Reached cutoff date for ${collection} on page ${pageCount}`);
294
+
console.log(`Reached cutoff date for ${collection} on page ${pageCount} (oldest: ${new Date(oldestRecordTime).toISOString()})`);
281
295
282
296
// Filter records from this page to only include those after cutoff
283
297
const filteredRecords = processedRecords.filter(record => {
···
286
300
return new Date(timestamp) >= cutoffDate;
287
301
});
288
302
303
+
console.log(` - Kept ${filteredRecords.length} of ${processedRecords.length} records from final page`);
289
304
collectionRecords.push(...filteredRecords);
290
305
} else {
291
-
// All records on this page are within our date range
306
+
// All records on this page are within our date range
307
+
// OR we need to keep paginating through high-volume collections
292
308
collectionRecords.push(...processedRecords);
309
+
310
+
// If we found some records close to the cutoff date but haven't reached it yet,
311
+
// log this for debugging purposes
312
+
if (oldestRecordTime < cutoffDate.getTime() + (7 * 24 * 60 * 60 * 1000)) { // within 7 days of cutoff
313
+
console.log(` - Getting close to cutoff date, oldest record = ${new Date(oldestRecordTime).toISOString()}`);
314
+
}
293
315
}
294
316
} else {
295
317
// For regular browsing, include all records from the page
···
316
338
} else {
317
339
// No more records for this collection
318
340
delete newCursors[collection];
341
+
}
342
+
343
+
console.log(`Finished fetching ${collection}: Retrieved ${totalRecordsForCollection} records in ${pageCount} pages`);
344
+
console.log(` - After filtering: ${collectionRecords.length} records in 90-day window`);
345
+
if (reachedCutoff) {
346
+
console.log(` - Stopped because records older than 90 days were found`);
347
+
} else if (!cursor) {
348
+
console.log(` - Stopped because no more records were available`);
349
+
} else if (pageCount >= maxPages) {
350
+
console.log(` - Stopped because max page limit (${maxPages}) was reached`);
319
351
}
320
352
321
353
// Add records to appropriate arrays
···
358
390
displayRecords = filterAndSort(allRecords);
359
391
}
360
392
361
-
console.log(`Fetched ${sortedChartRecords.length} records for chart, showing ${displayRecords.length} in timeline`);
393
+
// Create a summary of records per collection for debugging
394
+
const collectionSummary = {};
395
+
sortedChartRecords.forEach(record => {
396
+
if (!collectionSummary[record.collection]) {
397
+
collectionSummary[record.collection] = 0;
398
+
}
399
+
collectionSummary[record.collection]++;
400
+
});
401
+
402
+
console.log("Collection record counts in 90-day period:");
403
+
Object.entries(collectionSummary)
404
+
.sort((a, b) => b[1] - a[1]) // Sort by count descending
405
+
.forEach(([collection, count]) => {
406
+
console.log(` - ${collection}: ${count} records`);
407
+
});
408
+
409
+
console.log(`Fetched ${sortedChartRecords.length} total records for chart, showing ${displayRecords.length} in timeline`);
362
410
363
411
// Update state
364
412
setRecords(displayRecords);