From 929ebb19de710a56e2c24329ff549ab1d634f3cc Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 26 May 2026 02:30:34 +0000 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=90=9B=20Fix=20Strava=20sync=20missin?= =?UTF-8?q?g=20delayed=20activities?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avoid relying on the activity start-time checkpoint for discovery so late uploads with older start dates are still imported. Co-authored-by: Yu Le --- scripts/sync-strava-activities-core.mjs | 110 ++++++++++++++++ scripts/sync-strava-activities-core.test.mjs | 71 ++++++++++ scripts/sync-strava-activities.mjs | 131 +------------------ 3 files changed, 187 insertions(+), 125 deletions(-) create mode 100644 scripts/sync-strava-activities-core.mjs create mode 100644 scripts/sync-strava-activities-core.test.mjs diff --git a/scripts/sync-strava-activities-core.mjs b/scripts/sync-strava-activities-core.mjs new file mode 100644 index 0000000..a426e81 --- /dev/null +++ b/scripts/sync-strava-activities-core.mjs @@ -0,0 +1,110 @@ +import fs from 'node:fs' +import path from 'node:path' + +const IDS_FILE = '_index.json' +const META_FILE = '_meta.json' + +function readJson(file, fallback) { + try { + return JSON.parse(fs.readFileSync(file, 'utf-8')) + } + catch { + return fallback + } +} + +function writeJson(file, value) { + fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`) +} + +function readIds(activitiesDir) { + const data = readJson(path.join(activitiesDir, IDS_FILE), []) + return Array.isArray(data) ? data : [] +} + +function readMeta(activitiesDir) { + const data = readJson(path.join(activitiesDir, META_FILE), {}) + return data && typeof data === 'object' ? data : {} +} + +function hasDetail(activitiesDir, id) { + return fs.existsSync(path.join(activitiesDir, `${id}.json`)) +} + +function saveDetail(activitiesDir, id, data) { + writeJson(path.join(activitiesDir, `${id}.json`), data) +} + +function getLatestActivityTimestamp(activities) { + return Math.max( + ...activities.map(activity => Math.floor(new Date(activity.start_date).getTime() / 1000)), + ) +} + +async function fetchAllActivities(getStravaActivities) { + const all = [] + let page = 1 + while (true) { + const batch = await getStravaActivities({ page, perPage: 100 }) + if (!batch.length) + break + all.push(...batch) + if (batch.length < 100) + break + page++ + } + return all +} + +export async function syncActivities({ + activitiesDir, + getStravaActivities, + getStravaActivityById, + logger = console, +}) { + fs.mkdirSync(activitiesDir, { recursive: true }) + + logger.log('🔄 Syncing Strava activities...') + logger.log(' Full sync: fetching all activities') + + const knownIds = readIds(activitiesDir) + const knownSet = new Set(knownIds) + const meta = readMeta(activitiesDir) + + const activities = await fetchAllActivities(async (options) => { + const batch = await getStravaActivities(options) + if (batch.length) + logger.log(` Page ${options.page}: ${batch.length} activities`) + return batch + }) + + if (activities.length === 0) { + logger.log('✅ No activities found') + return + } + + const freshIds = activities.map(a => a.id) + const missingIds = freshIds.filter(id => !hasDetail(activitiesDir, id)) + + if (missingIds.length === 0) { + const latestTimestamp = getLatestActivityTimestamp(activities) + writeJson(path.join(activitiesDir, META_FILE), { ...meta, lastSync: latestTimestamp }) + logger.log('✅ All activities up to date') + return + } + + logger.log(`⬇️ Fetching ${missingIds.length} activity details...`) + for (const id of missingIds) { + logger.log(` ${id}`) + const detail = await getStravaActivityById(id) + saveDetail(activitiesDir, id, detail) + } + + for (const id of freshIds) knownSet.add(id) + writeJson(path.join(activitiesDir, IDS_FILE), [...knownSet]) + + const latestTimestamp = getLatestActivityTimestamp(activities) + writeJson(path.join(activitiesDir, META_FILE), { ...meta, lastSync: latestTimestamp }) + + logger.log(`✅ Synced ${missingIds.length} activities (total: ${knownSet.size})`) +} diff --git a/scripts/sync-strava-activities-core.test.mjs b/scripts/sync-strava-activities-core.test.mjs new file mode 100644 index 0000000..3b6665f --- /dev/null +++ b/scripts/sync-strava-activities-core.test.mjs @@ -0,0 +1,71 @@ +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { syncActivities } from './sync-strava-activities-core.mjs' + +const tmpDirs = [] + +function makeActivitiesDir() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'strava-sync-')) + tmpDirs.push(dir) + fs.mkdirSync(dir, { recursive: true }) + return dir +} + +function writeJson(file, value) { + fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`) +} + +function readJson(file) { + return JSON.parse(fs.readFileSync(file, 'utf-8')) +} + +afterEach(() => { + for (const dir of tmpDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }) + } +}) + +describe('syncActivities', () => { + it('discovers late-uploaded activities whose start date predates the checkpoint', async () => { + const activitiesDir = makeActivitiesDir() + writeJson(path.join(activitiesDir, '_index.json'), [200]) + writeJson(path.join(activitiesDir, '_meta.json'), { lastSync: 1_700_000_000 }) + writeJson(path.join(activitiesDir, '200.json'), { + id: 200, + start_date: '2023-11-14T22:13:20Z', + }) + + const listCalls = [] + const lateUploadedActivity = { + id: 100, + start_date: '2023-01-01T00:00:00Z', + } + const knownActivity = { + id: 200, + start_date: '2023-11-14T22:13:20Z', + } + + await syncActivities({ + activitiesDir, + getStravaActivities: async (options) => { + listCalls.push(options) + return listCalls.length === 1 ? [knownActivity, lateUploadedActivity] : [] + }, + getStravaActivityById: async id => ({ + id, + start_date: lateUploadedActivity.start_date, + detail: true, + }), + logger: { log() {} }, + }) + + expect(listCalls[0]).not.toHaveProperty('after') + expect(readJson(path.join(activitiesDir, '_index.json'))).toEqual([200, 100]) + expect(readJson(path.join(activitiesDir, '100.json'))).toMatchObject({ + id: 100, + detail: true, + }) + }) +}) diff --git a/scripts/sync-strava-activities.mjs b/scripts/sync-strava-activities.mjs index 98649e0..2743305 100644 --- a/scripts/sync-strava-activities.mjs +++ b/scripts/sync-strava-activities.mjs @@ -1,4 +1,3 @@ -import fs from 'node:fs' import path from 'node:path' import process from 'node:process' import { fileURLToPath } from 'node:url' @@ -6,134 +5,16 @@ import { getStravaActivities, getStravaActivityById, } from '../src/utils/strava.ts' +import { syncActivities } from './sync-strava-activities-core.mjs' const __dirname = path.dirname(fileURLToPath(import.meta.url)) const ACTIVITIES_DIR = path.join(__dirname, '..', 'data', 'strava', 'activities') -const IDS_FILE = path.join(ACTIVITIES_DIR, '_index.json') -const META_FILE = path.join(ACTIVITIES_DIR, '_meta.json') -fs.mkdirSync(ACTIVITIES_DIR, { recursive: true }) - -function readIds() { - try { - const data = JSON.parse(fs.readFileSync(IDS_FILE, 'utf-8')) - return Array.isArray(data) ? data : [] - } - catch { - return [] - } -} - -function readMeta() { - try { - const data = JSON.parse(fs.readFileSync(META_FILE, 'utf-8')) - return data || {} - } - catch { - return {} - } -} - -function writeMeta(meta) { - fs.writeFileSync(META_FILE, `${JSON.stringify(meta, null, 2)}\n`) -} - -function writeIds(ids) { - fs.writeFileSync(IDS_FILE, `${JSON.stringify(ids, null, 2)}\n`) -} - -function hasDetail(id) { - return fs.existsSync(path.join(ACTIVITIES_DIR, `${id}.json`)) -} - -function saveDetail(id, data) { - fs.writeFileSync( - path.join(ACTIVITIES_DIR, `${id}.json`), - `${JSON.stringify(data, null, 2)}\n`, - ) -} - -function getLatestActivityTimestamp(activities) { - return Math.max( - ...activities.map(activity => Math.floor(new Date(activity.start_date).getTime() / 1000)), - ) -} - -async function fetchAllActivities(after = null) { - const all = [] - let page = 1 - while (true) { - const options = { page, perPage: 100 } - if (after) { - options.after = after - } - const batch = await getStravaActivities(options) - if (!batch.length) - break - all.push(...batch) - console.log(` Page ${page}: ${batch.length} activities`) - if (batch.length < 100) - break - page++ - } - return all -} - -async function syncActivities() { - console.log('🔄 Syncing Strava activities...') - - const knownIds = readIds() - const knownSet = new Set(knownIds) - const meta = readMeta() - - // Use last sync timestamp for incremental fetch - const lastSync = meta.lastSync || null - const isFullSync = !lastSync - - if (isFullSync) { - console.log(' Full sync: fetching all activities (no previous sync)') - } - else { - const syncDate = new Date(lastSync * 1000).toISOString() - console.log(` Incremental sync: fetching activities since ${syncDate}`) - } - - // Fetch activities (incremental if we have a lastSync timestamp) - const activities = await fetchAllActivities(lastSync) - - if (activities.length === 0) { - console.log('✅ No new activities found') - return - } - - const freshIds = activities.map(a => a.id) - const missingIds = freshIds.filter(id => !hasDetail(id)) - - if (missingIds.length === 0) { - const latestTimestamp = getLatestActivityTimestamp(activities) - writeMeta({ ...meta, lastSync: latestTimestamp }) - console.log('✅ All activities up to date') - return - } - - console.log(`⬇️ Fetching ${missingIds.length} activity details...`) - for (const id of missingIds) { - console.log(` ${id}`) - const detail = await getStravaActivityById(id) - saveDetail(id, detail) - } - - for (const id of freshIds) knownSet.add(id) - writeIds([...knownSet]) - - // Update lastSync timestamp based on the latest activity fetched - const latestTimestamp = getLatestActivityTimestamp(activities) - writeMeta({ ...meta, lastSync: latestTimestamp }) - - console.log(`✅ Synced ${missingIds.length} activities (total: ${knownSet.size})`) -} - -syncActivities().catch((err) => { +syncActivities({ + activitiesDir: ACTIVITIES_DIR, + getStravaActivities, + getStravaActivityById, +}).catch((err) => { console.error('❌ Sync failed:', err.message) process.exit(1) }) From 088243ee4db51b1c12644e0845a8d0750bba6e75 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 26 May 2026 02:35:37 +0000 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=90=9B=20Repair=20Strava=20activity?= =?UTF-8?q?=20index=20during=20sync=20EOF=20&&=20git=20push=20-u=20origin?= =?UTF-8?q?=20cursor/critical-correctness-bugs-082b?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Yu Le --- scripts/sync-strava-activities-core.mjs | 2 ++ scripts/sync-strava-activities-core.test.mjs | 27 ++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/scripts/sync-strava-activities-core.mjs b/scripts/sync-strava-activities-core.mjs index a426e81..98c3bda 100644 --- a/scripts/sync-strava-activities-core.mjs +++ b/scripts/sync-strava-activities-core.mjs @@ -87,6 +87,8 @@ export async function syncActivities({ const missingIds = freshIds.filter(id => !hasDetail(activitiesDir, id)) if (missingIds.length === 0) { + for (const id of freshIds) knownSet.add(id) + writeJson(path.join(activitiesDir, IDS_FILE), [...knownSet]) const latestTimestamp = getLatestActivityTimestamp(activities) writeJson(path.join(activitiesDir, META_FILE), { ...meta, lastSync: latestTimestamp }) logger.log('✅ All activities up to date') diff --git a/scripts/sync-strava-activities-core.test.mjs b/scripts/sync-strava-activities-core.test.mjs index 3b6665f..5d24f8b 100644 --- a/scripts/sync-strava-activities-core.test.mjs +++ b/scripts/sync-strava-activities-core.test.mjs @@ -68,4 +68,31 @@ describe('syncActivities', () => { detail: true, }) }) + + it('repairs the index when activity details already exist locally', async () => { + const activitiesDir = makeActivitiesDir() + writeJson(path.join(activitiesDir, '_index.json'), []) + writeJson(path.join(activitiesDir, '100.json'), { + id: 100, + start_date: '2023-01-01T00:00:00Z', + }) + + let detailFetches = 0 + + await syncActivities({ + activitiesDir, + getStravaActivities: async () => [{ + id: 100, + start_date: '2023-01-01T00:00:00Z', + }], + getStravaActivityById: async (id) => { + detailFetches++ + return { id } + }, + logger: { log() {} }, + }) + + expect(detailFetches).toBe(0) + expect(readJson(path.join(activitiesDir, '_index.json'))).toEqual([100]) + }) })