Fix Strava sync missing delayed activities

scripts/sync-strava-activities-core.mjs

@@ -0,0 +1,112 @@
+import fs from 'node:fs'
+import path from 'node:path'
+
+const IDS_FILE = '_index.json'
+const META_FILE = '_meta.json'
+
+function readJson(file, fallback) {
+  try {
+    return JSON.parse(fs.readFileSync(file, 'utf-8'))
+  }
+  catch {
+    return fallback
+  }
+}
+
+function writeJson(file, value) {
+  fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`)
+}
+
+function readIds(activitiesDir) {
+  const data = readJson(path.join(activitiesDir, IDS_FILE), [])
+  return Array.isArray(data) ? data : []
+}
+
+function readMeta(activitiesDir) {
+  const data = readJson(path.join(activitiesDir, META_FILE), {})
+  return data && typeof data === 'object' ? data : {}
+}
+
+function hasDetail(activitiesDir, id) {
+  return fs.existsSync(path.join(activitiesDir, `${id}.json`))
+}
+
+function saveDetail(activitiesDir, id, data) {
+  writeJson(path.join(activitiesDir, `${id}.json`), data)
+}
+
+function getLatestActivityTimestamp(activities) {
+  return Math.max(
+    ...activities.map(activity => Math.floor(new Date(activity.start_date).getTime() / 1000)),
+  )
+}
+
+async function fetchAllActivities(getStravaActivities) {
+  const all = []
+  let page = 1
+  while (true) {
+    const batch = await getStravaActivities({ page, perPage: 100 })
+    if (!batch.length)
+      break
+    all.push(...batch)
+    if (batch.length < 100)
+      break
+    page++
+  }
+  return all
+}
+
+export async function syncActivities({
+  activitiesDir,
+  getStravaActivities,
+  getStravaActivityById,
+  logger = console,
+}) {
+  fs.mkdirSync(activitiesDir, { recursive: true })
+
+  logger.log('🔄 Syncing Strava activities...')
+  logger.log('  Full sync: fetching all activities')
+
+  const knownIds = readIds(activitiesDir)
+  const knownSet = new Set(knownIds)
+  const meta = readMeta(activitiesDir)
+
+  const activities = await fetchAllActivities(async (options) => {
+    const batch = await getStravaActivities(options)
+    if (batch.length)
+      logger.log(`  Page ${options.page}: ${batch.length} activities`)
+    return batch
+  })
+
+  if (activities.length === 0) {
+    logger.log('✅ No activities found')
+    return
+  }
+
+  const freshIds = activities.map(a => a.id)
+  const missingIds = freshIds.filter(id => !hasDetail(activitiesDir, id))
+
+  if (missingIds.length === 0) {
+    for (const id of freshIds) knownSet.add(id)
+    writeJson(path.join(activitiesDir, IDS_FILE), [...knownSet])
+    const latestTimestamp = getLatestActivityTimestamp(activities)
+    writeJson(path.join(activitiesDir, META_FILE), { ...meta, lastSync: latestTimestamp })
+    logger.log('✅ All activities up to date')
+    return
+  }
+
+  logger.log(`⬇️  Fetching ${missingIds.length} activity details...`)
+  for (const id of missingIds) {
+    logger.log(`  ${id}`)
+    const detail = await getStravaActivityById(id)
+    saveDetail(activitiesDir, id, detail)
+  }
+
+  for (const id of freshIds) knownSet.add(id)
+  writeJson(path.join(activitiesDir, IDS_FILE), [...knownSet])
+
+  const latestTimestamp = getLatestActivityTimestamp(activities)
+  writeJson(path.join(activitiesDir, META_FILE), { ...meta, lastSync: latestTimestamp })
+
+  logger.log(`✅ Synced ${missingIds.length} activities (total: ${knownSet.size})`)
+}

scripts/sync-strava-activities-core.test.mjs

@@ -0,0 +1,98 @@
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+import { afterEach, describe, expect, it } from 'vitest'
+import { syncActivities } from './sync-strava-activities-core.mjs'
+
+const tmpDirs = []
+
+function makeActivitiesDir() {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'strava-sync-'))
+  tmpDirs.push(dir)
+  fs.mkdirSync(dir, { recursive: true })
+  return dir
+}
+
+function writeJson(file, value) {
+  fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`)
+}
+
+function readJson(file) {
+  return JSON.parse(fs.readFileSync(file, 'utf-8'))
+}
+
+afterEach(() => {
+  for (const dir of tmpDirs.splice(0)) {
+    fs.rmSync(dir, { recursive: true, force: true })
+  }
+})
+
+describe('syncActivities', () => {
+  it('discovers late-uploaded activities whose start date predates the checkpoint', async () => {
+    const activitiesDir = makeActivitiesDir()
+    writeJson(path.join(activitiesDir, '_index.json'), [200])
+    writeJson(path.join(activitiesDir, '_meta.json'), { lastSync: 1_700_000_000 })
+    writeJson(path.join(activitiesDir, '200.json'), {
+      id: 200,
+      start_date: '2023-11-14T22:13:20Z',
+    })
+
+    const listCalls = []
+    const lateUploadedActivity = {
+      id: 100,
+      start_date: '2023-01-01T00:00:00Z',
+    }
+    const knownActivity = {
+      id: 200,
+      start_date: '2023-11-14T22:13:20Z',
+    }
+
+    await syncActivities({
+      activitiesDir,
+      getStravaActivities: async (options) => {
+        listCalls.push(options)
+        return listCalls.length === 1 ? [knownActivity, lateUploadedActivity] : []
+      },
+      getStravaActivityById: async id => ({
+        id,
+        start_date: lateUploadedActivity.start_date,
+        detail: true,
+      }),
+      logger: { log() {} },
+    })
+
+    expect(listCalls[0]).not.toHaveProperty('after')
+    expect(readJson(path.join(activitiesDir, '_index.json'))).toEqual([200, 100])
+    expect(readJson(path.join(activitiesDir, '100.json'))).toMatchObject({
+      id: 100,
+      detail: true,
+    })
+  })
+
+  it('repairs the index when activity details already exist locally', async () => {
+    const activitiesDir = makeActivitiesDir()
+    writeJson(path.join(activitiesDir, '_index.json'), [])
+    writeJson(path.join(activitiesDir, '100.json'), {
+      id: 100,
+      start_date: '2023-01-01T00:00:00Z',
+    })
+
+    let detailFetches = 0
+
+    await syncActivities({
+      activitiesDir,
+      getStravaActivities: async () => [{
+        id: 100,
+        start_date: '2023-01-01T00:00:00Z',
+      }],
+      getStravaActivityById: async (id) => {
+        detailFetches++
+        return { id }
+      },
+      logger: { log() {} },
+    })
+
+    expect(detailFetches).toBe(0)
+    expect(readJson(path.join(activitiesDir, '_index.json'))).toEqual([100])
+  })
+})

scripts/sync-strava-activities.mjs

@@ -1,139 +1,20 @@
-import fs from 'node:fs'
 import path from 'node:path'
 import process from 'node:process'
 import { fileURLToPath } from 'node:url'
 import {
   getStravaActivities,
   getStravaActivityById,
 } from '../src/utils/strava.ts'
+import { syncActivities } from './sync-strava-activities-core.mjs'
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
 const ACTIVITIES_DIR = path.join(__dirname, '..', 'data', 'strava', 'activities')
-const IDS_FILE = path.join(ACTIVITIES_DIR, '_index.json')
-const META_FILE = path.join(ACTIVITIES_DIR, '_meta.json')
 
-fs.mkdirSync(ACTIVITIES_DIR, { recursive: true })
-
-function readIds() {
-  try {
-    const data = JSON.parse(fs.readFileSync(IDS_FILE, 'utf-8'))
-    return Array.isArray(data) ? data : []
-  }
-  catch {
-    return []
-  }
-}
-
-function readMeta() {
-  try {
-    const data = JSON.parse(fs.readFileSync(META_FILE, 'utf-8'))
-    return data || {}
-  }
-  catch {
-    return {}
-  }
-}
-
-function writeMeta(meta) {
-  fs.writeFileSync(META_FILE, `${JSON.stringify(meta, null, 2)}\n`)
-}
-
-function writeIds(ids) {
-  fs.writeFileSync(IDS_FILE, `${JSON.stringify(ids, null, 2)}\n`)
-}
-
-function hasDetail(id) {
-  return fs.existsSync(path.join(ACTIVITIES_DIR, `${id}.json`))
-}
-
-function saveDetail(id, data) {
-  fs.writeFileSync(
-    path.join(ACTIVITIES_DIR, `${id}.json`),
-    `${JSON.stringify(data, null, 2)}\n`,
-  )
-}
-
-function getLatestActivityTimestamp(activities) {
-  return Math.max(
-    ...activities.map(activity => Math.floor(new Date(activity.start_date).getTime() / 1000)),
-  )
-}
-
-async function fetchAllActivities(after = null) {
-  const all = []
-  let page = 1
-  while (true) {
-    const options = { page, perPage: 100 }
-    if (after) {
-      options.after = after
-    }
-    const batch = await getStravaActivities(options)
-    if (!batch.length)
-      break
-    all.push(...batch)
-    console.log(`  Page ${page}: ${batch.length} activities`)
-    if (batch.length < 100)
-      break
-    page++
-  }
-  return all
-}
-
-async function syncActivities() {
-  console.log('🔄 Syncing Strava activities...')
-
-  const knownIds = readIds()
-  const knownSet = new Set(knownIds)
-  const meta = readMeta()
-
-  // Use last sync timestamp for incremental fetch
-  const lastSync = meta.lastSync || null
-  const isFullSync = !lastSync
-
-  if (isFullSync) {
-    console.log('  Full sync: fetching all activities (no previous sync)')
-  }
-  else {
-    const syncDate = new Date(lastSync * 1000).toISOString()
-    console.log(`  Incremental sync: fetching activities since ${syncDate}`)
-  }
-
-  // Fetch activities (incremental if we have a lastSync timestamp)
-  const activities = await fetchAllActivities(lastSync)
-
-  if (activities.length === 0) {
-    console.log('✅ No new activities found')
-    return
-  }
-
-  const freshIds = activities.map(a => a.id)
-  const missingIds = freshIds.filter(id => !hasDetail(id))
-
-  if (missingIds.length === 0) {
-    const latestTimestamp = getLatestActivityTimestamp(activities)
-    writeMeta({ ...meta, lastSync: latestTimestamp })
-    console.log('✅ All activities up to date')
-    return
-  }
-
-  console.log(`⬇️  Fetching ${missingIds.length} activity details...`)
-  for (const id of missingIds) {
-    console.log(`  ${id}`)
-    const detail = await getStravaActivityById(id)
-    saveDetail(id, detail)
-  }
-
-  for (const id of freshIds) knownSet.add(id)
-  writeIds([...knownSet])
-
-  // Update lastSync timestamp based on the latest activity fetched
-  const latestTimestamp = getLatestActivityTimestamp(activities)
-  writeMeta({ ...meta, lastSync: latestTimestamp })
-
-  console.log(`✅ Synced ${missingIds.length} activities (total: ${knownSet.size})`)
-}
-
-syncActivities().catch((err) => {
+syncActivities({
+  activitiesDir: ACTIVITIES_DIR,
+  getStravaActivities,
+  getStravaActivityById,
+}).catch((err) => {
   console.error('❌ Sync failed:', err.message)
   process.exit(1)
 })