Fix news cron locks and fetching to work more than once per day.

bewcloud · Apr 19, 2024 · 2920de9 · 2920de9
1 parent a8076f6
commit 2920de9
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 8 deletions.
diff --git a/crons/news.ts b/crons/news.ts
@@ -10,9 +10,9 @@ export async function fetchNewArticles(forceFetch = false) {
 
   try {
     const feedsToCrawl = await db.query<NewsFeed>(
-      sql`SELECT * FROM "bewcloud_news_feeds" WHERE "last_crawled_at" IS NULL OR "last_crawled_at" <= $1`,
+      sql`SELECT * FROM "bewcloud_news_feeds" WHERE "last_crawled_at" IS NULL OR "last_crawled_at" <= $1 ORDER BY "last_crawled_at" ASC`,
       [
-        fourHoursAgo.toISOString().substring(0, 10),
+        fourHoursAgo.toISOString(),
       ],
     );
 

diff --git a/lib/data/news.ts b/lib/data/news.ts
@@ -199,8 +199,8 @@ async function fetchNewsArticles(newsFeed: NewsFeed): Promise<Feed['entries'] |
 
     return (feed as Feed)?.entries || (feed as JsonFeed)?.items || [];
   } catch (error) {
-    console.log('Failed parsing feed to get articles', newsFeed.feed_url);
-    console.log(error);
+    console.error('Failed parsing feed to get articles', newsFeed.feed_url);
+    console.error(error);
   }
 
   return [];
@@ -296,11 +296,13 @@ export async function crawlNewsFeed(newsFeed: NewsFeed) {
       }
     }
 
-    console.log('Added', addedArticlesCount, 'new articles');
+    console.info('Added', addedArticlesCount, 'new articles');
 
     newsFeed.last_crawled_at = new Date();
 
     await updateNewsFeed(newsFeed);
+
+    lock.release();
   } catch (error) {
     lock.release();
 

diff --git a/lib/interfaces/locker.ts b/lib/interfaces/locker.ts
@@ -4,7 +4,7 @@ function wait(milliseconds = 100) {
   return new Promise((resolve) => setTimeout(() => resolve(true), milliseconds));
 }
 
-const expirationInMilliseconds = 15 * 1000;
+const expirationInMilliseconds = 15_000;
 
 const locks: Map<string, { expiresAt: Date }> = new Map();
 
@@ -16,13 +16,20 @@ export default class Locker {
   }
 
   public async acquire() {
-    const currentLock = locks.get(this.lockName);
+    console.debug('Acquiring lock:', this.lockName);
+    let currentLock = locks.get(this.lockName);
 
     while (currentLock) {
       // Only wait if the lock hasn't expired
       if (currentLock.expiresAt > new Date()) {
+        console.debug('Waiting for lock to expire:', this.lockName);
         await wait();
+      } else {
+        // Release lock since it has expired
+        this.release();
       }
+
+      currentLock = locks.get(this.lockName);
     }
 
     locks.set(this.lockName, {
@@ -31,6 +38,7 @@ export default class Locker {
   }
 
   public release() {
+    console.debug('Releasing lock:', this.lockName);
     locks.delete(this.lockName);
   }
 }
diff --git a/lib/utils/misc.ts b/lib/utils/misc.ts
@@ -121,7 +121,7 @@ export async function concurrentPromises<T>(
   return results;
 }
 
-const MAX_RESPONSE_TIME_IN_MS = 10 * 1000;
+const MAX_RESPONSE_TIME_IN_MS = 10_000;
 
 export async function fetchUrl(url: string) {
   const abortController = new AbortController();