-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
f221303
commit 94ef563
Showing
6 changed files
with
130 additions
and
49 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# flyctl launch added from .gitignore | ||
**/node_modules | ||
fly.toml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
# fly.toml app configuration file generated for rss-to-tana on 2023-07-21T12:04:26+02:00 | ||
# | ||
# See https://fly.io/docs/reference/configuration/ for information about how to use this file. | ||
# | ||
|
||
app = "rss-to-tana" | ||
primary_region = "cdg" | ||
|
||
[build] | ||
builder = "heroku/buildpacks:20" | ||
|
||
[env] | ||
NODE_ENV = "production" | ||
|
||
[processes] | ||
rss = "npm start" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,49 +1,62 @@ | ||
const RSSParser = require('rss-parser'); | ||
const cron = require('node-cron'); | ||
|
||
const { handleNewRSSItem } = require('./item'); | ||
const { saveItem } = require('./item'); | ||
const { run } = require('./runner') | ||
const Tana = require('./tana'); | ||
|
||
const parser = new RSSParser(); | ||
|
||
const rssFeeds = [ | ||
{ | ||
url: 'https://lesoreillescurieuses.com/feed/', | ||
// cron: '0 0 23,6 * * *', // 23:00 and 06:00 every day | ||
cron: '0 * * * * *', // 23:00 and 06:00 every day | ||
cron: '0 0 23,6 * * *', // 23:00 and 06:00 every day | ||
toTana: Tana.album, | ||
}, | ||
{ | ||
url: 'https://cmd.wuips.com/rss/feed.xml', | ||
// cron: '0 0 * * * *', // every hour every day | ||
cron: '0 * * * * *', // every hour every day | ||
cron: '0 0 * * * *', // every hour every day | ||
toTana: Tana.website, | ||
}, | ||
|
||
]; | ||
|
||
const startTime = new Date('2022-09-27 18:00:00.000.'); | ||
function parseFeed(feed) { | ||
return async function (lastRunDate) { | ||
try { | ||
console.log(feed.url, `parsing for items published after ${lastRunDate.toISOString()}`) | ||
const parsedFeed = await parser.parseURL(feed.url); | ||
|
||
async function parseFeed(feed) { | ||
try { | ||
console.log(feed.url, 'parsing') | ||
const parsedFeed = await parser.parseURL(feed.url); | ||
for (const item of parsedFeed.items) { | ||
const pubDate = new Date(item.pubDate); | ||
if (pubDate > lastRunDate) { | ||
console.log(feed.url, `new ${item.title} detected`); | ||
|
||
for (const item of parsedFeed.items) { | ||
const pubDate = new Date(item.pubDate); | ||
if (pubDate > startTime) { | ||
const tanaNode = feed.toTana(item) | ||
handleNewRSSItem(feed.url, item, tanaNode); | ||
const tanaNode = feed.toTana(item) | ||
saveItem(tanaNode); | ||
} | ||
} | ||
} catch (error) { | ||
console.error(feed.url, `parsing error`, error); | ||
} | ||
} catch (error) { | ||
console.error(feed.url, `parsing error`, error); | ||
} | ||
} | ||
|
||
|
||
for (const feed of rssFeeds) { | ||
console.log('Scheduling', feed.url, 'on', feed.cron) | ||
/** | ||
* We can use FORCE=true env var to run the feeds parsing directly, without | ||
* cron schedule | ||
*/ | ||
if (process.env.FORCE === 'true') { | ||
run(parseFeed(feed))() | ||
} else { | ||
console.log('Scheduling', feed.url, 'on', feed.cron) | ||
|
||
if (!cron.validate(feed.cron)) { | ||
throw new Error(`${feed.cron} not a valid cron expression`) | ||
} | ||
|
||
cron.schedule(feed.cron, () => parseFeed(feed)) | ||
cron.schedule(feed.cron, run(parseFeed(feed))) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
// starting 1 minute before deployment time to handle potential downtime | ||
const processStartTimeMs = Date.now() - 1 * 60 * 1000; | ||
|
||
/** | ||
* We can pass an optional START_TIME env var to control the RSS parsing start | ||
* date. Default is process start time minus one minute. | ||
* | ||
* Iso date, eg. 2023-06-27T18:00:00.000Z | ||
*/ | ||
const startTime = new Date(process.env.START_TIME || processStartTimeMs); | ||
|
||
console.log(`Using ${startTime.toISOString()} as start time`) | ||
|
||
function run(callback) { | ||
let lastRunDate = startTime; | ||
|
||
return function () { | ||
const now = new Date() | ||
callback(lastRunDate) | ||
lastRunDate = now | ||
} | ||
} | ||
|
||
module.exports = { run } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters