From db4b791f075d90b3c3da4351062ab52956e22481 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Tue, 18 Jun 2024 11:18:05 -0400 Subject: [PATCH 1/2] feat: save feed entry contents into topic contents This change utilises the `content` property from rss-parser to populate the topic post content. That property contains HTML, so additional logic is present here to instruct the markdown plugin to ignore it. For versions of NodeBB that don't have an up-to-date Markdown plugin, the HTML will be parsed (and likely escaped), but admins can toggle "Allow HTML" to have it rendered properly. --- index.js | 15 +++++++++++++++ lib/feed.js | 1 + lib/pull.js | 8 ++++++-- plugin.json | 4 +++- 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 0e11e8f..2d88e64 100644 --- a/index.js +++ b/index.js @@ -3,6 +3,7 @@ const db = require.main.require('./src/database'); const pubsub = require.main.require('./src/pubsub'); const routeHelpers = require.main.require('./src/routes/helpers'); +const utils = require.main.require('./src/utils'); const database = require('./lib/database'); const controllers = require('./lib/controllers'); @@ -96,3 +97,17 @@ RssPlugin.widgets.defineWidgets = widget.defineWidgets; */ RssPlugin.widgets.renderRssWidget = widget.render; +RssPlugin.skipMarkdown = async ({ env, data }) => { + const feedUrls = await db.getSetMembers('nodebb-plugin-rss:feeds'); + const keys = feedUrls.map(url => `nodebb-plugin-rss:feed:${url}:uuid`); + const { tid } = data.postData; + + if (utils.isNumber(tid)) { + const entries = await db.getSortedSetRangeByScore(keys, 0, 1, tid, tid); + if (entries.length) { + env.parse = false; + } + } + + return { env, data }; +}; diff --git a/lib/feed.js b/lib/feed.js index cc6082f..f53e05e 100644 --- a/lib/feed.js +++ b/lib/feed.js @@ -21,6 +21,7 @@ Feed.getItems = async function (feedUrl, entriesToPull = Feed.DEFAULT_ENTRIES_TO published: item.pubDate, link: { href: item.link }, id: item.guid || item.id, + content: item.content, tags: item.categories, })); }; diff --git a/lib/pull.js b/lib/pull.js index 3ddb582..4fbca1f 100644 --- a/lib/pull.js +++ b/lib/pull.js @@ -75,10 +75,14 @@ async function postEntry(feed, entry) { winston.info(`[plugin-rss] posting, ${feed.url} - title: ${entry.title}, published date: ${getEntryDate(entry)}`); + const url = entry.link && entry.link.href; + const content = !entry.content.includes(url) ? + `${entry.content}

${url}` : + entry.content; const result = await topics.post({ uid: posterUid, title: entry.title, - content: entry.link && entry.link.href, + content, cid: feed.category, tags: tags, }); @@ -92,7 +96,7 @@ async function postEntry(feed, entry) { const max = Math.max(parseInt(meta.config.postDelay, 10) || 10, parseInt(meta.config.newbiePostDelay, 10) || 10) + 1; await user.setUserField(posterUid, 'lastposttime', Date.now() - (max * 1000)); - const uuid = entry.id || (entry.link && entry.link.href) || entry.title; + const uuid = entry.id || (url) || entry.title; await db.sortedSetAdd(`nodebb-plugin-rss:feed:${feed.url}:uuid`, topicData.tid, uuid); } diff --git a/plugin.json b/plugin.json index caafa8f..6b23da0 100644 --- a/plugin.json +++ b/plugin.json @@ -14,7 +14,9 @@ { "hook": "action:topic.purge", "method": "onTopicPurge"}, { "hook": "filter:widgets.getWidgets", "method": "widgets.defineWidgets" }, - { "hook": "filter:widget.render:rss", "method": "widgets.renderRssWidget" } + { "hook": "filter:widget.render:rss", "method": "widgets.renderRssWidget" }, + + { "hook": "filter:markdown.beforeParse", "method": "skipMarkdown" } ], "templates": "./templates", "modules": { From a7a6d1bb271a45af1de7d0d07a995b86e9783f21 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Thu, 20 Jun 2024 21:59:14 -0400 Subject: [PATCH 2/2] fix: use contentSnippet instead of content (so no markdown skipParsing shenanigans) --- index.js | 15 --------------- lib/feed.js | 2 +- lib/pull.js | 5 +---- plugin.json | 4 +--- 4 files changed, 3 insertions(+), 23 deletions(-) diff --git a/index.js b/index.js index 2d88e64..573aa9b 100644 --- a/index.js +++ b/index.js @@ -96,18 +96,3 @@ RssPlugin.widgets.defineWidgets = widget.defineWidgets; * Called on `filter:widget.render:rss` */ RssPlugin.widgets.renderRssWidget = widget.render; - -RssPlugin.skipMarkdown = async ({ env, data }) => { - const feedUrls = await db.getSetMembers('nodebb-plugin-rss:feeds'); - const keys = feedUrls.map(url => `nodebb-plugin-rss:feed:${url}:uuid`); - const { tid } = data.postData; - - if (utils.isNumber(tid)) { - const entries = await db.getSortedSetRangeByScore(keys, 0, 1, tid, tid); - if (entries.length) { - env.parse = false; - } - } - - return { env, data }; -}; diff --git a/lib/feed.js b/lib/feed.js index f53e05e..1670d5d 100644 --- a/lib/feed.js +++ b/lib/feed.js @@ -21,7 +21,7 @@ Feed.getItems = async function (feedUrl, entriesToPull = Feed.DEFAULT_ENTRIES_TO published: item.pubDate, link: { href: item.link }, id: item.guid || item.id, - content: item.content, + content: item.contentSnippet, tags: item.categories, })); }; diff --git a/lib/pull.js b/lib/pull.js index 4fbca1f..1ce3cc6 100644 --- a/lib/pull.js +++ b/lib/pull.js @@ -76,13 +76,10 @@ async function postEntry(feed, entry) { winston.info(`[plugin-rss] posting, ${feed.url} - title: ${entry.title}, published date: ${getEntryDate(entry)}`); const url = entry.link && entry.link.href; - const content = !entry.content.includes(url) ? - `${entry.content}

${url}` : - entry.content; const result = await topics.post({ uid: posterUid, title: entry.title, - content, + content: `${entry.content}\n\n----\n\n${url}`, cid: feed.category, tags: tags, }); diff --git a/plugin.json b/plugin.json index 6b23da0..caafa8f 100644 --- a/plugin.json +++ b/plugin.json @@ -14,9 +14,7 @@ { "hook": "action:topic.purge", "method": "onTopicPurge"}, { "hook": "filter:widgets.getWidgets", "method": "widgets.defineWidgets" }, - { "hook": "filter:widget.render:rss", "method": "widgets.renderRssWidget" }, - - { "hook": "filter:markdown.beforeParse", "method": "skipMarkdown" } + { "hook": "filter:widget.render:rss", "method": "widgets.renderRssWidget" } ], "templates": "./templates", "modules": {