feat(bot): switch to metascraper for url unfurling

Closes #10

Author: ericrallen

components/routes/feed.js

@@ -95,18 +95,20 @@ const generateFeed = (controller, teamId, channel, req, res, next) => {
         feed.item(link.item);
       });
 
-      const cache = {
-        id: `${teamId}::${channel}`,
-        feed,
-      };
-
-      controller.storage.feeds.save(cache, function(err, id) {
-        if (err) {
-          console.error('ERROR: could not cache feed:', err);
-        }
-
-        console.log(`${teamId}::${channel} feed cached.`);
-      });
+      if (process.env.CACHE === 'TRUE') {
+        const cache = {
+          id: `${teamId}::${channel}`,
+          feed,
+        };
+
+        controller.storage.feeds.save(cache, function(err, id) {
+          if (err) {
+            console.error('ERROR: could not cache feed:', err);
+          }
+
+          console.log(`${teamId}::${channel} feed cached.`);
+        });
+      }
 
       res
         .set('Content-Type', 'application/rss+xml')

demo/scrape.js

@@ -0,0 +1,28 @@
+const metascraper = require('metascraper')([
+  require('metascraper-youtube')(),
+  require('metascraper-soundcloud')(),
+  require('metascraper-readability')(),
+  require('metascraper-amazon')(),
+  require('metascraper-audio')(),
+  require('metascraper-video')(),
+  require('metascraper-author')(),
+  require('metascraper-date')(),
+  require('metascraper-description')(),
+  require('metascraper-image')(),
+  require('metascraper-clearbit-logo')(),
+  require('metascraper-logo')(),
+  require('metascraper-logo-favicon')(),
+  require('metascraper-media-provider')(),
+  require('metascraper-publisher')(),
+  require('metascraper-title')(),
+  require('metascraper-url')(),
+]);
+
+const got = require('got');
+
+;(async () => {
+  console.log(`Scraping ${process.argv[2]}...`);
+  const { body: html, url } = await got(process.argv[2]);
+  const metadata = await metascraper({ html, url });
+  console.log(metadata);
+})();

example.env

@@ -9,6 +9,6 @@ clientSecret=
 clientSigningSecret=
 studio_token=
 PORT=
-CACHE=FALSE
+CACHE=FALSE # TRUE | FALSE
 
 # note: .env is a shell file so there can’t be spaces around =