Merge bb74a19 into ce035ba

microlinkhq · Mar 20, 2023 · 5fb9f96 · 5fb9f96
2 parents ce035ba + bb74a19
commit 5fb9f96
Show file tree

Hide file tree

Showing 13 changed files with 27,368 additions and 17,276 deletions.
diff --git a/packages/cheerio-repl/bin/index.mjs b/packages/cheerio-repl/bin/index.mjs
diff --git a/packages/metascraper-date/index.js b/packages/metascraper-date/index.js
@@ -10,20 +10,9 @@ const dateRules = () => {
     toDate($ => $('[itemprop*="date" i]').attr('content')),
     toDate($ => $('time[itemprop*="date" i]').attr('datetime')),
     toDate($ => $('time[datetime]').attr('datetime')),
-    toDate($ => $('meta[name*="dc.date" i]').attr('content')),
-    toDate($ => $('meta[name*="dcterms.date" i]').attr('content')),
-    toDate($ => $('[property*="dc:date" i]').attr('content')),
     toDate($ => $filter($, $('[class*="byline" i]'))),
-    toDate($ => $filter($, $('[class*="dateline" i]'))),
-    toDate($ => $filter($, $('[id*="metadata" i]'))),
-    toDate($ => $filter($, $('[class*="metadata" i]'))), // twitter, move into a bundle of rules
     toDate($ => $filter($, $('[id*="date" i]'))),
     toDate($ => $filter($, $('[class*="date" i]'))),
-    toDate($ => $filter($, $('[id*="post-timestamp" i]'))),
-    toDate($ => $filter($, $('[class*="post-timestamp" i]'))),
-    toDate($ => $filter($, $('[id*="post-meta" i]'))),
-    toDate($ => $filter($, $('[class*="post-meta" i]'))),
-    toDate($ => $filter($, $('[id*="time" i]'))),
     toDate($ => $filter($, $('[class*="time" i]')))
   ]
 }
@@ -33,27 +22,25 @@ const datePublishedRules = () => {
     toDate($jsonld('datePublished')),
     toDate($jsonld('dateCreated')),
     toDate($ => $('meta[property*="published_time" i]').attr('content')),
-    toDate($ => $('meta[property*="release_date" i]').attr('content')),
     toDate($ => $('[itemprop="datepublished" i]').attr('content')),
-    toDate($ => $('time[datetime][pubdate]').attr('datetime')),
-    toDate($ => $('meta[name*="dc.date.issued" i]').attr('content')),
-    toDate($ => $('meta[name*="dc.date.created" i]').attr('content')),
-    toDate($ => $('[property*="dc:created" i]').attr('content')),
-    toDate($ => $filter($, $('[id*="publish" i]'))),
     toDate($ => $filter($, $('[class*="publish" i]')))
   ]
 }
 
 const dateModifiedRules = () => {
   return [
     toDate($jsonld('dateModified')),
-    toDate($ => $('meta[property*="updated_time" i]').attr('content')),
     toDate($ => $('meta[property*="modified_time" i]').attr('content')),
     toDate($ => $('[itemprop*="datemodified" i]').attr('content'))
   ]
 }
 
-module.exports = ({ datePublished, dateModified } = { datePublished: false, dateModified: false }) => {
+module.exports = (
+  { datePublished, dateModified } = {
+    datePublished: false,
+    dateModified: false
+  }
+) => {
   const result = {
     date: dateModifiedRules().concat(datePublishedRules(), dateRules())
   }

diff --git a/packages/metascraper/test/integration/forbes/input.html b/packages/metascraper/test/integration/forbes/input.html
diff --git a/packages/metascraper/test/integration/forbes/snapshots/index.js.md b/packages/metascraper/test/integration/forbes/snapshots/index.js.md
@@ -11,13 +11,13 @@ Generated by [AVA](https://avajs.dev).
     {
       audio: null,
       author: 'George Anders',
-      date: '2015-09-30T15:49:44.000Z',
+      date: '2015-09-30T15:12:00.000Z',
       description: 'HackerRank, a fast-growing company that runs coding contests to identify top software engineers, has hired Facebook and Google veteran Grady Burnett to be its chief operating officer.',
-      image: 'http://blogs-images.forbes.com/georgeanders/files/2015/09/HackerRank-1940x827.jpg',
-      lang: null,
-      logo: 'http://i.forbesimg.com/media/assets/appicons/forbes-app-icon_144x144.png',
+      image: 'https://imageio.forbes.com/blogs-images/georgeanders/files/2015/09/HackerRank-1940x827.jpg?format=jpg&width=1200',
+      lang: 'en',
+      logo: 'https://i.forbesimg.com/48X48-F.png',
       publisher: 'Forbes',
       title: 'Facebook Veteran Grady Burnett Joins HackerRank As COO',
-      url: 'http://www.forbes.com/sites/georgeanders/2015/09/30/facebook-veteran-grady-burnett-joins-hackerrank-as-coo/',
+      url: 'https://www.forbes.com/sites/georgeanders/2015/09/30/facebook-veteran-grady-burnett-joins-hackerrank-as-coo/',
       video: null,
     }
diff --git a/packages/metascraper/test/integration/forbes/snapshots/index.js.snap b/packages/metascraper/test/integration/forbes/snapshots/index.js.snap
diff --git a/packages/metascraper/test/integration/twitter-gif/index.js b/packages/metascraper/test/integration/twitter-gif/index.js
@@ -5,6 +5,7 @@ const { resolve } = require('path')
 const test = require('ava')
 
 const metascraper = require('../../..')([
+  require('metascraper-twitter')(),
   require('metascraper-author')(),
   require('metascraper-date')(),
   require('metascraper-description')(),