From 33d49c5f525140a4555fc52202a1d04b95a80519 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Sat, 16 Mar 2019 21:32:26 +0100 Subject: [PATCH] add metascraper-readability (#155) --- README.md | 1 + packages/metascraper-readability/.npmrc | 4 + packages/metascraper-readability/README.md | 19 + .../__snapshots__/index.js.snap-shot | 21 + packages/metascraper-readability/index.js | 33 + packages/metascraper-readability/package.json | 50 + .../test/fixtures/kikobeats.com.html | 552 +++++++++++ .../test/fixtures/learnnode.com.html | 895 ++++++++++++++++++ .../test/fixtures/microlink.io.html | 181 ++++ .../metascraper-readability/test/index.js | 33 + .../metascraper-readability/test/mocha.opts | 5 + .../__snapshots__/index.js.snap-shot | 2 +- .../test/integration/anandtech/index.js | 3 +- .../test/integration/arstechnica/index.js | 3 +- .../test/integration/astier/index.js | 3 +- .../test/integration/atlasobscura/index.js | 3 +- .../test/integration/audiense/index.js | 3 +- .../metascraper/test/integration/bbc/index.js | 3 +- .../test/integration/bloomberg/index.js | 3 +- .../test/integration/business-today/index.js | 3 +- .../metascraper/test/integration/cbr/index.js | 3 +- .../metascraper/test/integration/cio/index.js | 3 +- .../test/integration/cloud-pro/index.js | 3 +- .../test/integration/cnet/index.js | 3 +- .../test/integration/computerworld/index.js | 3 +- .../metascraper/test/integration/crn/index.js | 3 +- .../test/integration/economic-times/index.js | 3 +- .../test/integration/engadget/index.js | 6 +- .../test/integration/entrepreneur/index.js | 3 +- .../test/integration/eweek/index.js | 3 +- .../test/integration/fast-company/index.js | 3 +- .../test/integration/fastersite/index.js | 3 +- .../test/integration/fierce-devops/index.js | 3 +- .../test/integration/financial-times/index.js | 6 +- .../test/integration/forbes/index.js | 3 +- .../test/integration/fortune/index.js | 3 +- .../test/integration/geek-time/index.js | 3 +- .../test/integration/github/index.js | 3 +- .../test/integration/googleblog/index.js | 3 +- .../metascraper/test/integration/inc/index.js | 3 +- .../integration/information-week/index.js | 3 +- .../test/integration/instagram/index.js | 3 +- .../integration/jewish-business-news/index.js | 3 +- .../test/integration/lean-data/index.js | 6 +- .../test/integration/learnnode/index.js | 3 +- .../integration/linkedin-company/index.js | 3 +- .../test/integration/linkedin-pulse/index.js | 3 +- .../integration/los-angeles-times/index.js | 6 +- .../test/integration/mac-rumors/index.js | 6 +- .../test/integration/market-wired/index.js | 3 +- .../test/integration/marketing-land/index.js | 6 +- .../test/integration/mashable/index.js | 3 +- .../test/integration/medium/index.js | 3 +- .../test/integration/motherboard/index.js | 6 +- .../test/integration/nytimes/index.js | 3 +- .../test/integration/pikabu/index.js | 3 +- .../test/integration/pr-newswire/index.js | 3 +- .../test/integration/recode/index.js | 3 +- .../test/integration/reuters/index.js | 6 +- .../san-francisco-chronicle/index.js | 3 +- .../test/integration/segment-academy/index.js | 6 +- .../test/integration/segment/index.js | 3 +- .../test/integration/silicon-angle/index.js | 3 +- .../test/integration/silicon-beat/index.js | 3 +- .../test/integration/silicon-tap/index.js | 3 +- .../silicon-valley-business-journal/index.js | 3 +- .../test/integration/smitten-kitchen/index.js | 3 +- .../test/integration/startup-grind/index.js | 3 +- .../test/integration/techcrunch/index.js | 3 +- .../integration/the-boston-globe/index.js | 3 +- .../test/integration/the-guardian/index.js | 3 +- .../test/integration/the-register/index.js | 6 +- .../test/integration/the-verge/index.js | 3 +- .../test/integration/twitter-gif/index.js | 3 +- .../test/integration/twitter-image/index.js | 3 +- .../test/integration/usa-today/index.js | 6 +- .../test/integration/venture-beat/index.js | 3 +- .../test/integration/vimeo/index.js | 3 +- .../test/integration/washington-post/index.js | 3 +- .../test/integration/wikipedia/index.js | 3 +- .../test/integration/wired/index.js | 6 +- .../metascraper/test/integration/wsj/index.js | 3 +- .../test/integration/xconomy/index.js | 3 +- .../test/integration/yahoo-news/index.js | 6 +- .../test/integration/zdnet/index.js | 3 +- 85 files changed, 1954 insertions(+), 100 deletions(-) create mode 100644 packages/metascraper-readability/.npmrc create mode 100644 packages/metascraper-readability/README.md create mode 100644 packages/metascraper-readability/__snapshots__/index.js.snap-shot create mode 100644 packages/metascraper-readability/index.js create mode 100644 packages/metascraper-readability/package.json create mode 100644 packages/metascraper-readability/test/fixtures/kikobeats.com.html create mode 100644 packages/metascraper-readability/test/fixtures/learnnode.com.html create mode 100644 packages/metascraper-readability/test/fixtures/microlink.io.html create mode 100644 packages/metascraper-readability/test/index.js create mode 100644 packages/metascraper-readability/test/mocha.opts diff --git a/README.md b/README.md index 3b6525453..d5874ce24 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,7 @@ const metascraper = require('metascraper')([ | [`metascraper-logo-favicon`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-logo-favicon) | [![npm](https://img.shields.io/npm/v/metascraper-logo-favicon.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-logo-favicon) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-logo-favicon&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-logo-favicon) | | [`metascraper-media-provider`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-media-provider) | [![npm](https://img.shields.io/npm/v/metascraper-media-provider.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-media-provider) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-media-provider&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-media-provider) | | [`metascraper-publisher`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-publisher) | [![npm](https://img.shields.io/npm/v/metascraper-publisher.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-publisher) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-publisher&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-publisher) | +| [`metascraper-readability`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-readability) | [![npm](https://img.shields.io/npm/v/metascraper-readability.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-readability) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-readability&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-readability) | | [`metascraper-soundcloud`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-soundcloud) | [![npm](https://img.shields.io/npm/v/metascraper-soundcloud.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-soundcloud) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-soundcloud&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-soundcloud) | | [`metascraper-title`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-title) | [![npm](https://img.shields.io/npm/v/metascraper-title.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-title) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-title&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-title) | | [`metascraper-url`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-url) | [![npm](https://img.shields.io/npm/v/metascraper-url.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-url) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-url&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-url) | diff --git a/packages/metascraper-readability/.npmrc b/packages/metascraper-readability/.npmrc new file mode 100644 index 000000000..e03e941f7 --- /dev/null +++ b/packages/metascraper-readability/.npmrc @@ -0,0 +1,4 @@ +unsafe-perm=true +save-prefix=~ +shrinkwrap=false +save=false diff --git a/packages/metascraper-readability/README.md b/packages/metascraper-readability/README.md new file mode 100644 index 000000000..8740c8c32 --- /dev/null +++ b/packages/metascraper-readability/README.md @@ -0,0 +1,19 @@ +# metascraper-readability + +[![npm](https://img.shields.io/npm/v/metascraper-readability.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-readability) +[![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-readability&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-readability) + +> A Readability connector for metascraper + +## Install + +```bash +$ npm install metascraper-readability --save +``` + +## License + +**metascraper-readability** © [microlink.io](https://microlink.io), Released under the [MIT](https://github.com/microlinkhq/metascraper-readability/blob/master/LICENSE.md) License.
+Authored and maintained by microlink.io with help from [contributors](https://github.com/microlinkhq/metascraper-readability/contributors). + +> [microlink.io](https://microlink.io) · GitHub [@microlink.io](https://github.com/microlinkhq) · Twitter [@microlinkhq](https://twitter.com/microlinkhq) diff --git a/packages/metascraper-readability/__snapshots__/index.js.snap-shot b/packages/metascraper-readability/__snapshots__/index.js.snap-shot new file mode 100644 index 000000000..01f833e3a --- /dev/null +++ b/packages/metascraper-readability/__snapshots__/index.js.snap-shot @@ -0,0 +1,21 @@ +exports['learnnode.com 1'] = { + "description": "A premium training course to learn to build apps with Node.js, Express, MongoDB, and friends.", + "publisher": "Learn Node", + "author": null, + "title": "Learn Node" +} + +exports['microlink.io 1'] = { + "description": "Extract structured data from any website. Enter an URL, receive information. Get relevant information from any link & easily create beautiful previews.", + "publisher": "Microlink", + "author": null, + "title": "Turns any website into data" +} + +exports['kikobeats.com 1'] = { + "description": "A millennial doing stuff on internet that ships software every day and builds digital products.", + "publisher": "Kikobeats", + "author": "kikobeats", + "title": "Kikobeats" +} + diff --git a/packages/metascraper-readability/index.js b/packages/metascraper-readability/index.js new file mode 100644 index 000000000..8a19237e5 --- /dev/null +++ b/packages/metascraper-readability/index.js @@ -0,0 +1,33 @@ +'use strict' + +const helpers = require('@metascraper/helpers') +const { eq, get, invoke } = require('lodash') +const Readability = require('readability') +const memoizeOne = require('memoize-one') +const { JSDOM } = require('jsdom') + +const memoFn = (newArgs, oldArgs) => eq(newArgs.url, oldArgs.url) + +const readability = opts => { + const dom = new JSDOM(opts) + const reader = new Readability(dom.window.document) + return reader.parse() +} + +const getReadbility = ({ from, to = from }) => + memoizeOne(({ htmlDom, url }) => { + const data = readability(htmlDom.html(), { url }) + const value = get(data, from) + return invoke(helpers, to, value) + }, memoFn) + +module.exports = () => { + return { + description: getReadbility({ from: 'excerpt', to: 'description' }), + publisher: getReadbility({ from: 'siteName', to: 'publisher' }), + author: getReadbility({ from: 'byline', to: 'author' }), + title: getReadbility({ from: 'title' }) + } +} + +module.exports.readability = readability diff --git a/packages/metascraper-readability/package.json b/packages/metascraper-readability/package.json new file mode 100644 index 000000000..f277164ae --- /dev/null +++ b/packages/metascraper-readability/package.json @@ -0,0 +1,50 @@ +{ + "name": "metascraper-readability", + "description": "A Readability connector for metascraper", + "homepage": "https://metascraper.js.org", + "version": "4.10.1", + "main": "index.js", + "repository": { + "type": "git", + "url": "https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-readability" + }, + "bugs": { + "url": "https://github.com/microlinkhq/metascraper/issues" + }, + "keywords": [ + "metascraper", + "url" + ], + "dependencies": { + "@metascraper/helpers": "^4.10.1", + "jsdom": "~14.0.0", + "lodash": "~4.17.11", + "memoize-one": "~5.0.0", + "readability": "github:mozilla/readability" + }, + "devDependencies": { + "metascraper": "latest", + "mocha": "latest", + "nyc": "latest", + "snap-shot": "latest", + "standard": "latest" + }, + "engines": { + "node": ">= 8" + }, + "files": [ + "index.js" + ], + "scripts": { + "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" + }, + "license": "MIT", + "peerDependencies": { + "metascraper": "^4" + }, + "standard": { + "env": [ + "mocha" + ] + } +} diff --git a/packages/metascraper-readability/test/fixtures/kikobeats.com.html b/packages/metascraper-readability/test/fixtures/kikobeats.com.html new file mode 100644 index 000000000..3fa25d4e3 --- /dev/null +++ b/packages/metascraper-readability/test/fixtures/kikobeats.com.html @@ -0,0 +1,552 @@ + + + + + + + + + + + + + + + + + + + + + + + + + +Kikobeats | A millennial doing stuff on internet that ships software every day and builds digital products. + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ +
+

Hey,

+

+ I’m Kiko Beats, a + millennial doing stuff on internet working at IBM. +

+
+
+
Age
+
26
+
+
+
From
+
Murcia, Spain
+
+
+
Language
+
JavaScript
+
+
+
Favorite Sport
+
Windsurfing
+
+
+ +
+
+
+
+
+ + + +
+
+
+

Featured Projects

+
+
+ +
+ +
+
+
+

Libraries

+
+
+ +
+ + + +
+

+ ... and a + lot more! +

+
+
+
+ +
+
+
+

Microservices

+
+
+ + +
+ +
+
+
+

Writings

+
+
+ + + +
+

+ Read more on my blog. +

+
+
+ + + + + + + + + + + + + + + + + + + diff --git a/packages/metascraper-readability/test/fixtures/learnnode.com.html b/packages/metascraper-readability/test/fixtures/learnnode.com.html new file mode 100644 index 000000000..ed546ee75 --- /dev/null +++ b/packages/metascraper-readability/test/fixtures/learnnode.com.html @@ -0,0 +1,895 @@ + + + + Learn Node — The best way to learn Node.js, Express, MongoDB, and Friends + + + + + + + + + + + + + + + + + + + + +
+
+

The best way to learn server-side JavaScript!

+ +

A premium training course to learn to build apps with Node.js, Express, MongoDB, and friends. +

Start Learning Now → +
+
+
+
+
+ +
+
+

Learn to build applications and APIs with Node.js

+

Node.js, Express, MongoDB and friends are some of the most in-demand web development skills. This course is the cumulation of everything I've learned building dozens of Node.js applications over the past five years.

+

With a focus on modern JavaScript and real world application, Learn Node is an efficient way to add server side JavaScript to your skill-set and start building the applications you have been dreaming about.

+

Join me as I show you how to build full stack applications and APIs from start to finish with some of today's top JavaScript technology.

Buy the Course → +
+
+
+
+
+
+
+
+
    +
  • Node.js
  • +
  • MongoDB
  • +
  • Google Maps SDK
  • +
  • Passport JS
  • +
  • Express JS
  • +
  • Sass
  • +
  • Pug
  • +
  • Webpack
  • +
  • ES6
  • +
  • As Seen on TV Logo
  • +
+
+
+
+ +
+

Together we will build "Now That's Delicious!", a full stack restaurant application which users can search, geolocate, review and curate their favourite restaurants from around the world.

+

The application has three main models — Users, Stores and Reviews — all of which are relational. It is designed to hit upon many of today's application needs such as user authentication, database storage, Ajax REST API, file upload and image resizing.

+

Check out the live application here or see a comprehensive listing of all topics covered below.

+
+
+
Laptop Showing example Application we build in the course + +
+
iPhone Showing example Application we build in the course + +
+
+
+ +
+
+
+
+

Sweet Code! — The code in the application has been reviewed and influenced by some of our industry's best Node.js developers.

+
+
+

Jed Watson

+

JavaScript developer creator of Keystone JS which is a CMS built on top of Express. Elemental UI and Touchstone JS. Co-host of React Sydney and the React Podcast.

+
+
+

Jamund Ferguson

+

JavaScript Engineer at PayPal. Has done a lot of research into async + await with Node.js and is a key advisor to how the error handling is done in this course.

+
+
+
+
+
+ +
+
Async + Await APIs used in this course
+
+

You'll learn with the latest and greatest updates to JavaScript.

+

ES2017 Async + Await allows us to craft easy to read, logical flows without relying on external flow control libraries, chaining multiple promises, or writing spaghetti code by nesting callbacks.

+

ES6 features are heavily used throughout the course — from destructuring to arrow functions you'll get to see how to implement these new features in real world scenarios.

+
+
+

Wondering where the try / catch is in the screenshot? Join the course to see how we handle errors with something called function composition.

+
+
+
+
+
Wes Bos +

Meet Wes Bos

+

Wes Bos is a Full Stack Developer, Speaker and Teacher from Canada. He works as an independent web developer and teaches as a lead instructor with HackerYou. Wes has taught over 500 students in 200+ classes and spoken at dozens of conferences around the world.

+

Wes is the author of React For Beginners, ES6 for Everyone and Sublime Text Power User which together have sold over 25,000 copies. He is also the author of JavaScript30.com, LearnRedux.com, Flexbox.io and Command Line Power User, a set of free video series. 145,000 people have taken at least one of Wes' free video courses.

+
+
+
+
+
+
+ +
+

Almost anyone who is looking to get stronger with JavaScript. We start beginner and go pretty deep.

+

Advanced devs will be able to go through the course a little more quickly, but I've designed this course to be as approachable as possible.

+

This course is done in ES6 so there is liberal use of arrow functions, template strings, destructuring and other ES6 goodies. If you haven't done much ES6, you'll have an easier time with the course if you do at least the first half of my ES6 for Everyone course first.

+

This isn't JavaScript 101. You should have knowledge of how functions, variables, Objects, Arrays and other fundamentals work.

+
    +
  • Front End Devs looking to go Full Stack
  • +
  • Existing server side developers looking to understand the Node stack.
  • +
  • WordPress developers looking for a little more control
  • +
  • JS developers looking to advance their career.
  • +
  • Existing Node.js developers looking to fill in any gaps and update to modern workflows and design patterns.
  • +
  • Anyone who wants to get better at JavaScript and learns well from seeing both fundamental and advanced concepts in practice.
  • +
+
+

Not sure if it's for you?

+

100% money back if you don't think it's a good fit 💰

+
+
+
+
+ +
+

In each video, we learn and implement a concept into our application. Many of the concepts we return to over and over for different parts of the application, hitting and solving new problems each time.

+

This course covers a ton. Along with general JavaScript best practices, here are some of the things you can expect to learn:

+
    +
  • User Accounts
  • +
  • REST API endpoints
  • +
  • Database Schemas
  • +
  • Document Relationships
  • +
  • Templating and Mixins
  • +
  • Restricting Operations
  • +
  • Middleware
  • +
  • Image Resizing
  • +
  • Password Reset Flow
  • +
  • Storing Geospatial Data
  • +
  • Routing
  • +
  • Flow Control
  • +
  • Error Handling
  • +
  • Advanced DB Queries
  • +
  • File Uploading
  • +
  • Sending Email
  • +
  • MVC Pattern
  • +
  • Geocoding Addresses
  • +
  • Pagination
  • +
  • Server Deployment
  • +
+
+
+
+

👌 Video pacing is just right
fast enough to keep you interested without passing by or assuming any details.

+

Seriously, it won't put you to sleep - I'm even funny sometimes.

+
+
+
+
+
FOO
+
BAR
+
BAZ
+
+

foo? bar? baz? wtf?

+

This series was made for humans.
Our application build covers real world scenarios and solves real world problems.
100% free of jargon and metasyntactic variables. Foo Yea!

+
+
+
+
+
+
+
15Modules
+
44Videos
+
9.5Hours
+
+

...and 145,523 words provided with the videos captions
Incremental, Referenceable and Easy To Digest

+
+ +
+

Each video breaks down a specific part of building a Node application and allows for quick referencing in the future. You can binge watch them all today or do a few each day during your lunch or on your commute.

+

Closed Captioning and Transcripts are provided for every video

+
+
    +

    Module #1
    Introduction and Setup

    +
  1. +

    Getting Setup

    05:17#1 +
  2. +
  3. +

    Setting up Mongo DB

    09:30#2 +
  4. +
  5. +

    Starter Files and Environmental Variables

    07:35#3 +
  6. +

    Module #2
    Core Concepts

    +
  7. +

    Core Concept - Routing

    10:19#4 +
  8. +
  9. +

    Core Concept - Templating

    16:48#5 +
  10. +
  11. +

    Core Concept - Template Helpers

    06:24#6 +
  12. +
  13. +

    Core Concept - Controllers and the MVC Pattern

    06:49#7 +
  14. +
  15. +

    Core Concept - Middleware and Error Handling

    14:13#8 +
  16. +

    Module #3
    Database Storage

    +
  17. +

    Creating our Store Model

    12:02#9 +
  18. +
  19. +

    Saving Stores and using Mixins

    17:22#10 +
  20. +

    Module #4
    Control Flow

    +
  21. +

    Using Async Await

    17:31#11 +
  22. +
  23. +

    Flash Messages

    10:51#12 +
  24. +
  25. +

    Querying our Database for Stores

    12:37#13 +
  26. +
  27. +

    Creating an Editing Flow for Stores

    20:43#14 +
  28. +

    Module #5
    Geolocation

    +
  29. +

    Saving Lat and Lng for each store

    14:59#15 +
  30. +
  31. +

    Geocoding Data with Google Maps

    10:28#16 +
  32. +
  33. +

    Quick Data Visualization Tip

    04:35#17 +
  34. +

    Module #6
    File Handing and Image Resizing

    +
  35. +

    Uploading and Resizing Images with Middleware

    23:28#18 +
  36. +
  37. +

    Routing and Templating Single Stores

    13:30#19 +
  38. +

    Module #7
    Custom Queries and Hooks

    +
  39. +

    Using Pre-Save hooks to make Unique Slugs

    06:35#20 +
  40. +
  41. +

    Custom MongoDB Aggregations

    17:52#21 +
  42. +
  43. +

    Multiple Query Promises with Async:Await

    07:14#22 +
  44. +

    Module #8
    User Accounts and Authentication

    +
  45. +

    Creating User Accounts

    30:07#23 +
  46. +
  47. +

    Saving Registered Users to the Database

    16:39#24 +
  48. +
  49. +

    Virtual Fields, Login:Logout middleware and Protecting Routes

    10:26#25 +
  50. +
  51. +

    Creating a User Account Edit Screen

    08:04#26 +
  52. +
  53. +

    Password Reset Flow

    24:59#27 +
  54. +

    Module #9
    Advanced - Email and Permissions

    +
  55. +

    Sending email with Nodejs

    19:07#28 +
  56. +
  57. +

    Locking down our application with User Permissions

    14:05#29 +
  58. +

    Module #10
    Ajax REST API 1

    +
  59. +

    Loading Sample Data

    05:10#30 +
  60. +
  61. +

    JSON endpoints and creating MongoDB Indexes

    15:12#31 +
  62. +
  63. +

    Creating an Ajax Search Interface

    26:17#32 +
  64. +

    Module #11
    Ajax REST API 2

    +
  65. +

    Creating a Geospatial Ajax Endpoint

    09:28#33 +
  66. +
  67. +

    Plotting Stores on a Custom Google Map

    26:34#34 +
  68. +

    Module #12
    Ajax REST API 3

    +
  69. +

    Pushing User Data to our API

    23:22#35 +
  70. +
  71. +

    Displaying our Hearted Stores

    03:46#36 +
  72. +

    Module #13
    Advanced Relationships + Aggregations

    +
  73. +

    Adding a Reviews Data Model

    14:09#37 +
  74. +
  75. +

    Advanced Relationship Population - Displaying Our Reviews

    14:13#38 +
  76. +
  77. +

    Advanced Aggregation

    22:51#39 +
  78. +

    Module #14
    Pagination

    +
  79. +

    Implementing Pagination

    12:34#40 +
  80. +

    Module #15
    Deployment

    +
  81. +

    Deployment Setup

    06:24#41 +
  82. +
  83. +

    Deploying to Now

    05:33#42 +
  84. +
  85. +

    Deploying to Heroku

    05:02#43 +
  86. +
  87. +

    Deploying to Digital Ocean Linux

    12:30#44 +
  88. +
+
+
+
+
+
+ +
+
+
+
+

Starter Course

+
    +
  • Access to the first 7 Modules / 22 HD Video Tutorials
  • +
  • Stream course from any device
  • +
  • Source Code — Completed Examples and Exercises
  • +
  • $25 Digital Ocean hosting credit (valid for new Customers only)
  • +
  • Unlimited Updates + Never Expires
  • +
  • SlackExclusive access to the Learn Node Slack Chat Room where you can ask for help and chat with other learners
  • +
+
+
Get the Starter Course + $89$82 +

Join 15,744 Others!

+

+
+
+
+
+

Master Package

+
    +
  • Access to all 15 Modules / 44 HD Videos — see above for a full listing of topics
  • +
  • Stream and Download DRM-free files from any device
  • +
  • Includes Authentication, Sending Email and AJAX REST API
  • +
  • Deployment, Pagination, Advanced Aggregations and more
  • +
  • All Source Code — Completed Examples, Exercises, Webpack files and npm scripts
  • +
  • $25 Digital Ocean hosting credit (valid for new Customers only)
  • +
  • Unlimited Updates + Never Expires
  • +
  • SlackExclusive access to the Learn Node Slack Chat Room where you can ask for help and chat with other learners
  • +
+
+
Get The Master Package + $139$97 +

15,744 already sold!

+

+
+
+
+
+
+

Team License Packages

+

With a team license you can buy a number of spots to allocate to employees. The spots do not expire and you can fill the spots via the dashboard whenever you like.

+

Perfect for on-boarding new hires, interns and contractors to your tech stack.

+

The team license includes everything in the Master Package above.

+
+

For larger groups or for on-site training contact me for more info!

+
+
+ +
+

All prices in USD

+
+
+
+
+
+

Worth It?

Need to convince your Boss?

+
+
+

I've put over 5 months of my time into this course gaining a deep understanding of each feature, creating examples and distilling it into something that is easy for you to consume.

+

This course should take you between 10 and 20 hours to complete versus anywhere from 100 to 400 hours to learn on your own, scraping together resources.

+

I've spent the time so you don't have to. Spend a few hours learning Node with me and you'll save yourself hours of Googling and Stack Overflowing in the future. I'll mama bird you the stuff you need to know to keep building the things you make.

+

As with all my courses, there is 100% money back if it's not a good fit. Give it a shot!

+
+
+
+
+
+ +
+
+
+

What is the front end written in?

+

The interactive parts — search, map, hearting stores and geocoding — are done with Vanilla ES6 JS modules, Axios (AJAX Library) and the Google Maps API. You could totally do these parts of the application in React, Vue, Angular or any other framework that you prefer though it might be a bit overkill.

+

The provided styles are written in Sass.

+

The templating is written in Pug (formerly named Jade).

+

All the front end dependencies are compiled with Webpack.

+
+
+

Do these videos expire? Is this a monthly cost?

+

Nope and nope. Pay once, have them forever.

+
+
+

Do you provide a certificate of completion?

+

Sure do! In your course dashboard you can download an official certificate of completion that can be used to be reimbursed by your employer or land that dream job you are applying for.

+
+
+

Can I put the code on GitHub? Can I put this app in my portfolio?

+

Absolutely - I just ask that you don't create your own course teaching Node.js with this app. Otherwise, go nuts! You can credit my course if you like, but it's not required as you might be using this app to get a job :)

+
+
+

What other courses do you have?

+

I've got a bunch of free and premium web development courses — check out the whole listing over at wesbos.com/courses.

+
+
+

I lost, deleted or never got my welcome email! What do I do?

+

Log into your account, or create an account with the same email you used to buy the series.

+
+
+

What format are the videos? How do I watch them?

+

Once you buy a package, you will be mailed access to your account dashboard where you can stream all the videos. Buyers of the Master Package will also be able to download them for off-line viewing. All the videos were professionally recorded at 1920×1080 with top quality audio — no pops, echoes, chair squeaks, breathing or gross mouth sounds here!

+
+
+

What if I'm not thrilled?

+

I want to make sure you get real value out of this so I only want your money if you are happy with the product! If you aren't satisfied, please send an email to [email protected] with a copy of your welcome email and I will refund you.

+
+
+

Do you offer a student discount?

+

Absolutely! Fill out this form with some proof that you are a student and I'll send you a discount code. This applies to anyone in any type of schooling, including evening classes and coding bootcamps!

+

If you have received a student discount for a previous product, it's the same code 😉.

+
+
+

What theme do you use? What is that font? What terminal do you use? What kind of bread do you buy?

+

I've detailed my entire setup over here, feel free to tweet me with any more questions you have about setting things up.

+
+
+

I have another question!

+

Sure - email me at [email protected] or give me a call 416-833-3641!

+
+
+
+ +
+ + +
+
× +
+

React For Beginners

+

Package

+
+
+ + +
+
+
+
+ +
+
+ + + + + +
+
+ + + + + +
+
+ +
+
+ +
+ +

Secure SSL Payment via Stripe

+
+
+
+ + +
+
+ +
+
+ +
+

Secure SSL Payment via PayPal

+
+
+
+
+
+ + + + + + + + + diff --git a/packages/metascraper-readability/test/fixtures/microlink.io.html b/packages/metascraper-readability/test/fixtures/microlink.io.html new file mode 100644 index 000000000..2139535a2 --- /dev/null +++ b/packages/metascraper-readability/test/fixtures/microlink.io.html @@ -0,0 +1,181 @@ +Turns any website into data | Microlink

Extract structured data from any website

import MicrolinkCard from '@microlink/react'
+
+    <MicrolinkCard
+      url='https://twitter.com/futurism/status/882987478541533189'
+      size='large'
+      video
+    />

Try another link →

Browser as service

Microlink is a powerful API for developers with top notch tecnologies of the industry.

  • Headless browser service.

  • Simple API integration.

  • Add it to your existing stack or cloud.

Features

Our feature at a glance.

  • Data on Demand

    Consume as much data as you need, customizing each payload.

  • Developer First

    Our developer friendly API makes it easy to get started.

  • Fully Programable

    Easily integrate it with any existing stack or cloud.

  • High Availability

    Cloud based solution with superior operational performance.

  • Cache Layer

    Query caching police for improving response timing.

  • Scale on Time

    Based on API call quotas, use just you need.

  • Screenshot Support

    Full or partial website snapshots with device support.

  • Data Normalization

    Type check rules for ensuring data extracted is correct.

  • Color Detection

    Predominant palette for each image detected.

  • Adblock Builtin

    Faster reponsive timing avoid unnecessary requests.

  • Video Support

    Detect the original streaming video source.

  • Embed Mode

    Incrust content directly in your HTML markup

  • Data on Demand

    Consume as much data as you need, customizing each payload.

  • Developer First

    Our developer friendly API makes it easy to get started.

  • Fully Programable

    Easily integrate it with any existing stack or cloud.

  • High Availability

    Cloud based solution with superior operational performance.

  • Cache Layer

    Query caching police for improving response timing.

  • Scale on Time

    Based on API call quotas, use just you need.

  • Screenshot Support

    Full or partial website snapshots with device support.

  • Data Normalization

    Type check rules for ensuring data extracted is correct.

  • Color Detection

    Predominant palette for each image detected.

  • Adblock Builtin

    Faster reponsive timing avoid unnecessary requests.

  • Video Support

    Detect the original streaming video source.

  • Embed Mode

    Incrust content directly in your HTML markup

  • Turns any website into data

    Microlink turns any website into useful structured data.

    • Works with any website.

    • Detect complementary information (video, colors, dimensions, etc).

    • Customize payload using Custom Rules.

    Pricing

    Pay as you go.

    FreePro

    Rate Limit

    Rate Limit

    250

    Request Concurrency

    Request Concurrency

    1

    Request Caching

    Request Caching

    Screenshot support

    Screenshot support

    Video detection

    Video detection

    Color Detection

    Color Detection

    Live Support

    Live Support

    0

    12

    Do you need more?

    diff --git a/packages/metascraper-readability/test/index.js b/packages/metascraper-readability/test/index.js new file mode 100644 index 000000000..27973ab34 --- /dev/null +++ b/packages/metascraper-readability/test/index.js @@ -0,0 +1,33 @@ +'use strict' + +const snapshot = require('snap-shot') +const path = require('path') +const fs = require('fs') + +const metascraper = require('metascraper')([require('metascraper-readability')()]) + +describe('metascraper-readability', () => { + describe('.readability', function () { + it('learnnode.com', async () => { + const url = 'https://learnnode.com' + const html = fs.readFileSync(path.resolve(__dirname, 'fixtures/learnnode.com.html'), 'utf-8') + + const metadata = await metascraper({ html, url }) + snapshot(metadata) + }) + + it('microlink.io', async () => { + const url = 'https://microlink.io' + const html = fs.readFileSync(path.resolve(__dirname, 'fixtures/microlink.io.html'), 'utf-8') + const metadata = await metascraper({ html, url }) + snapshot(metadata) + }) + + it('kikobeats.com', async () => { + const url = 'https://kikobeats.com' + const html = fs.readFileSync(path.resolve(__dirname, 'fixtures/kikobeats.com.html'), 'utf-8') + const metadata = await metascraper({ html, url }) + snapshot(metadata) + }) + }) +}) diff --git a/packages/metascraper-readability/test/mocha.opts b/packages/metascraper-readability/test/mocha.opts new file mode 100644 index 000000000..7dae578bb --- /dev/null +++ b/packages/metascraper-readability/test/mocha.opts @@ -0,0 +1,5 @@ +--reporter spec +--timeout 120000 +--slow 300 +--bail +--recursive diff --git a/packages/metascraper/__snapshots__/index.js.snap-shot b/packages/metascraper/__snapshots__/index.js.snap-shot index 38edc0e05..dd1434ecc 100644 --- a/packages/metascraper/__snapshots__/index.js.snap-shot +++ b/packages/metascraper/__snapshots__/index.js.snap-shot @@ -404,7 +404,7 @@ exports['lean-data 1'] = { exports['linkedin (company) 1'] = { "author": null, "date": null, - "description": null, + "description": "LinkedIn领英", "image": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "video": null, "lang": "en", diff --git a/packages/metascraper/test/integration/anandtech/index.js b/packages/metascraper/test/integration/anandtech/index.js index c5605900b..675731141 100644 --- a/packages/metascraper/test/integration/anandtech/index.js +++ b/packages/metascraper/test/integration/anandtech/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/arstechnica/index.js b/packages/metascraper/test/integration/arstechnica/index.js index 231812e25..991dce321 100644 --- a/packages/metascraper/test/integration/arstechnica/index.js +++ b/packages/metascraper/test/integration/arstechnica/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/astier/index.js b/packages/metascraper/test/integration/astier/index.js index 1899f2060..ac078d8f2 100644 --- a/packages/metascraper/test/integration/astier/index.js +++ b/packages/metascraper/test/integration/astier/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/atlasobscura/index.js b/packages/metascraper/test/integration/atlasobscura/index.js index 9e15425f1..ac7f25b80 100644 --- a/packages/metascraper/test/integration/atlasobscura/index.js +++ b/packages/metascraper/test/integration/atlasobscura/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/audiense/index.js b/packages/metascraper/test/integration/audiense/index.js index 2fc76814f..4fceec2fe 100644 --- a/packages/metascraper/test/integration/audiense/index.js +++ b/packages/metascraper/test/integration/audiense/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/bbc/index.js b/packages/metascraper/test/integration/bbc/index.js index 26adf55a8..a25834ae5 100644 --- a/packages/metascraper/test/integration/bbc/index.js +++ b/packages/metascraper/test/integration/bbc/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/bloomberg/index.js b/packages/metascraper/test/integration/bloomberg/index.js index 45c445243..8e263a096 100644 --- a/packages/metascraper/test/integration/bloomberg/index.js +++ b/packages/metascraper/test/integration/bloomberg/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/business-today/index.js b/packages/metascraper/test/integration/business-today/index.js index 009349703..ea4911193 100644 --- a/packages/metascraper/test/integration/business-today/index.js +++ b/packages/metascraper/test/integration/business-today/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/cbr/index.js b/packages/metascraper/test/integration/cbr/index.js index 0eb63db2e..9352c6f2f 100644 --- a/packages/metascraper/test/integration/cbr/index.js +++ b/packages/metascraper/test/integration/cbr/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/cio/index.js b/packages/metascraper/test/integration/cio/index.js index cf73691db..79a47ac9e 100644 --- a/packages/metascraper/test/integration/cio/index.js +++ b/packages/metascraper/test/integration/cio/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/cloud-pro/index.js b/packages/metascraper/test/integration/cloud-pro/index.js index 4388d6fcf..434d03e2f 100644 --- a/packages/metascraper/test/integration/cloud-pro/index.js +++ b/packages/metascraper/test/integration/cloud-pro/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/cnet/index.js b/packages/metascraper/test/integration/cnet/index.js index 52b1f023b..a37c23a30 100644 --- a/packages/metascraper/test/integration/cnet/index.js +++ b/packages/metascraper/test/integration/cnet/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/computerworld/index.js b/packages/metascraper/test/integration/computerworld/index.js index 876c2e575..d73125933 100644 --- a/packages/metascraper/test/integration/computerworld/index.js +++ b/packages/metascraper/test/integration/computerworld/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/crn/index.js b/packages/metascraper/test/integration/crn/index.js index aef2ca9d6..2fc5c7a83 100644 --- a/packages/metascraper/test/integration/crn/index.js +++ b/packages/metascraper/test/integration/crn/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/economic-times/index.js b/packages/metascraper/test/integration/economic-times/index.js index 98d26fd70..0b623f925 100644 --- a/packages/metascraper/test/integration/economic-times/index.js +++ b/packages/metascraper/test/integration/economic-times/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/engadget/index.js b/packages/metascraper/test/integration/engadget/index.js index 3e59d2f98..bd87a44b4 100644 --- a/packages/metascraper/test/integration/engadget/index.js +++ b/packages/metascraper/test/integration/engadget/index.js @@ -17,13 +17,13 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) -const url = - 'https://www.engadget.com/2019/01/07/all-github-users-keep-code-private' +const url = 'https://www.engadget.com/2019/01/07/all-github-users-keep-code-private' it('engadget', async () => { const html = await readFile(resolve(__dirname, 'input.html')) diff --git a/packages/metascraper/test/integration/entrepreneur/index.js b/packages/metascraper/test/integration/entrepreneur/index.js index bc02240d0..595fa13ce 100644 --- a/packages/metascraper/test/integration/entrepreneur/index.js +++ b/packages/metascraper/test/integration/entrepreneur/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/eweek/index.js b/packages/metascraper/test/integration/eweek/index.js index 7c5e35639..b6d193899 100644 --- a/packages/metascraper/test/integration/eweek/index.js +++ b/packages/metascraper/test/integration/eweek/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/fast-company/index.js b/packages/metascraper/test/integration/fast-company/index.js index 885c06fcc..22473ca8e 100644 --- a/packages/metascraper/test/integration/fast-company/index.js +++ b/packages/metascraper/test/integration/fast-company/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/fastersite/index.js b/packages/metascraper/test/integration/fastersite/index.js index 86d2cd8d7..fd6856fac 100644 --- a/packages/metascraper/test/integration/fastersite/index.js +++ b/packages/metascraper/test/integration/fastersite/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/fierce-devops/index.js b/packages/metascraper/test/integration/fierce-devops/index.js index bd5b02203..fe2bf69e3 100644 --- a/packages/metascraper/test/integration/fierce-devops/index.js +++ b/packages/metascraper/test/integration/fierce-devops/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/financial-times/index.js b/packages/metascraper/test/integration/financial-times/index.js index 970a75f7d..8e0d33191 100644 --- a/packages/metascraper/test/integration/financial-times/index.js +++ b/packages/metascraper/test/integration/financial-times/index.js @@ -17,13 +17,13 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) -const url = - 'http://www.ft.com/cms/s/2/796d1220-475e-11e5-af2f-4d6e0e5eda22.html' +const url = 'http://www.ft.com/cms/s/2/796d1220-475e-11e5-af2f-4d6e0e5eda22.html' it('financial-times', async () => { const html = await readFile(resolve(__dirname, 'input.html')) diff --git a/packages/metascraper/test/integration/forbes/index.js b/packages/metascraper/test/integration/forbes/index.js index 40567e959..38d678bc0 100644 --- a/packages/metascraper/test/integration/forbes/index.js +++ b/packages/metascraper/test/integration/forbes/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/fortune/index.js b/packages/metascraper/test/integration/fortune/index.js index 3a6ac5d18..b360420de 100644 --- a/packages/metascraper/test/integration/fortune/index.js +++ b/packages/metascraper/test/integration/fortune/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/geek-time/index.js b/packages/metascraper/test/integration/geek-time/index.js index 86bf8b68f..8216cb99a 100644 --- a/packages/metascraper/test/integration/geek-time/index.js +++ b/packages/metascraper/test/integration/geek-time/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/github/index.js b/packages/metascraper/test/integration/github/index.js index 76e14092f..522232ddc 100644 --- a/packages/metascraper/test/integration/github/index.js +++ b/packages/metascraper/test/integration/github/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/googleblog/index.js b/packages/metascraper/test/integration/googleblog/index.js index 07c7f101f..2bba30fdc 100644 --- a/packages/metascraper/test/integration/googleblog/index.js +++ b/packages/metascraper/test/integration/googleblog/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/inc/index.js b/packages/metascraper/test/integration/inc/index.js index 002a97b18..ce7ea4ba1 100644 --- a/packages/metascraper/test/integration/inc/index.js +++ b/packages/metascraper/test/integration/inc/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/information-week/index.js b/packages/metascraper/test/integration/information-week/index.js index e28d2cb67..af394e29e 100644 --- a/packages/metascraper/test/integration/information-week/index.js +++ b/packages/metascraper/test/integration/information-week/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/instagram/index.js b/packages/metascraper/test/integration/instagram/index.js index 051ff96de..230c66cfd 100644 --- a/packages/metascraper/test/integration/instagram/index.js +++ b/packages/metascraper/test/integration/instagram/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/jewish-business-news/index.js b/packages/metascraper/test/integration/jewish-business-news/index.js index 3fd377620..a4a4b3c40 100644 --- a/packages/metascraper/test/integration/jewish-business-news/index.js +++ b/packages/metascraper/test/integration/jewish-business-news/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/lean-data/index.js b/packages/metascraper/test/integration/lean-data/index.js index 10d021b6f..f5c298c49 100644 --- a/packages/metascraper/test/integration/lean-data/index.js +++ b/packages/metascraper/test/integration/lean-data/index.js @@ -17,13 +17,13 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) -const url = - 'http://www.leandatainc.com/account-based-sales-marketing/the-winds-of-change' +const url = 'http://www.leandatainc.com/account-based-sales-marketing/the-winds-of-change' it('lean-data', async () => { const html = await readFile(resolve(__dirname, 'input.html')) diff --git a/packages/metascraper/test/integration/learnnode/index.js b/packages/metascraper/test/integration/learnnode/index.js index 8030abe4a..3987913c3 100644 --- a/packages/metascraper/test/integration/learnnode/index.js +++ b/packages/metascraper/test/integration/learnnode/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/linkedin-company/index.js b/packages/metascraper/test/integration/linkedin-company/index.js index b86f983c0..147f39633 100644 --- a/packages/metascraper/test/integration/linkedin-company/index.js +++ b/packages/metascraper/test/integration/linkedin-company/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/linkedin-pulse/index.js b/packages/metascraper/test/integration/linkedin-pulse/index.js index 072ac1c9f..cac532999 100644 --- a/packages/metascraper/test/integration/linkedin-pulse/index.js +++ b/packages/metascraper/test/integration/linkedin-pulse/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/los-angeles-times/index.js b/packages/metascraper/test/integration/los-angeles-times/index.js index 9f9cea456..60d6e223d 100644 --- a/packages/metascraper/test/integration/los-angeles-times/index.js +++ b/packages/metascraper/test/integration/los-angeles-times/index.js @@ -17,13 +17,13 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) -const url = - 'http://www.latimes.com/business/technology/la-fi-tn-tech-downturn-20160429-story.html' +const url = 'http://www.latimes.com/business/technology/la-fi-tn-tech-downturn-20160429-story.html' it('los-angeles-times', async () => { const html = await readFile(resolve(__dirname, 'input.html')) diff --git a/packages/metascraper/test/integration/mac-rumors/index.js b/packages/metascraper/test/integration/mac-rumors/index.js index b0aadee24..6c58f871c 100644 --- a/packages/metascraper/test/integration/mac-rumors/index.js +++ b/packages/metascraper/test/integration/mac-rumors/index.js @@ -17,13 +17,13 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) -const url = - 'http://www.macrumors.com/2016/05/24/apple-siri-sdk-amazon-echo-competitor' +const url = 'http://www.macrumors.com/2016/05/24/apple-siri-sdk-amazon-echo-competitor' it('mac-rumors', async () => { const html = await readFile(resolve(__dirname, 'input.html')) diff --git a/packages/metascraper/test/integration/market-wired/index.js b/packages/metascraper/test/integration/market-wired/index.js index 38c8b6b26..98f7ea67d 100644 --- a/packages/metascraper/test/integration/market-wired/index.js +++ b/packages/metascraper/test/integration/market-wired/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/marketing-land/index.js b/packages/metascraper/test/integration/marketing-land/index.js index 0ff484c92..3d06f7689 100644 --- a/packages/metascraper/test/integration/marketing-land/index.js +++ b/packages/metascraper/test/integration/marketing-land/index.js @@ -17,13 +17,13 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) -const url = - 'http://marketingland.com/an-inside-look-at-the-upcoming-martech-conference-165927' +const url = 'http://marketingland.com/an-inside-look-at-the-upcoming-martech-conference-165927' it('marketing-land', async () => { const html = await readFile(resolve(__dirname, 'input.html')) diff --git a/packages/metascraper/test/integration/mashable/index.js b/packages/metascraper/test/integration/mashable/index.js index ea0e9ef66..ab01a7635 100644 --- a/packages/metascraper/test/integration/mashable/index.js +++ b/packages/metascraper/test/integration/mashable/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/medium/index.js b/packages/metascraper/test/integration/medium/index.js index a1df5e317..ffc2aa587 100644 --- a/packages/metascraper/test/integration/medium/index.js +++ b/packages/metascraper/test/integration/medium/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/motherboard/index.js b/packages/metascraper/test/integration/motherboard/index.js index da6c0d946..ded35decf 100644 --- a/packages/metascraper/test/integration/motherboard/index.js +++ b/packages/metascraper/test/integration/motherboard/index.js @@ -17,13 +17,13 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) -const url = - 'http://motherboard.vice.com/read/google-wins-trial-against-oracle-saves-9-billion' +const url = 'http://motherboard.vice.com/read/google-wins-trial-against-oracle-saves-9-billion' it('motherboard', async () => { const html = await readFile(resolve(__dirname, 'input.html')) diff --git a/packages/metascraper/test/integration/nytimes/index.js b/packages/metascraper/test/integration/nytimes/index.js index c9103fbba..a97b04da2 100644 --- a/packages/metascraper/test/integration/nytimes/index.js +++ b/packages/metascraper/test/integration/nytimes/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/pikabu/index.js b/packages/metascraper/test/integration/pikabu/index.js index c063e075b..41f4a041e 100644 --- a/packages/metascraper/test/integration/pikabu/index.js +++ b/packages/metascraper/test/integration/pikabu/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/pr-newswire/index.js b/packages/metascraper/test/integration/pr-newswire/index.js index 1d0410bf0..94987b28f 100644 --- a/packages/metascraper/test/integration/pr-newswire/index.js +++ b/packages/metascraper/test/integration/pr-newswire/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/recode/index.js b/packages/metascraper/test/integration/recode/index.js index 1070267b6..3f7611d72 100644 --- a/packages/metascraper/test/integration/recode/index.js +++ b/packages/metascraper/test/integration/recode/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/reuters/index.js b/packages/metascraper/test/integration/reuters/index.js index 491539944..c112c18e3 100644 --- a/packages/metascraper/test/integration/reuters/index.js +++ b/packages/metascraper/test/integration/reuters/index.js @@ -17,13 +17,13 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) -const url = - 'http://www.reuters.com/article/us-venture-fundraising-idUSKCN0Y41DQ' +const url = 'http://www.reuters.com/article/us-venture-fundraising-idUSKCN0Y41DQ' it('reuters', async () => { const html = await readFile(resolve(__dirname, 'input.html')) diff --git a/packages/metascraper/test/integration/san-francisco-chronicle/index.js b/packages/metascraper/test/integration/san-francisco-chronicle/index.js index 231f6bf99..9ab1b7245 100644 --- a/packages/metascraper/test/integration/san-francisco-chronicle/index.js +++ b/packages/metascraper/test/integration/san-francisco-chronicle/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/segment-academy/index.js b/packages/metascraper/test/integration/segment-academy/index.js index 8acf174f6..72d83bc8e 100644 --- a/packages/metascraper/test/integration/segment-academy/index.js +++ b/packages/metascraper/test/integration/segment-academy/index.js @@ -17,13 +17,13 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) -const url = - 'https://www.segment.com/academy/collecting-data/when-to-track-on-the-client-vs-server' +const url = 'https://www.segment.com/academy/collecting-data/when-to-track-on-the-client-vs-server' it('segment-academy', async () => { const html = await readFile(resolve(__dirname, 'input.html')) diff --git a/packages/metascraper/test/integration/segment/index.js b/packages/metascraper/test/integration/segment/index.js index dd02c8a77..57cd8bebe 100644 --- a/packages/metascraper/test/integration/segment/index.js +++ b/packages/metascraper/test/integration/segment/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/silicon-angle/index.js b/packages/metascraper/test/integration/silicon-angle/index.js index 125d5c600..ff124ce93 100644 --- a/packages/metascraper/test/integration/silicon-angle/index.js +++ b/packages/metascraper/test/integration/silicon-angle/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/silicon-beat/index.js b/packages/metascraper/test/integration/silicon-beat/index.js index eebcbd432..772828f85 100644 --- a/packages/metascraper/test/integration/silicon-beat/index.js +++ b/packages/metascraper/test/integration/silicon-beat/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/silicon-tap/index.js b/packages/metascraper/test/integration/silicon-tap/index.js index c560d373f..b2fe27041 100644 --- a/packages/metascraper/test/integration/silicon-tap/index.js +++ b/packages/metascraper/test/integration/silicon-tap/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/silicon-valley-business-journal/index.js b/packages/metascraper/test/integration/silicon-valley-business-journal/index.js index 1957755d5..1e6ce58c3 100644 --- a/packages/metascraper/test/integration/silicon-valley-business-journal/index.js +++ b/packages/metascraper/test/integration/silicon-valley-business-journal/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/smitten-kitchen/index.js b/packages/metascraper/test/integration/smitten-kitchen/index.js index aae8a95b8..e71477704 100644 --- a/packages/metascraper/test/integration/smitten-kitchen/index.js +++ b/packages/metascraper/test/integration/smitten-kitchen/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/startup-grind/index.js b/packages/metascraper/test/integration/startup-grind/index.js index d4256c1d8..aa52abe5b 100644 --- a/packages/metascraper/test/integration/startup-grind/index.js +++ b/packages/metascraper/test/integration/startup-grind/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/techcrunch/index.js b/packages/metascraper/test/integration/techcrunch/index.js index bbd786f2b..d29f67c18 100644 --- a/packages/metascraper/test/integration/techcrunch/index.js +++ b/packages/metascraper/test/integration/techcrunch/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/the-boston-globe/index.js b/packages/metascraper/test/integration/the-boston-globe/index.js index ac0fe9789..311ece391 100644 --- a/packages/metascraper/test/integration/the-boston-globe/index.js +++ b/packages/metascraper/test/integration/the-boston-globe/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/the-guardian/index.js b/packages/metascraper/test/integration/the-guardian/index.js index fa916158d..ceb99520e 100644 --- a/packages/metascraper/test/integration/the-guardian/index.js +++ b/packages/metascraper/test/integration/the-guardian/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/the-register/index.js b/packages/metascraper/test/integration/the-register/index.js index 057b7e980..a767f4fc8 100644 --- a/packages/metascraper/test/integration/the-register/index.js +++ b/packages/metascraper/test/integration/the-register/index.js @@ -17,13 +17,13 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) -const url = - 'http://www.theregister.co.uk/2016/05/03/emc_world_virtustream_announcement' +const url = 'http://www.theregister.co.uk/2016/05/03/emc_world_virtustream_announcement' it('the-register', async () => { const html = await readFile(resolve(__dirname, 'input.html')) diff --git a/packages/metascraper/test/integration/the-verge/index.js b/packages/metascraper/test/integration/the-verge/index.js index 8a899e1c3..df9332154 100644 --- a/packages/metascraper/test/integration/the-verge/index.js +++ b/packages/metascraper/test/integration/the-verge/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/twitter-gif/index.js b/packages/metascraper/test/integration/twitter-gif/index.js index 0926a0f33..65c6014ee 100644 --- a/packages/metascraper/test/integration/twitter-gif/index.js +++ b/packages/metascraper/test/integration/twitter-gif/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/twitter-image/index.js b/packages/metascraper/test/integration/twitter-image/index.js index 1eee1a41c..871049959 100644 --- a/packages/metascraper/test/integration/twitter-image/index.js +++ b/packages/metascraper/test/integration/twitter-image/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/usa-today/index.js b/packages/metascraper/test/integration/usa-today/index.js index 6733b6c2d..8b2666da3 100644 --- a/packages/metascraper/test/integration/usa-today/index.js +++ b/packages/metascraper/test/integration/usa-today/index.js @@ -17,13 +17,13 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) -const url = - 'http://www.usatoday.com/story/news/2016/05/24/evening-news-roundup-tuesday/84846520' +const url = 'http://www.usatoday.com/story/news/2016/05/24/evening-news-roundup-tuesday/84846520' it('usa-today', async () => { const html = await readFile(resolve(__dirname, 'input.html')) diff --git a/packages/metascraper/test/integration/venture-beat/index.js b/packages/metascraper/test/integration/venture-beat/index.js index 74e6a2390..313e82751 100644 --- a/packages/metascraper/test/integration/venture-beat/index.js +++ b/packages/metascraper/test/integration/venture-beat/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/vimeo/index.js b/packages/metascraper/test/integration/vimeo/index.js index 0dea33459..bbefc1a39 100644 --- a/packages/metascraper/test/integration/vimeo/index.js +++ b/packages/metascraper/test/integration/vimeo/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/washington-post/index.js b/packages/metascraper/test/integration/washington-post/index.js index 14246a482..bd72cdebb 100644 --- a/packages/metascraper/test/integration/washington-post/index.js +++ b/packages/metascraper/test/integration/washington-post/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/wikipedia/index.js b/packages/metascraper/test/integration/wikipedia/index.js index 765f4c300..c97026465 100644 --- a/packages/metascraper/test/integration/wikipedia/index.js +++ b/packages/metascraper/test/integration/wikipedia/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/wired/index.js b/packages/metascraper/test/integration/wired/index.js index dd33849f1..410468cfa 100644 --- a/packages/metascraper/test/integration/wired/index.js +++ b/packages/metascraper/test/integration/wired/index.js @@ -17,13 +17,13 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) -const url = - 'https://www.wired.com/story/giant-antarctic-icebergs-and-crushing-existential-dread' +const url = 'https://www.wired.com/story/giant-antarctic-icebergs-and-crushing-existential-dread' it('wired', async () => { const html = await readFile(resolve(__dirname, 'input.html')) diff --git a/packages/metascraper/test/integration/wsj/index.js b/packages/metascraper/test/integration/wsj/index.js index b9e053fc4..d3e2c7638 100644 --- a/packages/metascraper/test/integration/wsj/index.js +++ b/packages/metascraper/test/integration/wsj/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/xconomy/index.js b/packages/metascraper/test/integration/xconomy/index.js index 746655976..437ddf251 100644 --- a/packages/metascraper/test/integration/xconomy/index.js +++ b/packages/metascraper/test/integration/xconomy/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) diff --git a/packages/metascraper/test/integration/yahoo-news/index.js b/packages/metascraper/test/integration/yahoo-news/index.js index 800690c10..040264de6 100644 --- a/packages/metascraper/test/integration/yahoo-news/index.js +++ b/packages/metascraper/test/integration/yahoo-news/index.js @@ -17,13 +17,13 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile) -const url = - 'https://www.yahoo.com/news/justin-bieber-coldly-throws-fans-133800208.html' +const url = 'https://www.yahoo.com/news/justin-bieber-coldly-throws-fans-133800208.html' it('yahoo-news', async () => { const html = await readFile(resolve(__dirname, 'input.html')) diff --git a/packages/metascraper/test/integration/zdnet/index.js b/packages/metascraper/test/integration/zdnet/index.js index bf3992533..ce84bbff1 100644 --- a/packages/metascraper/test/integration/zdnet/index.js +++ b/packages/metascraper/test/integration/zdnet/index.js @@ -17,7 +17,8 @@ const metascraper = require('../../..')([ require('metascraper-logo-favicon')(), require('metascraper-publisher')(), require('metascraper-title')(), - require('metascraper-url')() + require('metascraper-url')(), + require('metascraper-readability')() ]) const readFile = promisify(fs.readFile)