initial commit

.editorconfig

@@ -0,0 +1,14 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 2
+indent_style = space
+trim_trailing_whitespace = true
+
+[**.{js,json,md}]
+insert_final_newline = true
+
+[**.html]
+insert_final_newline = false

.eslintrc.json

@@ -0,0 +1,26 @@
+{
+  "extends": "eslint:recommended",
+  "env": {
+    "node": true,
+    "commonjs": true,
+    "es6": true
+  },
+  "rules": {
+    "indent": ["error", 2, { "SwitchCase": 1 }],
+    "linebreak-style": ["error", "unix"],
+    "quotes": ["error", "backtick"],
+    "semi": ["error", "never"],
+    "array-callback-return": "error",
+    "curly": "error",
+    "eqeqeq": "error",
+    "handle-callback-err": "error",
+    "global-require": "error",
+    "no-console": "warn",
+    "no-constant-condition": 0,
+    "no-else-return": "error",
+    "no-prototype-builtins": "error",
+    "no-unsafe-negation": "error",
+    "no-shadow": ["error", { "allow": ["next", "done", "err"] }],
+    "no-shadow-restricted-names": "error"
+  }
+}

.gitignore

@@ -0,0 +1,22 @@
+lib-cov
+*.seed
+*.log
+*.csv
+*.dat
+*.out
+*.pid
+*.gz
+
+var
+pids
+logs
+results
+
+npm-debug.log
+
+lib/settings/dev.js
+app/settings/dev.js
+static/cache
+static/uploads
+.DS_Store
+node_modules/

index.js

@@ -0,0 +1,106 @@
+`use strict`
+
+const url = require(`url`)
+const request = require(`request`)
+const htmlparser = require(`htmlparser2`)
+const fileType = require(`file-type`)
+const rhttp = /^https?:\/\//i
+const whitelist = /charset|author|host|description|twitter:|og:|theme-color/im
+
+function extract (endpoint, options, done) {
+  if (!done) {
+    done = options
+    options = {}
+  }
+  options.url = endpoint
+
+  const payload = { title: `` }
+  const parser = createParser(options.url, payload)
+
+  request(options, gotResponse)
+
+  function gotResponse (err, res, body) {
+    if (err) {
+      done(err)
+      return
+    }
+
+    const file = fileType(body)
+    if (file) {
+      payload.file = file
+      return
+    }
+
+    const rdoublespaces = /\s{2,}|\n/img
+
+    parser.write(body)
+    payload.title = (payload.title || ``).replace(rdoublespaces, ``)
+    payload.images = Array.from(payload.images || [])
+
+    done(null, payload)
+  }
+}
+
+function createParser (base, payload) {
+  let inHead = false
+  let tagName = null
+  const events = { onopentag, ontext, onclosetag }
+  const options = { decodeEntities: true }
+  const parser = new htmlparser.Parser(events, options)
+  return parser
+
+  function onopentag (name, attrs) {
+    tagName = name
+    if (name === `head`) {
+      inHead = true
+    }
+    if (name === `meta`) {
+      const meta = parseMeta(attrs)
+      if (meta) {
+        const [key, value] = meta
+        payload[key] = value
+      }
+    }
+    if (name === `img`) {
+      const src = attrs.src
+      if (src && rhttp.test(src)) {
+        if (!payload.images) {
+          payload.images = new Set()
+        }
+        payload.images.add(url.resolve(base, src))
+      }
+    }
+  }
+
+  function ontext (text) {
+    if (inHead && tagName === `title`) {
+      payload.title += text
+    }
+  }
+
+  function onclosetag (name) {
+    if (name === `head`) {
+      inHead = false
+    }
+  }
+}
+
+function parseMeta (attr) {
+  const name = attr.name || attr.property || Object.keys(attr)[0]
+
+  if (whitelist.test(name)) {
+    return [
+      camelCase(name),
+      attr.content || attr[name]
+    ]
+  }
+}
+
+function camelCase (name) {
+  const rseparator = /(?:\:|_)(\w)/g
+  return name.replace(rseparator,
+    (all, letter) => letter.toUpperCase()
+  )
+}
+
+module.exports = extract

license

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright © 2017 Nicolas Bevacqua
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package.json

@@ -0,0 +1,19 @@
+{
+  "name": "scrape-metadata",
+  "version": "1.0.0",
+  "description": "HTML metadata scraper",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/bevacqua/scrape-metadata.git"
+  },
+  "author": "Nicolás Bevacqua <nico@ponyfoo.com>",
+  "license": "MIT",
+  "scripts": {
+    "test": "node test.js"
+  },
+  "dependencies": {
+    "file-type": "4.1.0",
+    "htmlparser2": "3.9.2",
+    "request": "2.81.0"
+  }
+}

readme.md

@@ -0,0 +1,30 @@
+# `scrape-metadata`
+
+> HTML metadata scraper
+
+# install
+
+```shell
+npm install --save scrape-metadata
+```
+
+# usage
+
+```
+const scrape = require('scrape-metadata')
+scrape('https://google.com', (err, meta) => {
+  console.log(meta)
+})
+```
+
+# api
+
+`scrape(url, options?, done)`
+
+- `url` is the URL of the page you want to scrape metadata from
+- `options` is optional, passed to `request`
+- `done` is a callback with an `(err, meta)` signature
+
+# license
+
+mit

test.js

@@ -0,0 +1,21 @@
+const scrape = require(`./`)
+
+scrape(`http://www.newyorker.com`, (err, meta) =>
+  console.log(err, meta)
+)
+
+scrape(`https://www.youtube.com/watch?v=dQw4w9WgXcQ`, (err, meta) =>
+  console.log(err, meta)
+)
+
+scrape(`http://www.w3.org/TR/html4/index/list.html`, (err, meta) =>
+  console.log(err, meta)
+)
+
+scrape(`https://www.npmjs.com/static/images/not-found.png`, (err, meta) =>
+  console.log(err, meta)
+)
+
+scrape(`https://medium.freecodecamp.com/functional-setstate-is-the-future-of-react-374f30401b6b#.ih2sesp1k`, (err, meta) =>
+  console.log(err, meta)
+)