Add metascraper lang detector package (#114)

* Fix tests

* Add metascraper-audio

* Add metascraper-lang-detector

* Update

* Simplify implementation

Author: Kikobeats

README.md

@@ -95,6 +95,9 @@ The output will be something like:
 
 Here is an example of the metadata that **metascraper** can collect:
 
+- `audio` — eg. *https://cf-media.sndcdn.com/U78RIfDPV6ok.128.mp3*<br/>
+A audio URL that best represents the article.
+
 - `author` — eg. *Noah Kulwin*<br/>
   A human-readable representation of the author's name.
 
@@ -103,7 +106,7 @@ Here is an example of the metadata that **metascraper** can collect:
 
 - `description` — eg. *Venture capitalists are raising money at the fastest rate...*<br/>
   The publisher's chosen description of the article.
-  
+
 - `audio` — eg. *https://cf-media.sndcdn.com/U78RIfDPV6ok.128.mp3*<br/>
 A audio URL that best represents the article.
 

packages/metascraper-lang-detector/.npmrc

@@ -0,0 +1,4 @@
+unsafe-perm=true
+save-prefix=~
+shrinkwrap=false
+save=false

packages/metascraper-lang-detector/README.md

@@ -0,0 +1,19 @@
+# metascraper-lang-detector
+
+[![npm](https://img.shields.io/npm/v/metascraper-lang-detector.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-lang-detector)
+[![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-lang-detector&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-lang-detector)
+
+> Get lang property from HTML markup based on natural language processor.
+
+## Install
+
+```bash
+$ npm install metascraper-lang-detector --save
+```
+
+## License
+
+**metascraper-lang-detector** © [microlink.io](https://microlink.io), Released under the [MIT](https://github.com/microlinkhq/metascraper-lang-detector/blob/master/LICENSE.md) License.<br>
+Authored and maintained by microlink.io with help from [contributors](https://github.com/microlinkhq/metascraper-lang-detector/contributors).
+
+> [microlink.io](https://microlink.io) · GitHub [@microlink.io](https://github.com/microlinkhq) · Twitter [@microlinkhq](https://twitter.com/microlinkhq)

packages/metascraper-lang-detector/index.js

@@ -0,0 +1,33 @@
+'use strict'
+
+const { lang } = require('@metascraper/helpers')
+const { reduce, get } = require('lodash')
+const langs = require('iso-639-3')
+const franc = require('franc')
+
+const toIso6391 = reduce(
+  langs,
+  (acc, { iso6393, iso6391 }) => {
+    if (iso6391) acc[iso6393] = iso6391
+    return acc
+  },
+  {}
+)
+
+const detectLang = (collection, field) => {
+  const value = get(collection, field)
+  const iso6393 = franc(value)
+  return lang(toIso6391[iso6393])
+}
+
+module.exports = ({ fields = ['description'] }) =>
+  reduce(
+    fields,
+    (acc, prop) => {
+      const fn = ({ meta }) => detectLang(meta, prop)
+      return acc.concat(fn)
+    },
+    { lang: [] }
+  )
+
+module.exports.detectLang = detectLang

packages/metascraper-lang-detector/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "metascraper-lang-detector",
+  "description": "Get lang property from HTML markup based on natural language processor",
+  "homepage": "https://metascraper.js.org",
+  "version": "4.2.0",
+  "main": "index.js",
+  "author": {
+    "email": "ian@ianstormtaylor.com",
+    "name": "Ian Storm Taylor"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-lang-detector"
+  },
+  "bugs": {
+    "url": "https://github.com/microlinkhq/metascraper/issues"
+  },
+  "dependencies": {
+    "@metascraper/helpers": "^4.2.0",
+    "franc": "~4.0.0",
+    "iso-639-3": "~1.1.0"
+  },
+  "devDependencies": {
+    "mocha": "latest",
+    "nyc": "latest",
+    "should": "latest",
+    "standard": "11"
+  },
+  "engines": {
+    "node": ">= 8"
+  },
+  "files": [
+    "index.js"
+  ],
+  "scripts": {
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test"
+  },
+  "license": "MIT",
+  "peerDependencies": {
+    "metascraper": "^4"
+  },
+  "standard": {
+    "env": [
+      "mocha"
+    ]
+  }
+}

packages/metascraper-lang-detector/test/index.js

@@ -0,0 +1,29 @@
+'use strict'
+
+const should = require('should')
+
+const { detectLang } = require('..')
+
+describe('metascraper-lang', () => {
+  it('.detectLang', () => {
+    should(
+      detectLang(
+        {
+          description:
+            'A library to easily scrape metadata from an article on the web using Open Graph metadata, regular HTML metadata, and series of fallbacks.'
+        },
+        'description'
+      )
+    ).be.equal('en')
+    should(
+      detectLang(
+        {
+          description:
+            'Una libreria para obtener fácilmente metadatos de cualquier artículo de internet usando Open Graph, HTML y una serie de fallbacks.'
+        },
+        'description'
+      )
+    ).be.equal('es')
+    should(detectLang({ description: null }, 'description')).be.equal(false)
+  })
+})

packages/metascraper-lang-detector/test/mocha.opts

@@ -0,0 +1,6 @@
+--require should
+--reporter spec
+--timeout 120000
+--slow 300
+--bail
+--recursive