Permalink
Browse files

Add prerender support

Close #41
  • Loading branch information...
Kikobeats committed Jun 26, 2018
1 parent e1148c6 commit 7a7adbeae82a36d31673c0218c42324a62d3bc52
Showing with 12 additions and 8 deletions.
  1. +3 −4 bin/cli/extract-urls.js
  2. +1 −1 bin/cli/help.js
  3. +7 −2 bin/cli/index.js
  4. +1 −1 package.json
@@ -2,15 +2,14 @@
const getUrlsFromHtml = require('html-urls')
const fromXML = require('xml-urls')
const getHTML = require('html-get')
const cheerio = require('cheerio')
const { map } = require('lodash')
const got = require('got')
const { isXmlUrl } = fromXML
const fromHTML = async (url, { selector, ...opts }) => {
// TODO: Add prerender support
const { body: rawHtml } = await got(url, opts)
const fromHTML = async (url, { selector, prerender, ...opts }) => {
const { html: rawHtml } = await getHTML(url, { prerender })
const $ = cheerio.load(rawHtml)
const html = selector ? $(selector).html() : rawHtml
const urls = await getUrlsFromHtml({ url, html, ...opts })
@@ -16,8 +16,8 @@ module.exports = gray(`${gray(description)}
-r, --retries Number of request retries when network errors happens (defaults to 2)
-s, --selector Only detects URLs inside the selector
-t, --timeout Milliseconds to wait before consider a timeout response
-p, --prerender Enable or disable prerendering for getting HTML markup (defaults to auto)
-v, --verbose Enable verbose output (defaults to false)
-v, --version Output the version number
-w, --whitelist Add one or multiple url pattern to ignore
Examples
@@ -5,8 +5,8 @@
const cosmiconfig = require('cosmiconfig')('urlint')
const { first, size, concat, isEmpty } = require('lodash')
const { ci } = require('ci-env')
const urlint = require('@urlint/core')
const { ci } = require('ci-env')
const extractUrls = require('./extract-urls')
const renderError = require('./render-error')
@@ -55,6 +55,11 @@ const cli = require('meow')(require('./help'), {
type: 'number',
default: 100
},
prerender: {
alias: 'p',
type: 'boolean',
default: 'auto'
},
timeout: {
alias: 't',
type: 'number',
@@ -68,7 +73,7 @@ const cli = require('meow')(require('./help'), {
}
})
;(async () => {
const { config } = (await cosmiconfig.search()) || {}
const { config = {} } = (await cosmiconfig.search()) || {}
const input = config.url || first(cli.input)
try {
@@ -39,7 +39,7 @@
"clean-stack": "~1.3.0",
"cosmiconfig": "~5.0.5",
"github-build": "~1.2.0",
"got": "~8.3.1",
"html-get": "~1.0.1",
"html-urls": "~2.0.0",
"lodash": "~4.17.10",
"meow": "~5.0.0",

0 comments on commit 7a7adbe

Please sign in to comment.