Permalink
Browse files

Disable streaming on CI

  • Loading branch information...
Kikobeats committed Feb 2, 2018
1 parent 4523ad7 commit a5a32407f674d17944b5fc118024fe4cc10e0775
Showing with 80 additions and 54 deletions.
  1. +4 −4 bin/cli/get-urls.js
  2. +2 −1 bin/cli/help.js
  3. +27 −12 bin/view/helpers.js
  4. +36 −35 bin/view/index.js
  5. +1 −2 bin/view/render.js
  6. +10 −0 bin/view/theme.js
@@ -3,16 +3,16 @@
const { getUrl, isUrl } = require('@metascraper/helpers')
const normalizeUrl = require('normalize-url')
const getUrlsFromHtml = require('html-urls')
const { map, reduce } = require('lodash')
const fromXML = require('xml-urls')
const { reduce } = require('lodash')
const got = require('got')
const { isXmlUrl } = fromXML
const fromHTML = async (url, { whitelist, ...opts }) => {
const fromHTML = async (url, opts) => {
const { body: html } = await got(url, opts)
const urls = await getUrlsFromHtml({ url, html, whitelist })
return urls.map(({ normalizeUrl }) => normalizeUrl)
const urls = await getUrlsFromHtml({ url, html, ...opts })
return map(urls, 'normalizeUrl')
}
module.exports = async (url, opts) => {
@@ -1,7 +1,8 @@
'use strict'
const chalk = require('chalk')
const { theme } = require('../view/helpers')
const theme = require('../view/theme')
const { description } = require('../../package.json')
const green = chalk.hex(theme.green)
@@ -1,16 +1,9 @@
'use strict'
const chalk = require('chalk')
const { mapValues } = require('lodash')
const THEME = {
red: '#ff5c57',
green: '#5af78e',
yellow: '#f3f99d',
blue: '#57c7ff',
magenta: '#ff6ac1',
cyan: '#9aedfe'
}
const { isEqual, includes, keys, size, mapValues, isNil } = require('lodash')
const theme = require('./theme')
const STATUS_CODE_COLOR = {
'2': 'green',
@@ -20,7 +13,7 @@ const STATUS_CODE_COLOR = {
'9': 'red'
}
const statusCodeColors = mapValues(THEME, hex => chalk.hex(hex))
const statusCodeColors = mapValues(theme, hex => chalk.hex(hex))
const STATUS_CODE_COLOR_FALLBACK = 'magenta'
@@ -39,8 +32,30 @@ const colorizeStatus = (statusCode, str = statusCode) => {
const colorizeLine = chalk.gray
const setState = (state, { statusCode, data }) => {
const status = state.count[statusCode]
const linkStatus = state.links[statusCode]
const linkItem = [data.statusCode, data.url]
const count = { [statusCode]: isNil(status) ? 1 : status + 1 }
const links = {
[statusCode]: isNil(linkStatus) ? [linkItem] : linkStatus.concat([linkItem])
}
return { count, links }
}
const processExit = data => {
const statusCodes = keys(data)
const isOnly2xx =
includes(statusCodes, '2xx') && isEqual(size(statusCodes), 1)
const exitCode = isOnly2xx ? 0 : 1
process.exit(exitCode)
}
module.exports = {
colorizeStatus,
colorizeLine,
theme: THEME
setState,
processExit
}
@@ -1,52 +1,53 @@
'use strict'
const { size, isNil } = require('lodash')
const neatLog = require('neat-log')
const { processExit, setState } = require('./helpers')
const render = require('./render')
const setState = (state, { statusCode, data }) => {
const status = state.count[statusCode]
const linkStatus = state.links[statusCode]
const linkItem = [data.statusCode, data.url]
const count = { [statusCode]: isNil(status) ? 1 : status + 1 }
const links = {
[statusCode]: isNil(linkStatus) ? [linkItem] : linkStatus.concat([linkItem])
}
return { count, links }
}
module.exports = ({ emitter, quiet, ...opts }) => {
const neat = neatLog(render, opts)
neat.use((state, bus) => {
state.quiet = quiet
state.count = {}
state.links = {}
state.end = false
setInterval(() => {
bus.emit('render')
}, 100)
if (quiet) {
const state = { count: {}, links: {} }
emitter.on('status', ({ statusCode, data }) => {
const newState = setState(state, { statusCode, data })
state.count = { ...state.count, ...newState.count }
state.links = { ...state.links, ...newState.links }
})
emitter.on('fetching', ({ url }) => {
state.fetchingUrl = url
})
emitter.on('end', data => {
state.end = true
bus.clear()
neat.render()
const exitCode = size(data) > 1 ? 1 : 0
process.exit(exitCode)
const output = render.links(state)
console.log(output)
processExit(data)
})
})
} else {
const neat = neatLog(render, opts)
neat.use((state, bus) => {
state.count = {}
state.links = {}
state.end = false
setInterval(() => {
bus.emit('render')
}, 100)
emitter.on('status', ({ statusCode, data }) => {
const newState = setState(state, { statusCode, data })
state.count = { ...state.count, ...newState.count }
state.links = { ...state.links, ...newState.links }
})
emitter.on('fetching', ({ url }) => {
state.fetchingUrl = url
})
emitter.on('end', data => {
state.end = true
bus.clear()
neat.render()
processExit(data)
})
})
}
}
@@ -15,8 +15,7 @@ const resumeCount = (count, statusCode) => {
}
const count = state => {
const { quiet, count, fetchingUrl = '' } = state
if (quiet) return ''
const { count, fetchingUrl = '' } = state
const resume = map(count, resumeCount).join(os.EOL)
const footer = fetchingUrl
@@ -0,0 +1,10 @@
'use strict'
module.exports = {
red: '#ff5c57',
green: '#5af78e',
yellow: '#f3f99d',
blue: '#57c7ff',
magenta: '#ff6ac1',
cyan: '#9aedfe'
}

0 comments on commit a5a3240

Please sign in to comment.