Skip to content

Commit

Permalink
Merge branch 'master' into feature/checkContentType
Browse files Browse the repository at this point in the history 
* master:
  Separate external-check from external-redirect operation
  Refactor away logResult function and lodash
  Refactor away url module dependency
  Escape newlines in literal output from html. This avoids breaking yaml blocks

Conflicts:
  lib/index.js
  • Loading branch information
@papandreou
papandreou committed Mar 30, 2018
2 parents 590a1ba + f77e47a commit 3c5d93f
Show file tree
Hide file tree
Showing 5 changed files with 231 additions and 154 deletions.
297 changes: 149 additions & 148 deletions lib/index.js
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
const _ = require('lodash');
const AssetGraph = require('assetgraph');
const async = require('async');
const request = require('request');
const version = require('../package.json').version;
const relationDebugDescription = require('./relationDebugDescription');
const prettyBytes = require('pretty-bytes');
const urlModule = require('url');
const net = require('net');
const tls = require('tls');

Expand Down Expand Up @@ -100,71 +98,10 @@ async function hyperlink({
t.push(null, report);
}

function logResult(status, url, redirects = [], incoming = []) {
const at = _.uniq(incoming.map(r => r.debugDescription)).join('\n ');

if (status === false) {
reportTest({
ok: false,
name: 'should accept connections',
operator: 'error',
expected: `connection accepted ${url}`,
actual: `${status} ${url}`,
at
});
} else if (status !== 200 && status !== true) {
reportTest({
ok: false,
name: 'should respond with HTTP status 200',
operator: 'error',
expected: `200 ${url}`,
actual: `${status} ${url}`,
at
});
}

if (typeof status !== 'boolean') {
const report = {
ok: true,
name: `URI should have no redirects - ${url}`,
operator: 'noRedirects',
expected: `200 ${url}`,
at
};

if (redirects.length) {
const log = [{ redirectUri: url }, ...redirects].map((item, idx, arr) => {
if (arr[idx + 1]) {
item.statusCode = arr[idx + 1].statusCode;
} else {
item.statusCode = 200;
}

return item;
});

const logLine = log.map(
redirect => `${redirect.statusCode} ${redirect.redirectUri}`
).join(' --> ');

report.actual = logLine;

if (log[0].statusCode !== 302) {
report.ok = false;
}
} else {
report.actual = `${status} ${url}`;
}

reportTest(report);
}
}

function tryConnect(url, incoming, connectionReport) {
const urlObj = urlModule.parse(url);
const hostname = urlObj.hostname;
const isTls = urlObj.protocol === 'https:';
const port = urlObj.port ? parseInt(urlObj.port, 10) : (isTls ? 443 : 80);
function tryConnect(asset, connectionReport) {
const hostname = asset.hostname;
const isTls = asset.protocol === 'https:';
const port = asset.port ? parseInt(asset.port, 10) : (isTls ? 443 : 80);

if (shouldSkip(connectionReport)) {
return callback => callback(undefined, true);
Expand All @@ -184,14 +121,12 @@ async function hyperlink({
const code = error.code;
const message = error.message;

let actual;
let actual = message || 'Unknown error';

switch (code) {
case 'ENOTFOUND':
actual = `DNS missing: ${hostname}`;
break;
default:
actual = message || 'Unknown error';
}

reportTest({
Expand All @@ -205,7 +140,21 @@ async function hyperlink({
};
}

function httpStatus(asset, relations, attempt = 1) {
function httpStatus(asset, attempt = 1) {
const url = asset.url;
const relations = asset._incoming;

const loadReport = {
operator: 'external-check',
name: `external-check ${url}`,
at: [...new Set(relations.map(r => r.debugDescription))].join('\n '),
expected: `200 ${url}`
};

if (shouldSkip(loadReport)) {
return callback => callback(undefined, true);
}

return callback => {
request({
method: attempt === 1 ? 'head' : 'get',
Expand All @@ -218,94 +167,152 @@ async function hyperlink({
'Accept-Encoding': 'gzip, deflate, sdch, br'
}
}, (error, res) => {
let status;

if (error) {
const code = error.code;
let actual = code || 'Unknown error';

switch (code) {
case 'ENOTFOUND':
actual = `DNS missing: ${asset.hostname}`;
break;
case 'HPE_INVALID_CONSTANT':
if (attempt === 1) {
return httpStatus(asset, attempt + 1)(callback);
}
break;
}

reportTest({
...loadReport,
ok: false,
actual
});

return callback(undefined, false);
}

const status = res.statusCode;
if (status >= 200 && status < 300) {
const contentType = res.headers['content-type'];
if (contentType && asset.type) {
const matchContentType = contentType.match(
/^\s*([\w\-+.]+\/[\w-+.]+)(?:\s|;|$)/i
);
if (matchContentType) {
const expected = asset.contentType || `A Content-Type compatible with ${asset.type}`;
asset.contentType = matchContentType[1].toLowerCase();
const inferredType = asset._inferType();
if (!asset._isCompatibleWith(inferredType)) {
const skip = shouldSkip(asset.url);

if (code) {
// Some servers send responses that request apparently handles badly when using the HEAD method...
if (code === 'HPE_INVALID_CONSTANT' && attempt === 1) {
return httpStatus(asset, relations, attempt + 1)(callback);
reportTest({
ok: false,
skip,
name: `${asset.urlOrDescription}: Should have the expected Content-Type`,
operator: 'error',
expected,
actual: contentType,
at: [...new Set(relations.map(r => r.debugDescription))].join('\n '),
});
}
}
} else if (!contentType) {
const skip = shouldSkip(asset.url);

if (code === 'ENOTFOUND') {
status = 'DNS Missing';
reportTest({
ok: false,
skip,
name: `${asset.urlOrDescription}: No Content-Type response header`,
operator: 'error',
expected: asset.contentType || `A Content-Type compatible with ${asset.type}`,
actual: contentType,
at: [...new Set(relations.map(r => r.debugDescription))].join('\n '),
});
}
}

// Some servers respond weirdly to HEAD requests. Make a second attempt with GET
if (attempt === 1 && status >= 400 && status < 600) {
return httpStatus(asset, attempt + 1)(callback);
}

// Some servers (jspm.io) respond with 502 if requesting HEAD, then GET to close in succession. Give the server a second to cool down
if (attempt === 2 && status === 502) {
setTimeout(
() => httpStatus(asset, attempt + 1)(callback),
1000
);
return;
}

const redirects = res.request._redirect.redirects;
if (redirects.length > 0) {
const log = [{ redirectUri: url }, ...redirects].map((item, idx, arr) => {
if (arr[idx + 1]) {
item.statusCode = arr[idx + 1].statusCode;
} else {
status = code;
item.statusCode = 200;
}
} else {
status = 'Unknown error';
}

logResult(status, asset.url, undefined, relations);
return item;
});

callback(undefined, status);
} else {
// Some servers respond weirdly to HEAD requests. Make a second attempt with GET
if (attempt === 1 && res.statusCode >= 400 && res.statusCode < 600) {
return httpStatus(asset, relations, attempt + 1)(callback);
}
const redirectReport = {
operator: 'external-redirect',
name: `external-redirect ${url}`,
at: [...new Set(relations.map(r => r.debugDescription))].join('\n '),
expected: `302 ${url} --> 200 ${log[log.length - 1].redirectUri}`
};

// Some servers (jspm.io) respond with 502 if requesting HEAD, then GET to close in succession. Give the server a second to cool down
if (attempt === 2 && res.statusCode === 502) {
setTimeout(
() => httpStatus(asset, relations, attempt + 1)(callback),
1000
);
return;
}
const actual = log.map(
redirect => `${redirect.statusCode} ${redirect.redirectUri}`
).join(' --> ');

status = res.statusCode;
if (status >= 200 && status < 300) {
const contentType = res.headers['content-type'];
if (contentType && asset.type) {
const matchContentType = contentType.match(
/^\s*([\w\-+.]+\/[\w-+.]+)(?:\s|;|$)/i
);
if (matchContentType) {
const expected = asset.contentType || `A Content-Type compatible with ${asset.type}`;
asset.contentType = matchContentType[1].toLowerCase();
const inferredType = asset._inferType();
if (!asset._isCompatibleWith(inferredType)) {
const at = _.uniq(relations.map(r => r.debugDescription)).join('\n ');
const skip = shouldSkip(asset.url);

reportTest({
ok: false,
skip,
name: `${asset.urlOrDescription}: Should have the expected Content-Type`,
operator: 'error',
expected,
actual: contentType,
at
});
}
if (!shouldSkip(redirectReport)) {
// A single temporary redirect is allowed
if ([302, 307].includes(log[0].statusCode)) {
if (log.length < 3) {
reportTest({
...redirectReport,
expected: actual,
actual,
ok: true
});
} else {
reportTest({
...redirectReport,
expected: `${log[0].statusCode} ${url} --> 200 ${log[log.length - 1].redirectUri}`,
actual,
ok: false
});
}
} else if (!contentType) {
const at = _.uniq(relations.map(r => r.debugDescription)).join('\n ');
const skip = shouldSkip(asset.url);

} else {
reportTest({
ok: false,
skip,
name: `${asset.urlOrDescription}: No Content-Type response header`,
operator: 'error',
expected: asset.contentType || `A Content-Type compatible with ${asset.type}`,
actual: contentType,
at
...redirectReport,
actual,
ok: false
});
}
}
}

const redirects = res.request._redirect.redirects || [];
const firstRedirectStatus = redirects[0] && redirects[0].statusCode;

logResult(status, asset.url, redirects, relations);
if (status === 200) {
reportTest({
...loadReport,
ok: true,
actual: loadReport.expected
});

callback(undefined, firstRedirectStatus || status);
return callback(undefined, true);
}

reportTest({
...loadReport,
actual: `${status} ${url}`,
ok: false
});

return callback(undefined, false);
});
};
}
Expand Down Expand Up @@ -581,13 +588,7 @@ async function hyperlink({
});

await new Promise((resolve, reject) => async.parallelLimit(
assetsToCheck
.filter(asset => !shouldSkip({
operator: 'external-check',
name: `external-check ${asset.url}`,
at: [...new Set(asset._incoming.map(r => r.debugDescription))].join('\n ')
}))
.map(asset => httpStatus(asset, asset._incoming)),
assetsToCheck.map(asset => httpStatus(asset)),
20,
err => {
if (err) {
Expand All @@ -608,7 +609,7 @@ async function hyperlink({

await new Promise((resolve, reject) => async.parallelLimit(
preconnectAssetsToCheck
.map(asset => tryConnect(asset.url, asset._incoming, {
.map(asset => tryConnect(asset, {
operator: 'preconnect-check',
name: `preconnect-check ${asset.url}`,
at: [...new Set(asset._incoming.map(r => r.debugDescription))].join('\n '),
Expand All @@ -634,7 +635,7 @@ async function hyperlink({

await new Promise((resolve, reject) => async.parallelLimit(
dnsPrefetchAssetsToCheck
.map(asset => tryConnect(asset.url, asset._incoming, {
.map(asset => tryConnect(asset, {
operator: 'dns-prefetch-check',
name: `dns-prefetch-check ${asset.hostname}`,
at: [...new Set(asset._incoming.map(r => r.debugDescription))].join('\n '),
Expand Down
4 changes: 3 additions & 1 deletion lib/relationDebugDescription.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@ module.exports = function relationDebugDescription(relation) {

// DOM node
if (node && node.outerHTML) {
details += node.outerHTML.split('>' + node.innerHTML + '<').join('>...<');
details += node.outerHTML.split('>' + node.innerHTML + '<')
.join('>...<')
.replace(/\n/g, '\\n');
}

// CSS node
Expand Down
1 change: 0 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@
"dependencies": {
"assetgraph": "^4.3.1",
"async": "^2.6.0",
"lodash": "^4.17.4",
"optimist": "^0.6.1",
"pretty-bytes": "^4.0.2",
"request": "^2.83.0",
Expand Down
Loading

0 comments on commit 3c5d93f

Please sign in to comment.