diff --git a/README.md b/README.md index c3f8f60..a533896 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ npm install cheerio-httpcli * [followMetaRefresh](#followmetarefresh) * [maxDataSize](#maxdatasize) * [forceHtml](#forcehtml) + * [agentOptions](#agentOptions) * [debug](#debug) * [download](#download-readonly) * [cheerioオブジェクトの独自拡張](#cheerio%E3%82%AA%E3%83%96%E3%82%B8%E3%82%A7%E3%82%AF%E3%83%88%E3%81%AE%E7%8B%AC%E8%87%AA%E6%8B%A1%E5%BC%B5) @@ -458,6 +459,21 @@ cheerio-httpcliは取得したページがXMLであると判別した場合、 `true`にするとこの自動判別を無効にして常にHTMLモードでコンテンツをパースするようになります。デフォルトは`false`(自動判別する)です。 +### agentOptions + +主にSSL接続などのセキュリティの設定を行うオプションです。cheerio-httpcli内部で使用しているrequestモジュールにそのまま渡されます。デフォルトは空連想配列です。 + +基本的には何も設定する必要はありませんが、httpsページへのアクセスができない場合にこのオプションを設定することにより解決する可能性があります。設定方法などの詳細は[requestモジュールのドキュメント](https://github.com/request/request#using-optionsagentoptions)を参照してください。 + +#### 設定例 + +```js +// TLS1.2での接続を強制する +client.set('agentOptions', { + secureProtocol: 'TLSv1_2_method' +}); +``` + ### debug `true`にするとリクエストの度にデバッグ情報を出力します(`stderr`)。デフォルトは`false`です。 @@ -1145,6 +1161,21 @@ var client = require('cheerio-httpcli'); client.fetch('http://foo.bar.baz/', ... ``` +### 今まで接続できていたhttpsのページに接続できなくなった場合 + +`0.7.2`からhttps接続方法に関する仕様に若干変更があり、その影響で今まで接続できていたページに接続できないというケースが発生するかもしれません。 + +`0.7.1`までと同じ挙動にする場合は以下のように設定してください。 + +```js +var client = require('cheerio-httpcli'); +var constants = require('constants'); // <- constantsモジュールを別途インストール + +client.set('agentOptions', { + secureOptions: constants.SSL_OP_NO_TLSv1_2 +}); +``` + ### XMLの名前空間付きタグの指定方法 ```xml diff --git a/lib/client.js b/lib/client.js index f009e13..29751eb 100644 --- a/lib/client.js +++ b/lib/client.js @@ -9,7 +9,6 @@ var each = require('foreach'); var typeOf = require('type-of'); var assign = require('object-assign'); var prettyjson = require('prettyjson'); -var constants = require('constants'); var spawnSync = require('spawn-sync'); var path = require('path'); var cutil = require('./cheerio/util'); @@ -535,7 +534,7 @@ module.exports = { time: true, followRedirect: true, jar: this.jar, - secureOptions: constants.SSL_OP_NO_TLSv1_2 // とりあえず付けてみた + agentOptions: cli.agentOptions }, encode: encode, callback: callback diff --git a/lib/core.js b/lib/core.js index 45c71b7..e31d7ed 100644 --- a/lib/core.js +++ b/lib/core.js @@ -72,6 +72,8 @@ var defineNormalProperties = function (cli) { maxDataSize : { types: [ 'number', 'null' ], value: null }, // XML自動判別を使用しない forceHtml : { types: [ 'boolean' ], value: null }, + // requestモジュールに渡すagentOptions + agentOptions : { types: [ 'object' ], value: null }, // デバッグオプション debug : { types: [ 'boolean' ], value: null } }; @@ -203,6 +205,8 @@ var CheerioHttpCli = (function () { this.set('maxDataSize', null); // XML自動判別を使用しない this.set('forceHtml', false); + // requestモジュールに渡すagentOptions + this.set('agentOptions', {}, true); // デバッグオプション this.set('debug', false); diff --git a/package.json b/package.json index 7129409..962a1c2 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,6 @@ "async": "^2.5.0", "cheerio": "^0.22.0", "colors": "^1.1.2", - "constants": "0.0.2", "foreach": "^2.0.5", "he": "^1.1.1", "iconv-lite": "^0.4.19", @@ -49,6 +48,7 @@ }, "devDependencies": { "dev-null": "^0.1.1", + "constants": "^0.0.2", "eslint": "^4.10.0", "espower-loader": "^1.2.2", "intelli-espower-loader": "^1.0.1", diff --git a/test/_helper.js b/test/_helper.js index abb21c5..7735b5b 100644 --- a/test/_helper.js +++ b/test/_helper.js @@ -2,12 +2,14 @@ /*jshint -W100*/ var nstatic = require('node-static'); var http = require('http'); +var https = require('https'); var path = require('path'); var strip = require('strip-ansi'); var each = require('foreach'); var random = require('random-string'); var fs = require('fs'); var qs = require('querystring'); +var EventEmitter = require('events').EventEmitter; /** * テスト用のヘルパーモジュール本体 @@ -18,8 +20,14 @@ module.exports = { */ // テストサーバー設定 - port: 5555, + httpPort: 55551, + httpsPort: 55552, root: path.join(__dirname, 'fixtures'), + ready: { + http: false, + https: false + }, + emitter: new EventEmitter(), /** * メソッド @@ -29,6 +37,9 @@ module.exports = { * テストHTMLページのURLを作成 */ url: function (dir, file) { + if (dir === '%https%') { + return 'https://localhost:' + this.httpsPort + '/'; + } if (! file) { file = dir; dir = ''; @@ -36,7 +47,7 @@ module.exports = { dir += '/'; file += '.html'; } - return 'http://localhost:' + this.port + '/' + dir + file; + return 'http://localhost:' + this.httpPort + '/' + dir + file; }, /** @@ -170,7 +181,16 @@ module.exports = { } }; - return http.createServer(function (req, res) { + this.emitter.on('start', (function (protocol) { + this.ready[protocol] = true; + if (Object.keys(this.ready).every((function (srv) { + return this.ready[srv]; + }).bind(this))) { + process.stderr.write('%%% server ready %%%\n'); + } + }).bind(this)); + + http.createServer(function (req, res) { var pdata = ''; req.on('data', function (data) { pdata += data; @@ -214,9 +234,29 @@ module.exports = { }, parseInt(wait, 10)); /*eslint-disable consistent-return*/ return; /*eslint-enable consistent-return*/ }).resume(); - }).listen(this.port, '0.0.0.0', function () { - process.stderr.write('%%% server start %%%'); + }).listen(this.httpPort, '0.0.0.0', (function () { + this.emitter.emit('start', 'http'); + }).bind(this)); + + // httpsサーバーも稼働 + var httpsOpts = { + secureProtocol: 'TLSv1_2_method' // TLS1.2のみ対応 + }; + [ 'key', 'cert' ].forEach(function (pem) { + httpsOpts[pem] = fs.readFileSync( + path.join(__dirname, 'pem/' + pem + '.pem'), + 'utf-8' + ); }); + + https.createServer(httpsOpts, function (req, res) { + res.writeHead(200, { + 'Content-Type': 'text/plain' + }); + res.end('hello, https'); + }).listen(this.httpsPort, '0.0.0.0', (function () { + this.emitter.emit('start', 'https'); + }).bind(this)); }, /** diff --git a/test/fixtures/refresh/absolute.html b/test/fixtures/refresh/absolute.html index 3a568fa..19806fd 100644 --- a/test/fixtures/refresh/absolute.html +++ b/test/fixtures/refresh/absolute.html @@ -1 +1 @@ - + diff --git a/test/fixtures/refresh/ie-only.html b/test/fixtures/refresh/ie-only.html index c2fefab..5a47239 100644 --- a/test/fixtures/refresh/ie-only.html +++ b/test/fixtures/refresh/ie-only.html @@ -3,7 +3,7 @@ Refresh IE only diff --git a/test/https.js b/test/https.js new file mode 100644 index 0000000..3791668 --- /dev/null +++ b/test/https.js @@ -0,0 +1,85 @@ +/*eslint-env mocha*/ +/*eslint no-invalid-this:0*/ +/*jshint -W100*/ +var assert = require('power-assert'); +var typeOf = require('type-of'); +var constants = require('constants'); +var helper = require('./_helper'); +var cli = require('../index'); + +// オレオレ証明書のサーバーにアクセスできるようにする +process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0; + +describe('https', function () { + it('agentOptions未設定 => TLS1.2 Onlyのサーバーに接続可能', function (done) { + cli.fetch(helper.url('%https%'), function (err, $, res, body) { + assert(! err); + assert(typeOf(res) === 'object'); + assert(typeOf($) === 'function'); + assert(typeOf(body) === 'string'); + assert(body === 'hello, https'); + done(); + }); + }); + + it('agentOptions: TLS1.2強制 => TLS1.2 Onlyのサーバーに接続可能', function (done) { + cli.set('agentOptions', { + secureProtocol: 'TLSv1_2_method' + }); + cli.fetch(helper.url('%https%'), function (err, $, res, body) { + assert(! err); + assert(typeOf(res) === 'object'); + assert(typeOf($) === 'function'); + assert(typeOf(body) === 'string'); + assert(body === 'hello, https'); + done(); + }); + }); + + it('agentOptions: TLS1.2強制 => httpのサーバーにも接続可能', function (done) { + cli.set('agentOptions', { + secureProtocol: 'TLSv1_2_method' + }); + cli.fetch(helper.url('~info'), function (err, $, res, body) { + assert(! err); + assert(typeOf(res) === 'object'); + assert(typeOf($) === 'function'); + assert(typeOf(body) === 'string'); + done(); + }); + }); + + it('agentOptions: TLS1.1強制 => TLS1.2 Onlyのサーバーに接続不可', function (done) { + cli.set('agentOptions', { + secureProtocol: 'TLSv1_1_method' + }); + var url = helper.url('%https%'); + cli.fetch(url, function (err, $, res, body) { + assert(err.errno === 'EPROTO'); + assert(err.code === 'EPROTO'); + assert(err.message.indexOf('handshake failure:') !== -1); + assert(err.url === url); + assert(! res); + assert(! $); + assert(! body); + done(); + }); + }); + + it('agentOptions: TLS1.2無効 => TLS1.2 Onlyのサーバーに接続不可', function (done) { + cli.set('agentOptions', { + secureOptions: constants.SSL_OP_NO_TLSv1_2 + }); + var url = helper.url('%https%'); + cli.fetch(url, function (err, $, res, body) { + assert(err.errno === 'EPROTO'); + assert(err.code === 'EPROTO'); + assert(err.message.indexOf('handshake failure:') !== -1); + assert(err.url === url); + assert(! res); + assert(! $); + assert(! body); + done(); + }); + }); +}); diff --git a/test/pem/cert.pem b/test/pem/cert.pem new file mode 100644 index 0000000..eeeb114 --- /dev/null +++ b/test/pem/cert.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDTDCCAjQCCQCIfNWNVBmJrTANBgkqhkiG9w0BAQsFADBnMQswCQYDVQQGEwJK +UDEOMAwGA1UECAwFVG9reW8xFjAUBgNVBAcMDU11c2FzaGluby1zaGkxDDAKBgNV +BAoMA0ZvbzEMMAoGA1UECwwDQmFyMRQwEgYDVQQDDAtmb28uYmFyLmNvbTAgFw0x +NzExMTAwMDUyNTlaGA8yMjE3MDkyMzAwNTI1OVowZzELMAkGA1UEBhMCSlAxDjAM +BgNVBAgMBVRva3lvMRYwFAYDVQQHDA1NdXNhc2hpbm8tc2hpMQwwCgYDVQQKDANG +b28xDDAKBgNVBAsMA0JhcjEUMBIGA1UEAwwLZm9vLmJhci5jb20wggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDs2GLh7pYfzD2rfvokOGud4uRJY7FuhprY +3FL6ICRQBZVtuEZMqDIVUFz89m/VW0GFNz290Pj0UP2WHcXGUr5pfRtNbMLCer1g +DdHDsevSVRZzbgP68EW6q3te5pti7wcso1U2dhFfGZTZNE0PoFdeJYV7bPoByAWH +I/EPWfSOAMb8VcjZbVfVfn69OC1rc07EtVRBa2H9B2ovs03EVc0iuXkNFGzQSsXA +3R98xUaKvimQYPtMnn5mXbN0xrJCIpVMj/HPhDlMX3ZFcSy8W1ICmeH4RT670RnD +j/3H1qdgogKFq+n4eh7Fi3jIZ8Sh3mbQVekIsDLMS8bgeKI9TXeVAgMBAAEwDQYJ +KoZIhvcNAQELBQADggEBAGermRUgss3JjhSL+KizeYhqvJP0+GnmsCHAzItScCUH +ONRjoASb+XkJJbNKreaSn/O7H76sambkUPH6HYpulGnwQvNjhipcs4sEtsMduoUF +zuOwEJDGOuDhi6mbxif8sNuN6Gh0xXPzNxzyGzS5C3sKjxT7G7Wm04FzSUC2dOhS +89xU4r4T9v4otVpieZfHyi81Jua6eSBDaMw0IMXH3SpMdsorbR66c+hsG1eBDyyg +w1i+v0GlB3NAPmomjdtkysjwNpW9l7NbvV3MhR/HRmMPmUp4TSWvAIF6BTgzMHjE +Ichze9UqovoCPO881JWV+nBFQ1EQw0WS4QUSB5nm8JU= +-----END CERTIFICATE----- diff --git a/test/pem/key.pem b/test/pem/key.pem new file mode 100644 index 0000000..b6b85a0 --- /dev/null +++ b/test/pem/key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEA7Nhi4e6WH8w9q376JDhrneLkSWOxboaa2NxS+iAkUAWVbbhG +TKgyFVBc/PZv1VtBhTc9vdD49FD9lh3FxlK+aX0bTWzCwnq9YA3Rw7Hr0lUWc24D ++vBFuqt7XuabYu8HLKNVNnYRXxmU2TRND6BXXiWFe2z6AcgFhyPxD1n0jgDG/FXI +2W1X1X5+vTgta3NOxLVUQWth/QdqL7NNxFXNIrl5DRRs0ErFwN0ffMVGir4pkGD7 +TJ5+Zl2zdMayQiKVTI/xz4Q5TF92RXEsvFtSApnh+EU+u9EZw4/9x9anYKIChavp ++HoexYt4yGfEod5m0FXpCLAyzEvG4HiiPU13lQIDAQABAoIBAQDIBA2t48FgZSmH +lRpGUGeB1MUZvVlwj7hhf9+LYG2KLsz89exYfIqfOVjuQGg9dG2mxPodPUehfGxL +xCTr0aEAkSjnf/wSJXmcjs8hRzZyUG0/Wh9+Yj9g38S2ZmW/bUFPzzf9YERXXdE4 +hVS256Ag3+sUSvnvWy5f7Fh9sGg5KorpIYvJYf7dWfIu+wIHd5soJtxOGD/KLQuG +TmKaTZtAL48bAmsghHePw7XyvsLTBtJb2ZdKGGq/QVD3h6xIu7LZY4Fp/4KIfOFA +O8RFCSwLlQyURBhgW3+gKBrYN7ghzvvhkPojrGqWEGwW/DfO+rWxxeHNfzfnOUPL +kJYn5SlBAoGBAPrGvxGIi7PKkIi7pET5hUdDwzDNJjxZysBpCQHMXssXIBCzRCVq +CsUsXjV/1qhQSKghx/bJW+0QmuX5jMx/gvBMvYtKLndqlAq4VS4WBKgATZ1Bvndj +61KuZUUg21Rd+9bUnStoHDy3n+x9hDQ5gZNshOkAh/IwrZAyPXANGYBZAoGBAPHH +WlvnT6t3BcGkTgNHmXIMIB9hHkG9acM/KvQ4f4aHQJC4F3bCeVuBi0ZINmzuaxUq +azrMZiItgpA8jGN6X8v6QE5/odWRY3gSX38FTTIH1+jwI09iiWOZpBRDK9WWwpAn +X2BlZM0uoGlayi9qbrICrLq1Yn+wDD+Ov6kwh6mdAoGAZ9fY0ufaAa9FvnkFAtLY +T7RNpW2uAZulC5vy8N2x+yMuUfwJofyRTSicMkcnmjb0fzrN1PF4sWgI3GZD2YKL +s/nzGzSynRxzBSVjkFvpva+ydAX/WuzzSx+QK9n5OKxaVpFgK9NGrhXTkVhAYGfX +sjZjqyBfKvjhRi6npjimcLECgYEA2wjrR08q0f+l62PaeQYocTWi9Eqbipr6cbOM +SmvUvB9T0se0GhbcspWNg0JwbAciY65mLoJ2FIh+PAVeedCncLdqArOF/WEVZ/Xd +Jcm7wZNxesnyczylkuHhz6l60Kkf4lCJC19QDsIq+McTXBlj50idCxi//0WSExJT +eAdLH9ECgYEAxh/IanVVsIHed8tU4CoBbHIbyvrjjMlAlhexsrQ0gazJKvdGWGXC +WQnCGNMZ9cnXoYP07upEpkc5hG+P+mqpU2Lpm5zbmteDGKP0e+HG9kBHToKmXVNi +VgyNFITaOpTWERIwVSnQDjFLbIzL8VgQtUUP/bD4QJ7uKE0//TkirBM= +-----END RSA PRIVATE KEY----- diff --git a/testrunner.js b/testrunner.js index 4e1c462..4b4d3f4 100755 --- a/testrunner.js +++ b/testrunner.js @@ -22,18 +22,6 @@ fs.readdirSync(testDir).filter(function (file) { mocha.addFile(path.join(testDir, file)); }); -// Ctrl-C -var stdin = process.stdin; -stdin.setRawMode(true); -stdin.resume(); -stdin.setEncoding('utf-8'); -stdin.on('data', function (key) { - if (key === '\u0003') { - process.kill(server.pid); - process.exit(); - } -}); - var server = spawn(process.execPath, [ path.join(__dirname, 'test/_server.js') ], { @@ -43,12 +31,25 @@ server.stdout.on('data', function (data) { process.stdout.write(data); }); server.stderr.on('data', function (data) { - if (data.toString() === '%%% server start %%%') { + if (data.toString().trim() === '%%% server ready %%%') { // start mocha - return mocha.run(function (failures) { + mocha.run(function (failures) { process.kill(server.pid); process.exit(failures); }); + return; } process.stderr.write(data); }); + +// Ctrl-C +var stdin = process.stdin; +stdin.setRawMode(true); +stdin.resume(); +stdin.setEncoding('utf-8'); +stdin.on('data', function (key) { + if (key === '\u0003') { + process.kill(server.pid); + process.exit(); + } +});