agentOptions追加 & デフォルトのsecureOptionsを削除

ktty1220 · Nov 10, 2017 · 49741ff · 49741ff
1 parent 18765bc
commit 49741ff
Show file tree

Hide file tree

Showing 11 changed files with 231 additions and 24 deletions.
diff --git a/README.md b/README.md
@@ -75,6 +75,7 @@ npm install cheerio-httpcli
   * [followMetaRefresh](#followmetarefresh)
   * [maxDataSize](#maxdatasize)
   * [forceHtml](#forcehtml)
+  * [agentOptions](#agentOptions)
   * [debug](#debug)
   * [download](#download-readonly)
 * [cheerioオブジェクトの独自拡張](#cheerio%E3%82%AA%E3%83%96%E3%82%B8%E3%82%A7%E3%82%AF%E3%83%88%E3%81%AE%E7%8B%AC%E8%87%AA%E6%8B%A1%E5%BC%B5)
@@ -458,6 +459,21 @@ cheerio-httpcliは取得したページがXMLであると判別した場合、
 
 `true`にするとこの自動判別を無効にして常にHTMLモードでコンテンツをパースするようになります。デフォルトは`false`(自動判別する)です。
 
+### agentOptions
+
+主にSSL接続などのセキュリティの設定を行うオプションです。cheerio-httpcli内部で使用しているrequestモジュールにそのまま渡されます。デフォルトは空連想配列です。
+
+基本的には何も設定する必要はありませんが、httpsページへのアクセスができない場合にこのオプションを設定することにより解決する可能性があります。設定方法などの詳細は[requestモジュールのドキュメント](https://github.com/request/request#using-optionsagentoptions)を参照してください。
+
+#### 設定例
+
+```js
+// TLS1.2での接続を強制する
+client.set('agentOptions', {
+  secureProtocol: 'TLSv1_2_method'
+});
+```
+
 ### debug
 
 `true`にするとリクエストの度にデバッグ情報を出力します(`stderr`)。デフォルトは`false`です。
@@ -1145,6 +1161,21 @@ var client = require('cheerio-httpcli');
 client.fetch('http://foo.bar.baz/', ...
 ```
 
+### 今まで接続できていたhttpsのページに接続できなくなった場合
+
+`0.7.2`からhttps接続方法に関する仕様に若干変更があり、その影響で今まで接続できていたページに接続できないというケースが発生するかもしれません。
+
+`0.7.1`までと同じ挙動にする場合は以下のように設定してください。
+
+```js
+var client = require('cheerio-httpcli');
+var constants = require('constants');  // <- constantsモジュールを別途インストール
+
+client.set('agentOptions', {
+  secureOptions: constants.SSL_OP_NO_TLSv1_2
+});
+```
+
 ### XMLの名前空間付きタグの指定方法
 
 ```xml

diff --git a/lib/client.js b/lib/client.js
@@ -9,7 +9,6 @@ var each       = require('foreach');
 var typeOf     = require('type-of');
 var assign     = require('object-assign');
 var prettyjson = require('prettyjson');
-var constants  = require('constants');
 var spawnSync  = require('spawn-sync');
 var path       = require('path');
 var cutil      = require('./cheerio/util');
@@ -535,7 +534,7 @@ module.exports = {
         time: true,
         followRedirect: true,
         jar: this.jar,
-        secureOptions: constants.SSL_OP_NO_TLSv1_2  // とりあえず付けてみた
+        agentOptions: cli.agentOptions
       },
       encode: encode,
       callback: callback

diff --git a/lib/core.js b/lib/core.js
@@ -72,6 +72,8 @@ var defineNormalProperties = function (cli) {
     maxDataSize       : { types: [ 'number', 'null' ], value: null },
     // XML自動判別を使用しない
     forceHtml         : { types: [ 'boolean' ], value: null },
+    // requestモジュールに渡すagentOptions
+    agentOptions      : { types: [ 'object' ], value: null },
     // デバッグオプション
     debug             : { types: [ 'boolean' ], value: null }
   };
@@ -203,6 +205,8 @@ var CheerioHttpCli = (function () {
     this.set('maxDataSize', null);
     // XML自動判別を使用しない
     this.set('forceHtml', false);
+    // requestモジュールに渡すagentOptions
+    this.set('agentOptions', {}, true);
     // デバッグオプション
     this.set('debug', false);
 

diff --git a/package.json b/package.json
@@ -26,7 +26,6 @@
     "async": "^2.5.0",
     "cheerio": "^0.22.0",
     "colors": "^1.1.2",
-    "constants": "0.0.2",
     "foreach": "^2.0.5",
     "he": "^1.1.1",
     "iconv-lite": "^0.4.19",
@@ -49,6 +48,7 @@
   },
   "devDependencies": {
     "dev-null": "^0.1.1",
+    "constants": "^0.0.2",
     "eslint": "^4.10.0",
     "espower-loader": "^1.2.2",
     "intelli-espower-loader": "^1.0.1",

diff --git a/test/_helper.js b/test/_helper.js
@@ -2,12 +2,14 @@
 /*jshint -W100*/
 var nstatic = require('node-static');
 var http    = require('http');
+var https   = require('https');
 var path    = require('path');
 var strip   = require('strip-ansi');
 var each    = require('foreach');
 var random  = require('random-string');
 var fs      = require('fs');
 var qs      = require('querystring');
+var EventEmitter = require('events').EventEmitter;
 
 /**
  * テスト用のヘルパーモジュール本体
@@ -18,8 +20,14 @@ module.exports = {
    */
 
   // テストサーバー設定
-  port: 5555,
+  httpPort: 55551,
+  httpsPort: 55552,
   root: path.join(__dirname, 'fixtures'),
+  ready: {
+    http: false,
+    https: false
+  },
+  emitter: new EventEmitter(),
 
   /**
    * メソッド
@@ -29,14 +37,17 @@ module.exports = {
    * テストHTMLページのURLを作成
    */
   url: function (dir, file) {
+    if (dir === '%https%') {
+      return 'https://localhost:' + this.httpsPort + '/';
+    }
     if (! file) {
       file = dir;
       dir = '';
     } else {
       dir += '/';
       file += '.html';
     }
-    return 'http://localhost:' + this.port + '/' + dir + file;
+    return 'http://localhost:' + this.httpPort + '/' + dir + file;
   },
 
   /**
@@ -170,7 +181,16 @@ module.exports = {
       }
     };
 
-    return http.createServer(function (req, res) {
+    this.emitter.on('start', (function (protocol) {
+      this.ready[protocol] = true;
+      if (Object.keys(this.ready).every((function (srv) {
+        return this.ready[srv];
+      }).bind(this))) {
+        process.stderr.write('%%% server ready %%%\n');
+      }
+    }).bind(this));
+
+    http.createServer(function (req, res) {
       var pdata = '';
       req.on('data', function (data) {
         pdata += data;
@@ -214,9 +234,29 @@ module.exports = {
         }, parseInt(wait, 10));
         /*eslint-disable consistent-return*/ return; /*eslint-enable consistent-return*/
       }).resume();
-    }).listen(this.port, '0.0.0.0', function () {
-      process.stderr.write('%%% server start %%%');
+    }).listen(this.httpPort, '0.0.0.0', (function () {
+      this.emitter.emit('start', 'http');
+    }).bind(this));
+
+    // httpsサーバーも稼働
+    var httpsOpts = {
+      secureProtocol: 'TLSv1_2_method' // TLS1.2のみ対応
+    };
+    [ 'key', 'cert' ].forEach(function (pem) {
+      httpsOpts[pem] = fs.readFileSync(
+        path.join(__dirname, 'pem/' + pem + '.pem'),
+        'utf-8'
+      );
     });
+
+    https.createServer(httpsOpts, function (req, res) {
+      res.writeHead(200, {
+        'Content-Type': 'text/plain'
+      });
+      res.end('hello, https');
+    }).listen(this.httpsPort, '0.0.0.0', (function () {
+      this.emitter.emit('start', 'https');
+    }).bind(this));
   },
 
   /**

diff --git a/test/fixtures/refresh/absolute.html b/test/fixtures/refresh/absolute.html
@@ -1 +1 @@
-<meta http-equiv="refresh" content="0;URL=http://localhost:5555/~info">
+<meta http-equiv="refresh" content="0;URL=http://localhost:55551/~info">
diff --git a/test/fixtures/refresh/ie-only.html b/test/fixtures/refresh/ie-only.html
@@ -3,7 +3,7 @@
   <head>
     <meta charset="UTF-8">
     <!--[if IE]>
-    <meta http-equiv="refresh" content="0;URL=http://localhost:5555/form/utf-8.html">
+    <meta http-equiv="refresh" content="0;URL=http://localhost:55551/form/utf-8.html">
     <![endif]-->
     <title>Refresh IE only</title>
   </head>

diff --git a/test/https.js b/test/https.js
@@ -0,0 +1,85 @@
+/*eslint-env mocha*/
+/*eslint no-invalid-this:0*/
+/*jshint -W100*/
+var assert    = require('power-assert');
+var typeOf    = require('type-of');
+var constants = require('constants');
+var helper    = require('./_helper');
+var cli       = require('../index');
+
+// オレオレ証明書のサーバーにアクセスできるようにする
+process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0;
+
+describe('https', function () {
+  it('agentOptions未設定 => TLS1.2 Onlyのサーバーに接続可能', function (done) {
+    cli.fetch(helper.url('%https%'), function (err, $, res, body) {
+      assert(! err);
+      assert(typeOf(res) === 'object');
+      assert(typeOf($) === 'function');
+      assert(typeOf(body) === 'string');
+      assert(body === 'hello, https');
+      done();
+    });
+  });
+
+  it('agentOptions: TLS1.2強制 => TLS1.2 Onlyのサーバーに接続可能', function (done) {
+    cli.set('agentOptions', {
+      secureProtocol: 'TLSv1_2_method'
+    });
+    cli.fetch(helper.url('%https%'), function (err, $, res, body) {
+      assert(! err);
+      assert(typeOf(res) === 'object');
+      assert(typeOf($) === 'function');
+      assert(typeOf(body) === 'string');
+      assert(body === 'hello, https');
+      done();
+    });
+  });
+
+  it('agentOptions: TLS1.2強制 => httpのサーバーにも接続可能', function (done) {
+    cli.set('agentOptions', {
+      secureProtocol: 'TLSv1_2_method'
+    });
+    cli.fetch(helper.url('~info'), function (err, $, res, body) {
+      assert(! err);
+      assert(typeOf(res) === 'object');
+      assert(typeOf($) === 'function');
+      assert(typeOf(body) === 'string');
+      done();
+    });
+  });
+
+  it('agentOptions: TLS1.1強制 => TLS1.2 Onlyのサーバーに接続不可', function (done) {
+    cli.set('agentOptions', {
+      secureProtocol: 'TLSv1_1_method'
+    });
+    var url = helper.url('%https%');
+    cli.fetch(url, function (err, $, res, body) {
+      assert(err.errno === 'EPROTO');
+      assert(err.code === 'EPROTO');
+      assert(err.message.indexOf('handshake failure:') !== -1);
+      assert(err.url === url);
+      assert(! res);
+      assert(! $);
+      assert(! body);
+      done();
+    });
+  });
+
+  it('agentOptions: TLS1.2無効 => TLS1.2 Onlyのサーバーに接続不可', function (done) {
+    cli.set('agentOptions', {
+      secureOptions: constants.SSL_OP_NO_TLSv1_2
+    });
+    var url = helper.url('%https%');
+    cli.fetch(url, function (err, $, res, body) {
+      assert(err.errno === 'EPROTO');
+      assert(err.code === 'EPROTO');
+      assert(err.message.indexOf('handshake failure:') !== -1);
+      assert(err.url === url);
+      assert(! res);
+      assert(! $);
+      assert(! body);
+      done();
+    });
+  });
+});
diff --git a/test/pem/cert.pem b/test/pem/cert.pem
@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDTDCCAjQCCQCIfNWNVBmJrTANBgkqhkiG9w0BAQsFADBnMQswCQYDVQQGEwJK
+UDEOMAwGA1UECAwFVG9reW8xFjAUBgNVBAcMDU11c2FzaGluby1zaGkxDDAKBgNV
+BAoMA0ZvbzEMMAoGA1UECwwDQmFyMRQwEgYDVQQDDAtmb28uYmFyLmNvbTAgFw0x
+NzExMTAwMDUyNTlaGA8yMjE3MDkyMzAwNTI1OVowZzELMAkGA1UEBhMCSlAxDjAM
+BgNVBAgMBVRva3lvMRYwFAYDVQQHDA1NdXNhc2hpbm8tc2hpMQwwCgYDVQQKDANG
+b28xDDAKBgNVBAsMA0JhcjEUMBIGA1UEAwwLZm9vLmJhci5jb20wggEiMA0GCSqG
+SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDs2GLh7pYfzD2rfvokOGud4uRJY7FuhprY
+3FL6ICRQBZVtuEZMqDIVUFz89m/VW0GFNz290Pj0UP2WHcXGUr5pfRtNbMLCer1g
+DdHDsevSVRZzbgP68EW6q3te5pti7wcso1U2dhFfGZTZNE0PoFdeJYV7bPoByAWH
+I/EPWfSOAMb8VcjZbVfVfn69OC1rc07EtVRBa2H9B2ovs03EVc0iuXkNFGzQSsXA
+3R98xUaKvimQYPtMnn5mXbN0xrJCIpVMj/HPhDlMX3ZFcSy8W1ICmeH4RT670RnD
+j/3H1qdgogKFq+n4eh7Fi3jIZ8Sh3mbQVekIsDLMS8bgeKI9TXeVAgMBAAEwDQYJ
+KoZIhvcNAQELBQADggEBAGermRUgss3JjhSL+KizeYhqvJP0+GnmsCHAzItScCUH
+ONRjoASb+XkJJbNKreaSn/O7H76sambkUPH6HYpulGnwQvNjhipcs4sEtsMduoUF
+zuOwEJDGOuDhi6mbxif8sNuN6Gh0xXPzNxzyGzS5C3sKjxT7G7Wm04FzSUC2dOhS
+89xU4r4T9v4otVpieZfHyi81Jua6eSBDaMw0IMXH3SpMdsorbR66c+hsG1eBDyyg
+w1i+v0GlB3NAPmomjdtkysjwNpW9l7NbvV3MhR/HRmMPmUp4TSWvAIF6BTgzMHjE
+Ichze9UqovoCPO881JWV+nBFQ1EQw0WS4QUSB5nm8JU=
+-----END CERTIFICATE-----
diff --git a/test/pem/key.pem b/test/pem/key.pem
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEA7Nhi4e6WH8w9q376JDhrneLkSWOxboaa2NxS+iAkUAWVbbhG
+TKgyFVBc/PZv1VtBhTc9vdD49FD9lh3FxlK+aX0bTWzCwnq9YA3Rw7Hr0lUWc24D
++vBFuqt7XuabYu8HLKNVNnYRXxmU2TRND6BXXiWFe2z6AcgFhyPxD1n0jgDG/FXI
+2W1X1X5+vTgta3NOxLVUQWth/QdqL7NNxFXNIrl5DRRs0ErFwN0ffMVGir4pkGD7
+TJ5+Zl2zdMayQiKVTI/xz4Q5TF92RXEsvFtSApnh+EU+u9EZw4/9x9anYKIChavp
++HoexYt4yGfEod5m0FXpCLAyzEvG4HiiPU13lQIDAQABAoIBAQDIBA2t48FgZSmH
+lRpGUGeB1MUZvVlwj7hhf9+LYG2KLsz89exYfIqfOVjuQGg9dG2mxPodPUehfGxL
+xCTr0aEAkSjnf/wSJXmcjs8hRzZyUG0/Wh9+Yj9g38S2ZmW/bUFPzzf9YERXXdE4
+hVS256Ag3+sUSvnvWy5f7Fh9sGg5KorpIYvJYf7dWfIu+wIHd5soJtxOGD/KLQuG
+TmKaTZtAL48bAmsghHePw7XyvsLTBtJb2ZdKGGq/QVD3h6xIu7LZY4Fp/4KIfOFA
+O8RFCSwLlQyURBhgW3+gKBrYN7ghzvvhkPojrGqWEGwW/DfO+rWxxeHNfzfnOUPL
+kJYn5SlBAoGBAPrGvxGIi7PKkIi7pET5hUdDwzDNJjxZysBpCQHMXssXIBCzRCVq
+CsUsXjV/1qhQSKghx/bJW+0QmuX5jMx/gvBMvYtKLndqlAq4VS4WBKgATZ1Bvndj
+61KuZUUg21Rd+9bUnStoHDy3n+x9hDQ5gZNshOkAh/IwrZAyPXANGYBZAoGBAPHH
+WlvnT6t3BcGkTgNHmXIMIB9hHkG9acM/KvQ4f4aHQJC4F3bCeVuBi0ZINmzuaxUq
+azrMZiItgpA8jGN6X8v6QE5/odWRY3gSX38FTTIH1+jwI09iiWOZpBRDK9WWwpAn
+X2BlZM0uoGlayi9qbrICrLq1Yn+wDD+Ov6kwh6mdAoGAZ9fY0ufaAa9FvnkFAtLY
+T7RNpW2uAZulC5vy8N2x+yMuUfwJofyRTSicMkcnmjb0fzrN1PF4sWgI3GZD2YKL
+s/nzGzSynRxzBSVjkFvpva+ydAX/WuzzSx+QK9n5OKxaVpFgK9NGrhXTkVhAYGfX
+sjZjqyBfKvjhRi6npjimcLECgYEA2wjrR08q0f+l62PaeQYocTWi9Eqbipr6cbOM
+SmvUvB9T0se0GhbcspWNg0JwbAciY65mLoJ2FIh+PAVeedCncLdqArOF/WEVZ/Xd
+Jcm7wZNxesnyczylkuHhz6l60Kkf4lCJC19QDsIq+McTXBlj50idCxi//0WSExJT
+eAdLH9ECgYEAxh/IanVVsIHed8tU4CoBbHIbyvrjjMlAlhexsrQ0gazJKvdGWGXC
+WQnCGNMZ9cnXoYP07upEpkc5hG+P+mqpU2Lpm5zbmteDGKP0e+HG9kBHToKmXVNi
+VgyNFITaOpTWERIwVSnQDjFLbIzL8VgQtUUP/bD4QJ7uKE0//TkirBM=
+-----END RSA PRIVATE KEY-----
diff --git a/testrunner.js b/testrunner.js
@@ -22,18 +22,6 @@ fs.readdirSync(testDir).filter(function (file) {
   mocha.addFile(path.join(testDir, file));
 });
 
-// Ctrl-C
-var stdin = process.stdin;
-stdin.setRawMode(true);
-stdin.resume();
-stdin.setEncoding('utf-8');
-stdin.on('data', function (key) {
-  if (key === '\u0003') {
-    process.kill(server.pid);
-    process.exit();
-  }
-});
-
 var server = spawn(process.execPath, [
   path.join(__dirname, 'test/_server.js')
 ], {
@@ -43,12 +31,25 @@ server.stdout.on('data', function (data) {
   process.stdout.write(data);
 });
 server.stderr.on('data', function (data) {
-  if (data.toString() === '%%% server start %%%') {
+  if (data.toString().trim() === '%%% server ready %%%') {
     // start mocha
-    return mocha.run(function (failures) {
+    mocha.run(function (failures) {
       process.kill(server.pid);
       process.exit(failures);
     });
+    return;
   }
   process.stderr.write(data);
 });
+
+// Ctrl-C
+var stdin = process.stdin;
+stdin.setRawMode(true);
+stdin.resume();
+stdin.setEncoding('utf-8');
+stdin.on('data', function (key) {
+  if (key === '\u0003') {
+    process.kill(server.pid);
+    process.exit();
+  }
+});