Add http2 support.

itteco · Jun 3, 2019 · ced38bc · ced38bc
1 parent 8f48255
commit ced38bc
Show file tree

Hide file tree

Showing 9 changed files with 786 additions and 13 deletions.
diff --git a/config.js b/config.js
@@ -20,6 +20,7 @@
         relativeStaticUrl: "/s",
         DEBUG: false,
 
+        SPDY_AGENT_DEFAULT_PORT: 443,
         WHITELIST_URL: 'https://iframely.com/qa/whitelist.json',
         WHITELIST_URL_RELOAD_PERIOD: 60 * 60 * 1000,  // will reload WL every hour, if no local files are found in /whitelist folder
 

diff --git a/lib/core.js b/lib/core.js
@@ -2,6 +2,7 @@
 
     var _ = require('underscore'),
         request = require('request'),
+        spdy = require('spdy'),
         urlLib = require('url');
 
     var pluginUtils = require('./loader/utils'),
@@ -1069,7 +1070,7 @@
                 var link2 = canonical.replace(/\/+$/, '');
 
                 if (link1 === link2 && link.rel.indexOf(CONFIG.R.oembed) == -1) {
-                    // allow the canonical links for oEmbeds, as such mistakes are usually made for OG and Twitter: 
+                    // allow the canonical links for oEmbeds, as such mistakes are usually made for OG and Twitter:
                     // if publisher has oEmbed, he is most likely to have the valid embed codes
                     link.error = "Removed canonical link";
                 }
@@ -1408,6 +1409,12 @@
             requestWrapper(options_from_plugin, options, callback);
         }
 
+        options.agent = spdy.createAgent({
+            host: urlLib.parse(uri).hostname,
+            port: CONFIG.SPDY_AGENT_DEFAULT_PORT,
+            rejectUnauthorized: false
+        });
+
         var requiredPlugins = pluginsSet.initialPlugins,
             pluginsUrlMatches = pluginsSet.pluginsUrlMatches,
             usedDomains = pluginsSet.usedDomains,
@@ -1621,4 +1628,4 @@
 
     exports.getOembed = oembedUtils.getOembed;
 
-})(exports);
+})(exports);
diff --git a/lib/utils.js b/lib/utils.js
@@ -1,5 +1,7 @@
 var events = require('events');
 var request = require('request');
+var spdy = require('spdy');
+var URL = require('url');
 var zlib = require('zlib');
 var iconv = require('iconv-lite');
 var async = require('async');
@@ -91,8 +93,15 @@ var getUrl = exports.getUrl = function(url, options) {
 
             var supportGzip = !process.version.match(/^v0\.8/);
 
+            var agent = spdy.createAgent({
+                host: URL.parse(url).hostname,
+                port: CONFIG.SPDY_AGENT_DEFAULT_PORT,
+                rejectUnauthorized: false
+            });
+
             var r = request(prepareRequestOptions({
                 uri: url,
+                agent: agent,
                 method: 'GET',
                 headers: {
                     'User-Agent': options.user_agent || CONFIG.USER_AGENT,
@@ -167,9 +176,15 @@ var getHead = function(url, options) {
 
     process.nextTick(function() {
         try {
+            var agent = spdy.createAgent({
+                host: URL.parse(url).hostname,
+                port: CONFIG.SPDY_AGENT_DEFAULT_PORT,
+                rejectUnauthorized: false
+            });
 
             var r = request(prepareRequestOptions({
                 uri: url,
+                agent: agent,
                 method: 'HEAD',
                 headers: {
                     'User-Agent': CONFIG.USER_AGENT,
@@ -650,8 +665,15 @@ exports.sendLogToWhitelist = function(uri, meta, oembed, oembedLinks, whitelistR
             data.oembed = oembedHref;
         }
 
+        var agent = spdy.createAgent({
+            host: URL.parse(CONFIG.WHITELIST_LOG_URL).hostname,
+            port: CONFIG.SPDY_AGENT_DEFAULT_PORT,
+            rejectUnauthorized: false
+        });
+
         request({
             uri: CONFIG.WHITELIST_LOG_URL,
+            agent: agent,
             method: 'GET',
             qs: data
         })
@@ -823,8 +845,15 @@ var getUriStatus = function(uri, options, cb) {
 
     try {
 
+        var agent = spdy.createAgent({
+            host: URL.parse(uri).hostname,
+            port: CONFIG.SPDY_AGENT_DEFAULT_PORT,
+            rejectUnauthorized: false
+        });
+
         var r = request(prepareRequestOptions({
             uri: uri,
+            agent: agent,
             method: 'GET',
             headers: {
                 'User-Agent': CONFIG.USER_AGENT

diff --git a/lib/whitelist.js b/lib/whitelist.js
@@ -6,6 +6,8 @@
         crypto = require('crypto'),
         _ = require('underscore'),
         request = require('request'),
+        spdy = require('spdy'),
+        URL = require('url'),
         utils = require('./utils'),
         logging = require('../logging');
 
@@ -328,8 +330,15 @@
 
             logging.log("Loading whitelist from " + CONFIG.WHITELIST_URL);
 
+            var agent = spdy.createAgent({
+                host: URL.parse(CONFIG.WHITELIST_URL).hostname,
+                port: CONFIG.SPDY_AGENT_DEFAULT_PORT,
+                rejectUnauthorized: false
+            });
+
             var options = {
                 uri: CONFIG.WHITELIST_URL,
+                agent: agent,
                 json: true,
                 qs: {
                     domain: CONFIG.baseAppUrl && CONFIG.baseAppUrl.replace(/.+\/\//, ''),

diff --git a/modules/tests-ui/utils.js b/modules/tests-ui/utils.js
@@ -1,6 +1,7 @@
 var _ = require('underscore');
 var FeedParser = require('feedparser');
 var request = require('request');
+var spdy = require('spdy');
 var async = require('async');
 var url = require('url');
 
@@ -74,7 +75,13 @@ var fetchFeedUrls = exports.fetchFeedUrls = function(feedUrl, options, cb) {
         cb(error, urls);
     };
 
-    request(feedUrl)
+    var agent = spdy.createAgent({
+        host: url.parse(feedUrl, true).hostname,
+        port: CONFIG.SPDY_AGENT_DEFAULT_PORT,
+        rejectUnauthorized: false
+    });
+
+    request(feedUrl, {agent: agent})
         .pipe(new FeedParser({addmeta: false}))
         .on('error', function(error) {
             _cb(error);

diff --git a/package.json b/package.json
@@ -36,11 +36,13 @@
         "moment": "2.19.3",
         "node-cache": "1.*",
         "parse-iso-duration": "1.0.0",
+        "patch-package": "^6.1.2",
         "readabilitySAX": "1.6.1",
         "redis": "2.7.1",
         "request": "^2.88.0",
         "sax": "1.2.2",
         "send": "0.16.1",
+        "spdy": "^4.0.0",
         "underscore": "1.8.3"
     },
     "devDependencies": {
@@ -55,6 +57,7 @@
     "iframely-proxy-plugins": true,
     "main": "./lib/core",
     "scripts": {
+        "prepare": "patch-package",
         "test": "vows test/main.js --isolate --spec && npm run test-e2e",
         "test-e2e": "NODE_ENV=test PORT=8080 mocha --exit test/e2e.js"
     },

diff --git a/patches/spdy-transport+3.0.0.patch b/patches/spdy-transport+3.0.0.patch
@@ -0,0 +1,33 @@
+diff --git a/node_modules/spdy-transport/lib/spdy-transport/protocol/base/utils.js b/node_modules/spdy-transport/lib/spdy-transport/protocol/base/utils.js
+index 72a380f..042199e 100644
+--- a/node_modules/spdy-transport/lib/spdy-transport/protocol/base/utils.js
++++ b/node_modules/spdy-transport/lib/spdy-transport/protocol/base/utils.js
+@@ -45,13 +45,13 @@ exports.addHeaderLine = function addHeaderLine (field, value, dest) {
+
+   switch (field) {
+     // Array headers:
+-    case 'set-cookie':
+-      if (dest[field] !== undefined) {
+-        dest[field].push(value)
+-      } else {
+-        dest[field] = [ value ]
+-      }
+-      break
++    // case 'set-cookie':
++    //   if (dest[field] !== undefined) {
++    //     dest[field].push(value)
++    //   } else {
++    //     dest[field] = [ value ]
++    //   }
++    //   break
+
+     /* eslint-disable max-len */
+     // list is taken from:
+@@ -74,6 +74,7 @@ exports.addHeaderLine = function addHeaderLine (field, value, dest) {
+       }
+       break
+
++    case 'set-cookie':
+     case 'cookie':
+       // make semicolon-separated list
+       if (dest[field] !== undefined) {
diff --git a/server.js b/server.js
@@ -1,3 +1,4 @@
+var spdy = require('spdy');
 var sysUtils = require('./utils');
 var app = require('./app');
 
@@ -7,7 +8,7 @@ var server = app.listen(process.env.PORT || CONFIG.port, process.env.HOST || CON
 });
 
 if (CONFIG.ssl) {
-    require('https').createServer(CONFIG.ssl, app).listen(CONFIG.ssl.port);
+    spdy.createServer(CONFIG.ssl, app).listen(CONFIG.ssl.port);
 }
 
 console.log('');
@@ -24,4 +25,4 @@ if (!CONFIG.DEBUG) {
     });
 }
 
-module.exports = server;
+module.exports = server;