Support tunneling HTTPS requests over proxies

Thanks, @koichik!

main.js

@@ -26,6 +26,7 @@ var http = require('http')
   , Cookie = require('./vendor/cookie')
   , CookieJar = require('./vendor/cookie/jar')
   , cookieJar = new CookieJar
+  , tunnel = require('./tunnel')
   ;
 
 if (process.logging) {
@@ -133,6 +134,20 @@ Request.prototype.init = function (options) {
   }
   if (self.proxy) {
     if (typeof self.proxy == 'string') self.proxy = url.parse(self.proxy)
+
+    // do the HTTP CONNECT dance using koichik/node-tunnel
+    if (http.globalAgent && self.uri.protocol === "https:") {
+      self.tunnel = true
+      var tunnelFn = self.proxy.protocol === "http:"
+                   ? tunnel.httpsOverHttp : tunnel.httpsOverHttps
+
+      var tunnelOptions = { proxy: { host: self.proxy.hostname
+                                   , port: +self.proxy.port }
+                          , ca: this.ca }
+
+      self.agent = tunnelFn(tunnelOptions)
+      self.tunnel = true
+    }
   }
 
   self._redirectsFollowed = self._redirectsFollowed || 0
@@ -163,7 +178,7 @@ Request.prototype.init = function (options) {
     else if (self.uri.protocol == 'https:') {self.uri.port = 443}
   }
 
-  if (self.proxy) {
+  if (self.proxy && !self.tunnel) {
     self.port = self.proxy.port
     self.host = self.proxy.hostname
   } else {
@@ -207,7 +222,7 @@ Request.prototype.init = function (options) {
   if (self.uri.auth && !self.headers.authorization) {
     self.headers.authorization = "Basic " + toBase64(self.uri.auth.split(':').map(function(item){ return qs.unescape(item)}).join(':'))
   }
-  if (self.proxy && self.proxy.auth && !self.headers['proxy-authorization']) {
+  if (self.proxy && self.proxy.auth && !self.headers['proxy-authorization'] && !self.tunnel) {
     self.headers['proxy-authorization'] = "Basic " + toBase64(self.proxy.auth.split(':').map(function(item){ return qs.unescape(item)}).join(':'))
   }
 
@@ -221,7 +236,7 @@ Request.prototype.init = function (options) {
 
   if (self.path.length === 0) self.path = '/'
 
-  if (self.proxy) self.path = (self.uri.protocol + '//' + self.uri.host + self.path)
+  if (self.proxy && !self.tunnel) self.path = (self.uri.protocol + '//' + self.uri.host + self.path)
 
   if (options.json) {
     self.json(options.json)
@@ -250,7 +265,7 @@ Request.prototype.init = function (options) {
     }
   }
 
-  var protocol = self.proxy ? self.proxy.protocol : self.uri.protocol
+  var protocol = self.proxy && !self.tunnel ? self.proxy.protocol : self.uri.protocol
     , defaultModules = {'http:':http, 'https:':https}
     , httpModules = self.httpModules || {}
     ;

tests/run.js

@@ -18,6 +18,7 @@ var tests = [
   , 'test-qs.js'
   , 'test-redirect.js'
   , 'test-timeout.js'
+  , 'test-tunnel.js'
 ] 
 
 var next = function () {

tests/squid.conf

@@ -0,0 +1,77 @@
+#
+# Recommended minimum configuration:
+#
+acl manager proto cache_object
+acl localhost src 127.0.0.1/32 ::1
+acl to_localhost dst 127.0.0.0/8 0.0.0.0/32 ::1
+
+# Example rule allowing access from your local networks.
+# Adapt to list your (internal) IP networks from where browsing
+# should be allowed
+acl localnet src 10.0.0.0/8	# RFC1918 possible internal network
+acl localnet src 172.16.0.0/12	# RFC1918 possible internal network
+acl localnet src 192.168.0.0/16	# RFC1918 possible internal network
+acl localnet src fc00::/7       # RFC 4193 local private network range
+acl localnet src fe80::/10      # RFC 4291 link-local (directly plugged) machines
+
+acl SSL_ports port 443
+acl Safe_ports port 80		# http
+acl Safe_ports port 21		# ftp
+acl Safe_ports port 443		# https
+acl Safe_ports port 70		# gopher
+acl Safe_ports port 210		# wais
+acl Safe_ports port 1025-65535	# unregistered ports
+acl Safe_ports port 280		# http-mgmt
+acl Safe_ports port 488		# gss-http
+acl Safe_ports port 591		# filemaker
+acl Safe_ports port 777		# multiling http
+acl CONNECT method CONNECT
+
+#
+# Recommended minimum Access Permission configuration:
+#
+# Only allow cachemgr access from localhost
+http_access allow manager localhost
+http_access deny manager
+
+# Deny requests to certain unsafe ports
+http_access deny !Safe_ports
+
+# Deny CONNECT to other than secure SSL ports
+#http_access deny CONNECT !SSL_ports
+
+# We strongly recommend the following be uncommented to protect innocent
+# web applications running on the proxy server who think the only
+# one who can access services on "localhost" is a local user
+#http_access deny to_localhost
+
+#
+# INSERT YOUR OWN RULE(S) HERE TO ALLOW ACCESS FROM YOUR CLIENTS
+#
+
+# Example rule allowing access from your local networks.
+# Adapt localnet in the ACL section to list your (internal) IP networks
+# from where browsing should be allowed
+http_access allow localnet
+http_access allow localhost
+
+# And finally deny all other access to this proxy
+http_access deny all
+
+# Squid normally listens to port 3128
+http_port 3128
+
+# We recommend you to use at least the following line.
+hierarchy_stoplist cgi-bin ?
+
+# Uncomment and adjust the following to add a disk cache directory.
+#cache_dir ufs /usr/local/var/cache 100 16 256
+
+# Leave coredumps in the first cache dir
+coredump_dir /usr/local/var/cache
+
+# Add any of your own refresh_pattern entries above these.
+refresh_pattern ^ftp:		1440	20%	10080
+refresh_pattern ^gopher:	1440	0%	1440
+refresh_pattern -i (/cgi-bin/|\?) 0	0%	0
+refresh_pattern .		0	20%	4320

tests/ssl/npm-ca.crt

@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIIChzCCAfACCQDauvz/KHp8ejANBgkqhkiG9w0BAQUFADCBhzELMAkGA1UEBhMC
+VVMxCzAJBgNVBAgTAkNBMRAwDgYDVQQHEwdPYWtsYW5kMQwwCgYDVQQKEwNucG0x
+IjAgBgNVBAsTGW5wbSBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxDjAMBgNVBAMTBW5w
+bUNBMRcwFQYJKoZIhvcNAQkBFghpQGl6cy5tZTAeFw0xMTA5MDUwMTQ3MTdaFw0y
+MTA5MDIwMTQ3MTdaMIGHMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExEDAOBgNV
+BAcTB09ha2xhbmQxDDAKBgNVBAoTA25wbTEiMCAGA1UECxMZbnBtIENlcnRpZmlj
+YXRlIEF1dGhvcml0eTEOMAwGA1UEAxMFbnBtQ0ExFzAVBgkqhkiG9w0BCQEWCGlA
+aXpzLm1lMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDLI4tIqPpRW+ACw9GE
+OgBlJZwK5f8nnKCLK629Pv5yJpQKs3DENExAyOgDcyaF0HD0zk8zTp+ZsLaNdKOz
+Gn2U181KGprGKAXP6DU6ByOJDWmTlY6+Ad1laYT0m64fERSpHw/hjD3D+iX4aMOl
+y0HdbT5m1ZGh6SJz3ZqxavhHLQIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAC4ySDbC
+l7W1WpLmtLGEQ/yuMLUf6Jy/vr+CRp4h+UzL+IQpCv8FfxsYE7dhf/bmWTEupBkv
+yNL18lipt2jSvR3v6oAHAReotvdjqhxddpe5Holns6EQd1/xEZ7sB1YhQKJtvUrl
+ZNufy1Jf1r0ldEGeA+0ISck7s+xSh9rQD2Op
+-----END CERTIFICATE-----

tests/test-tunnel.js

@@ -0,0 +1,61 @@
+// test that we can tunnel a https request over an http proxy
+// keeping all the CA and whatnot intact.
+//
+// Note: this requires that squid is installed.
+// If the proxy fails to start, we'll just log a warning and assume success.
+
+var server = require('./server')
+  , assert = require('assert')
+  , request = require('../main.js')
+  , fs = require('fs')
+  , path = require('path')
+  , caFile = path.resolve(__dirname, 'ssl/npm-ca.crt')
+  , ca = fs.readFileSync(caFile)
+  , child_process = require('child_process')
+  , sqConf = path.resolve(__dirname, 'squid.conf')
+  , sqArgs = ['-f', sqConf, '-N', '-d', '5']
+  , proxy = 'http://localhost:3128'
+  , hadError = null
+
+var squid = child_process.spawn('squid', sqArgs);
+var ready = false
+
+squid.stderr.on('data', function (c) {
+  console.error('SQUIDERR ' + c.toString().trim().split('\n')
+               .join('\nSQUIDERR '))
+  ready = c.toString().match(/ready to serve requests/i)
+})
+
+squid.stdout.on('data', function (c) {
+  console.error('SQUIDOUT ' + c.toString().trim().split('\n')
+               .join('\nSQUIDOUT '))
+})
+
+squid.on('exit', function (c) {
+  console.error('exit '+c)
+  if (c && !ready) {
+    console.error('squid must be installed to run this test.')
+    c = null
+    hadError = null
+    process.exit(0)
+    return
+  }
+
+  if (c) {
+    hadError = hadError || new Error('Squid exited with '+c)
+  }
+  if (hadError) throw hadError
+})
+
+setTimeout(function F () {
+  if (!ready) return setTimeout(F, 100)
+  request({ uri: 'https://registry.npmjs.org/request/'
+          , proxy: 'http://localhost:3128'
+          , ca: ca
+          , json: true }, function (er, body) {
+    hadError = er
+    console.log(er || typeof body)
+    if (!er) console.log("ok")
+    squid.kill('SIGKILL')
+  })
+}, 100)