From 564048dc2984c45884e5eb58c903f9a0ebe29ab4 Mon Sep 17 00:00:00 2001 From: Hackzzila Date: Tue, 24 Apr 2018 19:37:43 -0500 Subject: [PATCH] http,https,tls: switch to WHATWG URL parser This switches the url parser from `url.parse()` to the WHATWG URL parser while keeping `url.parse()` as fallback. Also add tests for invalid url deprecations and correct hostname checks. PR-URL: https://github.com/nodejs/node/pull/20270 Fixes: https://github.com/nodejs/node/issues/19468 Reviewed-By: Luigi Pinca Reviewed-By: James M Snell Reviewed-By: Matteo Collina Reviewed-By: Tiancheng "Timothy" Gu Reviewed-By: Daijiro Wachi Reviewed-By: Trivikram Kamat Reviewed-By: Ruben Bridgewater --- doc/api/deprecations.md | 17 ++++++++++ doc/api/errors.md | 5 --- doc/api/http.md | 3 +- doc/api/https.md | 6 ++-- lib/_http_client.js | 22 ++++++++++--- lib/https.js | 22 ++++++++++--- lib/internal/errors.js | 1 - lib/tls.js | 18 +++++++++- test/parallel/test-http-correct-hostname.js | 28 ++++++++++++++++ test/parallel/test-http-deprecated-urls.js | 33 +++++++++++++++++++ test/parallel/test-http-invalid-urls.js | 2 +- test/parallel/test-internal-errors.js | 5 --- .../test-tls-check-server-identity.js | 14 ++++++++ 13 files changed, 149 insertions(+), 27 deletions(-) create mode 100644 test/parallel/test-http-correct-hostname.js create mode 100644 test/parallel/test-http-deprecated-urls.js diff --git a/doc/api/deprecations.md b/doc/api/deprecations.md index 1c3c488c8d6497..5d4a8714430171 100644 --- a/doc/api/deprecations.md +++ b/doc/api/deprecations.md @@ -994,6 +994,18 @@ because it also made sense to interpret the value as the number of bytes read by the engine, but is inconsistent with other streams in Node.js that expose values under these names. + +### DEP0109: http, https, and tls support for invalid URLs + +Type: Runtime + +Some previously supported (but strictly invalid) URLs were accepted through the +[`http.request()`][], [`http.get()`][], [`https.request()`][], +[`https.get()`][], and [`tls.checkServerIdentity()`][] APIs because those were +accepted by the legacy `url.parse()` API. The mentioned APIs now use the WHATWG +URL parser that requires strictly valid URLs. Passing an invalid URL is +deprecated and support will be removed in the future. + [`--pending-deprecation`]: cli.html#cli_pending_deprecation [`Buffer.allocUnsafeSlow(size)`]: buffer.html#buffer_class_method_buffer_allocunsafeslow_size [`Buffer.from(array)`]: buffer.html#buffer_class_method_buffer_from_array @@ -1035,6 +1047,10 @@ expose values under these names. [`fs.read()`]: fs.html#fs_fs_read_fd_buffer_offset_length_position_callback [`fs.readSync()`]: fs.html#fs_fs_readsync_fd_buffer_offset_length_position [`fs.stat()`]: fs.html#fs_fs_stat_path_callback +[`http.get()`]: http.html#http_http_get_options_callback +[`http.request()`]: http.html#http_http_request_options_callback +[`https.get()`]: https.html#https_https_get_options_callback +[`https.request()`]: https.html#https_https_request_options_callback [`os.networkInterfaces`]: os.html#os_os_networkinterfaces [`os.tmpdir()`]: os.html#os_os_tmpdir [`process.env`]: process.html#process_process_env @@ -1046,6 +1062,7 @@ expose values under these names. [`tls.SecureContext`]: tls.html#tls_tls_createsecurecontext_options [`tls.SecurePair`]: tls.html#tls_class_securepair [`tls.TLSSocket`]: tls.html#tls_class_tls_tlssocket +[`tls.checkServerIdentity()`]: tls.html#tls_tls_checkserveridentity_host_cert [`tls.createSecureContext()`]: tls.html#tls_tls_createsecurecontext_options [`util._extend()`]: util.html#util_util_extend_target_source [`util.debug()`]: util.html#util_util_debug_string diff --git a/doc/api/errors.md b/doc/api/errors.md index 2599573a61f017..b36cd241bce745 100644 --- a/doc/api/errors.md +++ b/doc/api/errors.md @@ -1111,11 +1111,6 @@ Invalid characters were detected in headers. A cursor on a given stream cannot be moved to a specified row without a specified column. - -### ERR_INVALID_DOMAIN_NAME - -`hostname` can not be parsed from a provided URL. - ### ERR_INVALID_FD diff --git a/doc/api/http.md b/doc/api/http.md index 6e20cd5c4ec6af..f50199dfaf9d9c 100644 --- a/doc/api/http.md +++ b/doc/api/http.md @@ -1900,7 +1900,7 @@ Node.js maintains several connections per server to make HTTP requests. This function allows one to transparently issue requests. `options` can be an object, a string, or a [`URL`][] object. If `options` is a -string, it is automatically parsed with [`url.parse()`][]. If it is a [`URL`][] +string, it is automatically parsed with [`new URL()`][]. If it is a [`URL`][] object, it will be automatically converted to an ordinary `options` object. The optional `callback` parameter will be added as a one-time listener for @@ -2050,6 +2050,7 @@ not abort the request or do anything besides add a `'timeout'` event. [`net.Server`]: net.html#net_class_net_server [`net.Socket`]: net.html#net_class_net_socket [`net.createConnection()`]: net.html#net_net_createconnection_options_connectlistener +[`new URL()`]: url.html#url_constructor_new_url_input_base [`removeHeader(name)`]: #http_request_removeheader_name [`request.end()`]: #http_request_end_data_encoding_callback [`request.getHeader()`]: #http_request_getheader_name diff --git a/doc/api/https.md b/doc/api/https.md index 3e7c2d95ade0d0..89229ca621d683 100644 --- a/doc/api/https.md +++ b/doc/api/https.md @@ -119,7 +119,7 @@ changes: Like [`http.get()`][] but for HTTPS. `options` can be an object, a string, or a [`URL`][] object. If `options` is a -string, it is automatically parsed with [`url.parse()`][]. If it is a [`URL`][] +string, it is automatically parsed with [`new URL()`][]. If it is a [`URL`][] object, it will be automatically converted to an ordinary `options` object. Example: @@ -173,7 +173,7 @@ The following additional `options` from [`tls.connect()`][] are also accepted: `secureOptions`, `secureProtocol`, `servername`, `sessionIdContext` `options` can be an object, a string, or a [`URL`][] object. If `options` is a -string, it is automatically parsed with [`url.parse()`][]. If it is a [`URL`][] +string, it is automatically parsed with [`new URL()`][]. If it is a [`URL`][] object, it will be automatically converted to an ordinary `options` object. Example: @@ -358,8 +358,8 @@ headers: max-age=0; pin-sha256="WoiWRyIOVNa9ihaBciRSC7XHjliYS9VwUGOIud4PB18="; p [`https.Agent`]: #https_class_https_agent [`https.request()`]: #https_https_request_options_callback [`net.Server`]: net.html#net_class_net_server +[`new URL()`]: url.html#url_constructor_new_url_input_base [`server.listen()`]: net.html#net_server_listen [`tls.connect()`]: tls.html#tls_tls_connect_options_callback [`tls.createSecureContext()`]: tls.html#tls_tls_createsecurecontext_options [`tls.createServer()`]: tls.html#tls_tls_createserver_options_secureconnectionlistener -[`url.parse()`]: url.html#url_url_parse_urlstring_parsequerystring_slashesdenotehost diff --git a/lib/_http_client.js b/lib/_http_client.js index 04880182dace91..1985c617bec537 100644 --- a/lib/_http_client.js +++ b/lib/_http_client.js @@ -37,12 +37,11 @@ const { OutgoingMessage } = require('_http_outgoing'); const Agent = require('_http_agent'); const { Buffer } = require('buffer'); const { defaultTriggerAsyncIdScope } = require('internal/async_hooks'); -const { urlToOptions, searchParamsSymbol } = require('internal/url'); +const { URL, urlToOptions, searchParamsSymbol } = require('internal/url'); const { outHeadersKey, ondrain } = require('internal/http'); const { ERR_HTTP_HEADERS_SENT, ERR_INVALID_ARG_TYPE, - ERR_INVALID_DOMAIN_NAME, ERR_INVALID_HTTP_TOKEN, ERR_INVALID_PROTOCOL, ERR_UNESCAPED_CHARACTERS @@ -60,13 +59,26 @@ function validateHost(host, name) { return host; } +let urlWarningEmitted = false; function ClientRequest(options, cb) { OutgoingMessage.call(this); if (typeof options === 'string') { - options = url.parse(options); - if (!options.hostname) { - throw new ERR_INVALID_DOMAIN_NAME(); + const urlStr = options; + try { + options = urlToOptions(new URL(urlStr)); + } catch (err) { + options = url.parse(urlStr); + if (!options.hostname) { + throw err; + } + if (!urlWarningEmitted && !process.noDeprecation) { + urlWarningEmitted = true; + process.emitWarning( + `The provided URL ${urlStr} is not a valid URL, and is supported ` + + 'in the http module solely for compatibility.', + 'DeprecationWarning', 'DEP0109'); + } } } else if (options && options[searchParamsSymbol] && options[searchParamsSymbol][searchParamsSymbol]) { diff --git a/lib/https.js b/lib/https.js index 4cca2fb9eeea88..ead7a2bb927dad 100644 --- a/lib/https.js +++ b/lib/https.js @@ -34,8 +34,7 @@ const { const { ClientRequest } = require('_http_client'); const { inherits } = util; const debug = util.debuglog('https'); -const { urlToOptions, searchParamsSymbol } = require('internal/url'); -const { ERR_INVALID_DOMAIN_NAME } = require('internal/errors').codes; +const { URL, urlToOptions, searchParamsSymbol } = require('internal/url'); const { IncomingMessage, ServerResponse } = require('http'); const { kIncomingMessage } = require('_http_common'); const { kServerResponse } = require('_http_server'); @@ -254,11 +253,24 @@ Agent.prototype._evictSession = function _evictSession(key) { const globalAgent = new Agent(); +let urlWarningEmitted = false; function request(options, cb) { if (typeof options === 'string') { - options = url.parse(options); - if (!options.hostname) { - throw new ERR_INVALID_DOMAIN_NAME(); + const urlStr = options; + try { + options = urlToOptions(new URL(urlStr)); + } catch (err) { + options = url.parse(urlStr); + if (!options.hostname) { + throw err; + } + if (!urlWarningEmitted && !process.noDeprecation) { + urlWarningEmitted = true; + process.emitWarning( + `The provided URL ${urlStr} is not a valid URL, and is supported ` + + 'in the https module solely for compatibility.', + 'DeprecationWarning', 'DEP0109'); + } } } else if (options && options[searchParamsSymbol] && options[searchParamsSymbol][searchParamsSymbol]) { diff --git a/lib/internal/errors.js b/lib/internal/errors.js index 882e40af88ae75..51acc211d9a1c7 100644 --- a/lib/internal/errors.js +++ b/lib/internal/errors.js @@ -881,7 +881,6 @@ E('ERR_INVALID_CALLBACK', 'Callback must be a function', TypeError); E('ERR_INVALID_CHAR', invalidChar, TypeError); E('ERR_INVALID_CURSOR_POS', 'Cannot set cursor row without setting its column', TypeError); -E('ERR_INVALID_DOMAIN_NAME', 'Unable to determine the domain name', TypeError); E('ERR_INVALID_FD', '"fd" must be a positive integer: %s', RangeError); E('ERR_INVALID_FD_TYPE', 'Unsupported fd type: %s', TypeError); diff --git a/lib/tls.js b/lib/tls.js index 1e444d5d8898c2..28cd302674b04c 100644 --- a/lib/tls.js +++ b/lib/tls.js @@ -32,6 +32,7 @@ const url = require('url'); const binding = process.binding('crypto'); const { Buffer } = require('buffer'); const EventEmitter = require('events'); +const { URL } = require('internal/url'); const DuplexPair = require('internal/streams/duplexpair'); const { canonicalizeIP } = process.binding('cares_wrap'); const _tls_common = require('_tls_common'); @@ -169,6 +170,7 @@ function check(hostParts, pattern, wildcards) { return true; } +let urlWarningEmitted = false; exports.checkServerIdentity = function checkServerIdentity(host, cert) { const subject = cert.subject; const altNames = cert.subjectaltname; @@ -183,7 +185,21 @@ exports.checkServerIdentity = function checkServerIdentity(host, cert) { if (name.startsWith('DNS:')) { dnsNames.push(name.slice(4)); } else if (name.startsWith('URI:')) { - const uri = url.parse(name.slice(4)); + let uri; + try { + uri = new URL(name.slice(4)); + } catch (err) { + uri = url.parse(name.slice(4)); + if (!urlWarningEmitted && !process.noDeprecation) { + urlWarningEmitted = true; + process.emitWarning( + `The URI ${name.slice(4)} found in cert.subjectaltname ` + + 'is not a valid URI, and is supported in the tls module ' + + 'solely for compatibility.', + 'DeprecationWarning', 'DEP0109'); + } + } + uriNames.push(uri.hostname); // TODO(bnoordhuis) Also use scheme. } else if (name.startsWith('IP Address:')) { ips.push(canonicalizeIP(name.slice(11))); diff --git a/test/parallel/test-http-correct-hostname.js b/test/parallel/test-http-correct-hostname.js new file mode 100644 index 00000000000000..b4279c1da4339f --- /dev/null +++ b/test/parallel/test-http-correct-hostname.js @@ -0,0 +1,28 @@ +/* eslint-disable node-core/crypto-check */ +// Flags: --expose-internals +'use strict'; + +const common = require('../common'); +const assert = require('assert'); + +const { outHeadersKey } = require('internal/http'); + +const http = require('http'); +const modules = { http }; + +if (common.hasCrypto) { + const https = require('https'); + modules.https = https; +} + +Object.keys(modules).forEach((module) => { + const doNotCall = common.mustNotCall( + `${module}.request should not connect to ${module}://example.com%60x.example.com` + ); + const req = modules[module].request(`${module}://example.com%60x.example.com`, doNotCall); + assert.deepStrictEqual(req[outHeadersKey].host, [ + 'Host', + 'example.com`x.example.com', + ]); + req.abort(); +}); diff --git a/test/parallel/test-http-deprecated-urls.js b/test/parallel/test-http-deprecated-urls.js new file mode 100644 index 00000000000000..5bb9f66ebe7126 --- /dev/null +++ b/test/parallel/test-http-deprecated-urls.js @@ -0,0 +1,33 @@ +/* eslint-disable node-core/crypto-check */ + +'use strict'; + +const common = require('../common'); + +const http = require('http'); +const modules = { http }; + +const deprecations = [ + ['The provided URL http://[www.nodejs.org] is not a valid URL, and is supported ' + + 'in the http module solely for compatibility.', + 'DEP0109'], +]; + +if (common.hasCrypto) { + const https = require('https'); + modules.https = https; + deprecations.push( + ['The provided URL https://[www.nodejs.org] is not a valid URL, and is supported ' + + 'in the https module solely for compatibility.', + 'DEP0109'], + ); +} + +common.expectWarning('DeprecationWarning', deprecations); + +Object.keys(modules).forEach((module) => { + const doNotCall = common.mustNotCall( + `${module}.request should not connect to ${module}://[www.nodejs.org]` + ); + modules[module].request(`${module}://[www.nodejs.org]`, doNotCall).abort(); +}); diff --git a/test/parallel/test-http-invalid-urls.js b/test/parallel/test-http-invalid-urls.js index 0ddd72b374694c..7c9f67c4734810 100644 --- a/test/parallel/test-http-invalid-urls.js +++ b/test/parallel/test-http-invalid-urls.js @@ -21,7 +21,7 @@ function test(host) { const throws = () => { modules[module][fn](host, doNotCall); }; common.expectsError(throws, { type: TypeError, - code: 'ERR_INVALID_DOMAIN_NAME' + code: 'ERR_INVALID_URL' }); }); }); diff --git a/test/parallel/test-internal-errors.js b/test/parallel/test-internal-errors.js index cd2028c0f29b95..f3ca3c99ac87af 100644 --- a/test/parallel/test-internal-errors.js +++ b/test/parallel/test-internal-errors.js @@ -148,11 +148,6 @@ assert.strictEqual( 'Cannot render headers after they are sent to the client' ); -assert.strictEqual( - errors.message('ERR_INVALID_DOMAIN_NAME'), - 'Unable to determine the domain name' -); - assert.strictEqual( errors.message('ERR_INVALID_HTTP_TOKEN', ['Method', 'foo']), 'Method must be a valid HTTP token ["foo"]' diff --git a/test/parallel/test-tls-check-server-identity.js b/test/parallel/test-tls-check-server-identity.js index 1623e70a2af2ec..fe27770c1ebc84 100644 --- a/test/parallel/test-tls-check-server-identity.js +++ b/test/parallel/test-tls-check-server-identity.js @@ -30,6 +30,13 @@ const util = require('util'); const tls = require('tls'); +common.expectWarning('DeprecationWarning', [ + ['The URI http://[a.b.a.com]/ found in cert.subjectaltname ' + + 'is not a valid URI, and is supported in the tls module ' + + 'solely for compatibility.', + 'DEP0109'], +]); + const tests = [ // False-y values. { @@ -218,6 +225,13 @@ const tests = [ error: 'Host: a.b.a.com. is not in the cert\'s altnames: ' + 'URI:http://*.b.a.com/' }, + // Invalid URI + { + host: 'a.b.a.com', cert: { + subjectaltname: 'URI:http://[a.b.a.com]/', + subject: {} + } + }, // IP addresses { host: 'a.b.a.com', cert: {