Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

extend http request node #3174

Closed
wants to merge 10 commits into from
Closed

extend http request node #3174

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
e0744ad
Merge pull request #1 from mailsvb/mailsvb-typo-http-request
mailsvb Oct 7, 2021
2f4621f
Merge branch 'node-red:master' into master
mailsvb Oct 7, 2021
914cf06
peer certificate information in http request node
mailsvb Oct 7, 2021
5041e3e
add description to node
mailsvb Oct 7, 2021
463431c
german translations for node description
mailsvb Oct 7, 2021
a30beec
Update tests for peerCertificate
mailsvb Oct 8, 2021
3991e90
Merge branch 'dev' into mailsvb-extend-http-request-node
mailsvb Oct 8, 2021
a599dbc
handle redirects and make adding certificates configurable
mailsvb Oct 15, 2021
2156274
Merge branch 'mailsvb-extend-http-request-node' of https://github.com…
mailsvb Oct 15, 2021
1d60e5d
use separate agent to ensure certificate retrival
mailsvb Oct 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
6 changes: 6 additions & 0 deletions packages/node_modules/@node-red/nodes/core/network/21-httprequest.html
View file
Open in desktop
Expand Up @@ -86,6 +86,11 @@
</div>
</div>

<div class="form-row">
<input type="checkbox" id="node-input-peerCertificateOutput" style="display: inline-block; width: auto; vertical-align: top;">
<label for="node-input-peerCertificateOutput" style="width: auto" data-i18n="httpin.peerCertOutput"></label>
</div>

<div class="form-row">
<input type="checkbox" id="node-input-senderr" style="display: inline-block; width: auto; vertical-align: top;">
<label for="node-input-senderr" style="width: auto" data-i18n="httpin.senderr"></label>
Expand Down Expand Up @@ -119,6 +124,7 @@
url:{value:"",validate:function(v) { return (v.trim().length === 0) || (v.indexOf("://") === -1) || (v.trim().indexOf("http") === 0)} },
tls: {type:"tls-config",required: false},
persist: {value:false},
peerCertificateOutput: {value:false},
proxy: {type:"http proxy",required: false},
authType: {value: ""},
senderr: {value: false}
Expand Down
40 changes: 34 additions & 6 deletions packages/node_modules/@node-red/nodes/core/network/21-httprequest.js
View file
Open in desktop
Expand Up @@ -190,6 +190,8 @@ in your Node-RED user directory (${RED.settings.userDir}).
opts.cookieJar = new CookieJar();
opts.ignoreInvalidCookies = true;
opts.forever = nodeHTTPPersistent;
// use got stream API
opts.isStream = true;
if (msg.requestTimeout !== undefined) {
if (isNaN(msg.requestTimeout)) {
node.warn(RED._("httpin.errors.timeout-isnan"));
Expand All @@ -200,6 +202,13 @@ in your Node-RED user directory (${RED.settings.userDir}).
}
}
const originalHeaderMap = {};
var addPeerCertificateToOutput = n.peerCertificateOutput || false;
if (msg.peerCertificateOutput !== undefined) {
addPeerCertificateToOutput = msg.peerCertificateOutput;
}
if (addPeerCertificateToOutput === true && msg.peerCertificates === undefined) {
msg.peerCertificates = [];
}

opts.hooks = {
beforeRequest: [
Expand Down Expand Up @@ -451,7 +460,8 @@ in your Node-RED user directory (${RED.settings.userDir}).
},
maxFreeSockets: 256,
maxSockets: 256,
keepAlive: true
// for reliably retrieving the peer certificate, we must use agents with keepAlive=false
keepAlive: (addPeerCertificateToOutput === true) ? false : true
}
if (proxyConfig && proxyConfig.credentials) {
let proxyUsername = proxyConfig.credentials.username || '';
Expand All @@ -474,6 +484,12 @@ in your Node-RED user directory (${RED.settings.userDir}).
node.warn("Bad proxy url: "+ prox);
}
}
// for reliably retrieving the peer certificate, we must ensure using a separate agent with keepAlive=false
if (opts.agent === undefined && addPeerCertificateToOutput === true) {
opts.agent = {
https: new HTTPS_MODULE.Agent({keepAlive: false})
};
}
if (tlsNode) {
opts.https = {};
tlsNode.addTLSOptions(opts.https);
Expand All @@ -498,11 +514,21 @@ in your Node-RED user directory (${RED.settings.userDir}).
originalHeaderMap[h.toLowerCase()] = h
})
}
got(url,opts).then(res => {
const body = [];
const g = got(url,opts);
g.on('data', chunk => body.push(chunk));
if (addPeerCertificateToOutput === true) {
g.on('socket', socket => socket.on('secureConnect', () => {
const peerCert = socket.getPeerCertificate();
if (peerCert && typeof peerCert === 'object' && Object.keys(peerCert).length > 0) {
msg.peerCertificates.filter(cert => cert.fingerprint == peerCert.fingerprint).length === 0 && msg.peerCertificates.push(peerCert);
}
}));
}
g.on('response', res => {
msg.statusCode = res.statusCode;
msg.headers = res.headers;
msg.responseUrl = res.url;
msg.payload = res.body;
msg.redirectList = redirectList;
msg.retry = 0;

Expand All @@ -524,7 +550,9 @@ in your Node-RED user directory (${RED.settings.userDir}).
emitTimingMetricLog(res.timings, msg);
}
}

});
g.on('end', () => {
msg.payload = Buffer.concat(body);
// Convert the payload to the required return type
if (node.ret !== "bin") {
msg.payload = msg.payload.toString('utf8'); // txt
Expand All @@ -537,13 +565,13 @@ in your Node-RED user directory (${RED.settings.userDir}).
node.status({});
nodeSend(msg);
nodeDone();
}).catch(err => {
});
g.on('error', err => {
// Pre 2.1, any errors would be sent to both Catch node and sent on as normal.
// This is not ideal but is the legacy behaviour of the node.
// 2.1 adds the 'senderr' option, if set to true, will *only* send errors
// to Catch nodes. If false, it still does both behaviours.
// TODO: 3.0 - make it one or the other.

if (err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') {
node.error(RED._("common.notification.errors.no-response"), msg);
node.status({fill:"red", shape:"ring", text:"common.notification.errors.no-response"});
Expand Down
1 change: 1 addition & 0 deletions packages/node_modules/@node-red/nodes/locales/de/messages.json
View file
Open in desktop
Expand Up @@ -468,6 +468,7 @@
"use-proxy": "Proxy verwenden",
"persist": "Verbindung aufrecht erhalten (keep-alive)",
"proxy-config": "Proxy-Konfiguration",
"peerCertOutput": "Füge Zertifikatsinformation der Gegenstelle zum Nachrichten-Objekt hinzu",
"use-proxyauth": "Proxy-Authentifizierung verwenden",
"noproxy-hosts": "Hosts ignorieren",
"utf8": "Eine UTF-8-Zeichenfolge",
Expand Down
4 changes: 4 additions & 0 deletions packages/node_modules/@node-red/nodes/locales/de/network/21-httprequest.html
View file
Open in desktop
Expand Up @@ -37,6 +37,8 @@ <h3>Eingangsdaten</h3>
<dt class="optional">requestTimeout</dt>
<dd>Wenn dieser Wert auf eine positive Zahl eingestellt ist,
wird damit der global eingestellte Parameter <code>httpRequestTimeout</code> überschrieben.</dd>
<dt class="optional">peerCertificateOutput</dt>
<dd>Wenn auf <code>true</code> gesetzt, wird bei https Anforderungen die Zertifikatsinformation der Gegenstelle ausgegeben.<code>false</code> im Standard</dd>
</dl>
<h3>Ausgangsdaten</h3>
<dl class="message-properties">
Expand All @@ -53,6 +55,8 @@ <h3>Ausgangsdaten</h3>
Andernfalls die URL der ursprünglichen Anforderung.</dd>
<dt>responseCookies <span class="property-type">object</span></dt>
<dd>Wenn die Antwort Cookies enthält, ist dieses Element ein Objekt von Name/Wert-Paaren für jedes Cookie.</dd>
<dt>peerCertificates <span class="property-type">array</span></dt>
<dd>Wenn die Anforderung über https gesendet wurde, enthält diese Eigenschaft die Zertifikatsinformationen der Gegenstelle.</dd>
</dl>
<h3>Details</h3>
<p>Wenn innerhalb des Nodes eingestellt, kann die URL-Eigenschaft
Expand Down
1 change: 1 addition & 0 deletions packages/node_modules/@node-red/nodes/locales/en-US/messages.json
View file
Open in desktop
Expand Up @@ -486,6 +486,7 @@
"use-proxy": "Use proxy",
"persist": "Enable connection keep-alive",
"proxy-config": "Proxy Configuration",
"peerCertOutput": "Add peer certificate info to message output",
"use-proxyauth": "Use proxy authentication",
"noproxy-hosts": "Ignore hosts",
"senderr": "Only send non-2xx responses to Catch node",
Expand Down
4 changes: 4 additions & 0 deletions packages/node_modules/@node-red/nodes/locales/en-US/network/21-httprequest.html
View file
Open in desktop
Expand Up @@ -37,6 +37,8 @@ <h3>Inputs</h3>
<dd>If set to <code>false</code> prevent following Redirect (HTTP 301).<code>true</code> by default</dd>
<dt class="optional">requestTimeout</dt>
<dd>If set to a positive number of milliseconds, will override the globally set <code>httpRequestTimeout</code> parameter.</dd>
<dt class="optional">peerCertificateOutput</dt>
<dd>If set to <code>true</code> adds peer certificate information from https requests to message output.<code>false</code> by default</dd>
</dl>
<h3>Outputs</h3>
<dl class="message-properties">
Expand All @@ -55,6 +57,8 @@ <h3>Outputs</h3>
<dd>If the response includes cookies, this propery is an object of name/value pairs for each cookie.</dd>
<dt>redirectList <span class="property-type">array</span></dt>
<dd>If the request was redirected one or more times, the accumulated information will be added to this property. `location` is the next redirect destination. `cookies` is the cookies returned from the redirect source.</dd>
<dt>peerCertificates <span class="property-type">array</span></dt>
<dd>If the request was sent via https, the peer certificate information is available via this property.</dd>
</dl>
<h3>Details</h3>
<p>When configured within the node, the URL property can contain <a href="http://mustache.github.io/mustache.5.html" target="_blank">mustache-style</a> tags. These allow the
Expand Down
36 changes: 32 additions & 4 deletions test/nodes/core/network/21-httprequest_spec.js
View file
Open in desktop
Expand Up @@ -1503,8 +1503,32 @@ describe('HTTP Request Node', function() {
});

describe('protocol', function() {
it('should use msg.peerCertificateOutput', function(done) {
var flow = [{id:"n1",type:"http request",wires:[["n2"]],method:"GET",ret:"txt",url:getSslTestURL('/text'),peerCertificateOutput:false},
{id:"n2", type:"helper"}];
helper.load(httpRequestNode, flow, function() {
var n2 = helper.getNode("n2");
var n1 = helper.getNode("n1");
n2.on("input", function(msg) {
try {
msg.should.have.property('payload','hello');
msg.should.have.property('statusCode',200);
msg.should.have.property('headers');
msg.headers.should.have.property('content-length',''+('hello'.length));
msg.headers.should.have.property('content-type').which.startWith('text/html');
msg.should.have.property('responseUrl').which.startWith('https://');
msg.should.have.property('peerCertificates').which.is.a.Array();
done();
} catch(err) {
done(err);
}
});
n1.receive({payload:"foo", rejectUnauthorized: false, peerCertificateOutput: true});
});
});

it('should use msg.rejectUnauthorized', function(done) {
var flow = [{id:"n1",type:"http request",wires:[["n2"]],method:"GET",ret:"txt",url:getSslTestURL('/text')},
var flow = [{id:"n1",type:"http request",wires:[["n2"]],method:"GET",ret:"txt",url:getSslTestURL('/text'),peerCertificateOutput:true},
{id:"n2", type:"helper"}];
helper.load(httpRequestNode, flow, function() {
var n2 = helper.getNode("n2");
Expand All @@ -1517,6 +1541,7 @@ describe('HTTP Request Node', function() {
msg.headers.should.have.property('content-length',''+('hello'.length));
msg.headers.should.have.property('content-type').which.startWith('text/html');
msg.should.have.property('responseUrl').which.startWith('https://');
msg.should.have.property('peerCertificates').which.is.a.Array();
done();
} catch(err) {
done(err);
Expand All @@ -1528,7 +1553,7 @@ describe('HTTP Request Node', function() {

it('should use tls-config', function(done) {
var flow = [
{id:"n1",type:"http request",wires:[["n2"]],method:"GET",ret:"txt",url:getSslTestURLWithoutProtocol('/text'),tls:"n3"},
{id:"n1",type:"http request",wires:[["n2"]],method:"GET",ret:"txt",url:getSslTestURLWithoutProtocol('/text'),tls:"n3",peerCertificateOutput:true},
{id:"n2", type:"helper"},
{id:"n3", type:"tls-config", cert:"test/resources/ssl/server.crt", key:"test/resources/ssl/server.key", ca:"", verifyservercert:false}];
var testNodes = [httpRequestNode, tlsNode];
Expand All @@ -1544,6 +1569,7 @@ describe('HTTP Request Node', function() {
msg.headers.should.have.property('content-length',''+('hello'.length));
msg.headers.should.have.property('content-type').which.startWith('text/html');
msg.should.have.property('responseUrl').which.startWith('https://');
msg.should.have.property('peerCertificates').which.is.a.Array();
done();
} catch(err) {
done(err);
Expand All @@ -1555,7 +1581,7 @@ describe('HTTP Request Node', function() {

it('should use tls-config and verify serverCert', function(done) {
var flow = [
{id:"n1",type:"http request",wires:[["n2"]],method:"GET",ret:"txt",url:getSslTestURL('/text'),tls:"n3"},
{id:"n1",type:"http request",wires:[["n2"]],method:"GET",ret:"txt",url:getSslTestURL('/text'),tls:"n3",peerCertificateOutput:true},
{id:"n2", type:"helper"},
{id:"n3", type:"tls-config", cert:"test/resources/ssl/server.crt", key:"test/resources/ssl/server.key", ca:"test/resources/ssl/server.crt", verifyservercert:true}];
var testNodes = [httpRequestNode, tlsNode];
Expand All @@ -1571,6 +1597,7 @@ describe('HTTP Request Node', function() {
msg.headers.should.have.property('content-length',''+('hello'.length));
msg.headers.should.have.property('content-type').which.startWith('text/html');
msg.should.have.property('responseUrl').which.startWith('https://');
msg.should.have.property('peerCertificates').which.is.a.Array();
done();
} catch(err) {
done(err);
Expand All @@ -1582,7 +1609,7 @@ describe('HTTP Request Node', function() {

it('should use tls-config and send client cert', function(done) {
var flow = [
{id:"n1",type:"http request",wires:[["n2"]],method:"GET",ret:"txt",url:getSslClientTestURL('/getClientCert'),tls:"n3"},
{id:"n1",type:"http request",wires:[["n2"]],method:"GET",ret:"txt",url:getSslClientTestURL('/getClientCert'),tls:"n3",peerCertificateOutput:true},
{id:"n2", type:"helper"},
{id:"n3", type:"tls-config", cert:"test/resources/ssl/server.crt", key:"test/resources/ssl/server.key", ca:"test/resources/ssl/server.crt", verifyservercert:false}];
var testNodes = [httpRequestNode,tlsNode];
Expand All @@ -1598,6 +1625,7 @@ describe('HTTP Request Node', function() {
msg.headers.should.have.property('content-length',''+('hello'.length));
msg.headers.should.have.property('content-type').which.startWith('text/html');
msg.should.have.property('responseUrl').which.startWith('https://');
msg.should.have.property('peerCertificates').which.is.a.Array();
done();
} catch(err) {
done(err);
Expand Down