diff --git a/event_processor.js b/event_processor.js index 66fa4e1..0600382 100644 --- a/event_processor.js +++ b/event_processor.js @@ -1,6 +1,6 @@ const LRUCache = require('lrucache'); -const request = require('request'); const uuidv4 = require('uuid/v4'); + const EventSummarizer = require('./event_summarizer'); const UserFilter = require('./user_filter'); const errors = require('./errors'); @@ -228,47 +228,40 @@ function EventProcessor(sdkKey, config, errorReporter, diagnosticsManager) { } }; - const headers = Object.assign({ 'X-LaunchDarkly-Event-Schema': '3' }, httpUtils.getDefaultHeaders(sdkKey, config)); + const headers = Object.assign({ 'Content-Type': 'application/json' }, httpUtils.getDefaultHeaders(sdkKey, config)); if (payloadId) { headers['X-LaunchDarkly-Payload-ID'] = payloadId; headers['X-LaunchDarkly-Event-Schema'] = '3'; } - const options = Object.assign({}, config.tlsParams, { - method: 'POST', - url: uri, - headers, - json: true, - body: events, - timeout: config.timeout * 1000, - agent: config.proxyAgent, - }); - request(options) - .on('response', (resp, body) => { - if (resp.headers['date']) { - const date = Date.parse(resp.headers['date']); - if (date) { - lastKnownPastTime = date; - } + const options = { method: 'POST', headers }; + const body = JSON.stringify(events); + httpUtils.httpRequest(uri, options, body, config, (err, resp) => { + if (err) { + retryOrReject(err); + return; + } + if (resp.headers['date']) { + const date = Date.parse(resp.headers['date']); + if (date) { + lastKnownPastTime = date; } - if (resp.statusCode > 204) { - const err = new errors.LDUnexpectedResponseError( - messages.httpErrorMessage(resp.statusCode, 'event posting', 'some events were dropped') - ); - errorReporter && errorReporter(err); - if (!errors.isHttpErrorRecoverable(resp.statusCode)) { - reject(err); - shutdown = true; - } else { - retryOrReject(err); - } + } + if (resp.statusCode > 204) { + const err = new errors.LDUnexpectedResponseError( + messages.httpErrorMessage(resp.statusCode, 'event posting', 'some events were dropped') + ); + errorReporter && errorReporter(err); + if (!errors.isHttpErrorRecoverable(resp.statusCode)) { + reject(err); + shutdown = true; } else { - resolve(resp, body); + retryOrReject(err); } - }) - .on('error', err => { - retryOrReject(err); - }); + } else { + resolve(); + } + }); } function postDiagnosticEvent(event) { diff --git a/package-lock.json b/package-lock.json index 2096c96..a55a66d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1376,6 +1376,7 @@ "version": "6.10.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz", "integrity": "sha512-nffhOpkymDECQyR0mnsUtoCE8RlX38G0rYP+wgLWFyZuUyuuojSSvi/+euOiQBIn63whYwYVIIH1TvE3tu4OEg==", + "dev": true, "requires": { "fast-deep-equal": "^2.0.1", "fast-json-stable-stringify": "^2.0.0", @@ -1457,6 +1458,7 @@ "version": "0.2.4", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "dev": true, "requires": { "safer-buffer": "~2.1.0" } @@ -1464,7 +1466,8 @@ "assert-plus": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", + "dev": true }, "assign-symbols": { "version": "1.0.0", @@ -1492,7 +1495,8 @@ "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", + "dev": true }, "atob": { "version": "2.1.2", @@ -1503,12 +1507,14 @@ "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", - "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=", + "dev": true }, "aws4": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.9.1.tgz", - "integrity": "sha512-wMHVg2EOHaMRxbzgFJ9gtjOOCrI80OHLG14rxi28XwOW8ux6IiEbRCGGGqCtdAIg4FQCbW20k9RsT4y3gJlFug==" + "integrity": "sha512-wMHVg2EOHaMRxbzgFJ9gtjOOCrI80OHLG14rxi28XwOW8ux6IiEbRCGGGqCtdAIg4FQCbW20k9RsT4y3gJlFug==", + "dev": true }, "babel-jest": { "version": "24.7.1", @@ -1629,6 +1635,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "dev": true, "requires": { "tweetnacl": "^0.14.3" } @@ -1779,7 +1786,8 @@ "caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=", + "dev": true }, "chalk": { "version": "2.4.2", @@ -1921,6 +1929,7 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, "requires": { "delayed-stream": "~1.0.0" } @@ -2017,7 +2026,8 @@ "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", + "dev": true }, "cross-spawn": { "version": "6.0.5", @@ -2064,6 +2074,7 @@ "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "dev": true, "requires": { "assert-plus": "^1.0.0" } @@ -2180,7 +2191,8 @@ "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", + "dev": true }, "detect-newline": { "version": "2.1.0", @@ -2221,6 +2233,7 @@ "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "dev": true, "requires": { "jsbn": "~0.1.0", "safer-buffer": "^2.1.0" @@ -2668,7 +2681,8 @@ "extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "dev": true }, "extend-shallow": { "version": "3.0.2", @@ -2770,7 +2784,8 @@ "extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=", + "dev": true }, "eyes": { "version": "0.1.8", @@ -2780,7 +2795,8 @@ "fast-deep-equal": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", - "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=", + "dev": true }, "fast-diff": { "version": "1.2.0", @@ -2791,7 +2807,8 @@ "fast-json-stable-stringify": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", - "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=", + "dev": true }, "fast-levenshtein": { "version": "2.0.6", @@ -2884,12 +2901,14 @@ "forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=", + "dev": true }, "form-data": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "dev": true, "requires": { "asynckit": "^0.4.0", "combined-stream": "^1.0.6", @@ -3502,6 +3521,7 @@ "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "dev": true, "requires": { "assert-plus": "^1.0.0" } @@ -3570,12 +3590,14 @@ "har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", - "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=", + "dev": true }, "har-validator": { "version": "5.1.3", "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "dev": true, "requires": { "ajv": "^6.5.5", "har-schema": "^2.0.0" @@ -3653,6 +3675,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "dev": true, "requires": { "assert-plus": "^1.0.0", "jsprim": "^1.2.2", @@ -3961,7 +3984,8 @@ "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", + "dev": true }, "is-windows": { "version": "1.0.2", @@ -4646,7 +4670,8 @@ "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", + "dev": true }, "jsdom": { "version": "11.12.0", @@ -4715,12 +4740,14 @@ "json-schema": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=", + "dev": true }, "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true }, "json-stable-stringify-without-jsonify": { "version": "1.0.1", @@ -4731,7 +4758,8 @@ "json-stringify-safe": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=", + "dev": true }, "json5": { "version": "2.1.1", @@ -4754,6 +4782,7 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "dev": true, "requires": { "assert-plus": "1.0.0", "extsprintf": "1.3.0", @@ -5614,16 +5643,6 @@ "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", "dev": true }, - "lodash.assign": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz", - "integrity": "sha1-DZnzzNem0mHRm9rrkkUAXShYCOc=" - }, - "lodash.clonedeep": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", - "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=" - }, "lodash.sortby": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", @@ -5648,15 +5667,6 @@ "js-tokens": "^3.0.0 || ^4.0.0" } }, - "lru-cache": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.2.tgz", - "integrity": "sha512-wgeVXhrDwAWnIF/yZARsFnMBtdFXOg1b8RIrhilp+0iDYN4mdQcNZElDZ0e4B64BhaxeQ5zN7PMyvu7we1kPeQ==", - "requires": { - "pseudomap": "^1.0.2", - "yallist": "^2.1.2" - } - }, "lrucache": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/lrucache/-/lrucache-1.0.3.tgz", @@ -5763,12 +5773,14 @@ "mime-db": { "version": "1.43.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.43.0.tgz", - "integrity": "sha512-+5dsGEEovYbT8UY9yD7eE4XTc4UwJ1jBYlgaQQF38ENsKR3wj/8q8RFZrF9WIZpB2V1ArTVFUva8sAul1NzRzQ==" + "integrity": "sha512-+5dsGEEovYbT8UY9yD7eE4XTc4UwJ1jBYlgaQQF38ENsKR3wj/8q8RFZrF9WIZpB2V1ArTVFUva8sAul1NzRzQ==", + "dev": true }, "mime-types": { "version": "2.1.26", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.26.tgz", "integrity": "sha512-01paPWYgLrkqAyrlDorC1uDwl2p3qZT7yl806vW7DvDoxwXi46jsjFbg+WdwotBIk6/MbEhO/dh5aZ5sNj/dWQ==", + "dev": true, "requires": { "mime-db": "1.43.0" } @@ -6005,7 +6017,8 @@ "oauth-sign": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", - "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", + "dev": true }, "object-copy": { "version": "0.1.0", @@ -6283,7 +6296,8 @@ "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=", + "dev": true }, "pify": { "version": "3.0.0", @@ -6391,15 +6405,11 @@ "sisteransi": "^1.0.0" } }, - "pseudomap": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", - "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" - }, "psl": { "version": "1.1.31", "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz", - "integrity": "sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw==" + "integrity": "sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw==", + "dev": true }, "pump": { "version": "3.0.0", @@ -6420,7 +6430,8 @@ "qs": { "version": "6.5.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", + "dev": true }, "react-is": { "version": "16.8.6", @@ -6604,6 +6615,7 @@ "version": "2.88.2", "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", + "dev": true, "requires": { "aws-sign2": "~0.7.0", "aws4": "^1.8.0", @@ -6630,17 +6642,20 @@ "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "dev": true }, "safe-buffer": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", - "integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg==" + "integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg==", + "dev": true }, "tough-cookie": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "dev": true, "requires": { "psl": "^1.1.28", "punycode": "^2.1.1" @@ -6648,17 +6663,6 @@ } } }, - "request-etag": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/request-etag/-/request-etag-2.0.3.tgz", - "integrity": "sha1-E8kjdhfWxYnrcYX29FC4MTBovBc=", - "requires": { - "lodash.assign": "^4.0.0", - "lodash.clonedeep": "^4.0.1", - "lru-cache": "^4.0.0", - "request": "^2.67.0" - } - }, "request-promise-core": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.2.tgz", @@ -6773,7 +6777,8 @@ "safe-buffer": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", - "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" + "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==", + "dev": true }, "safe-regex": { "version": "1.1.0", @@ -6787,7 +6792,8 @@ "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true }, "sane": { "version": "4.1.0", @@ -7123,6 +7129,7 @@ "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", + "dev": true, "requires": { "asn1": "~0.2.3", "assert-plus": "^1.0.0", @@ -7488,6 +7495,7 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "dev": true, "requires": { "safe-buffer": "^5.0.1" } @@ -7495,7 +7503,8 @@ "tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", + "dev": true }, "type-check": { "version": "0.3.2", @@ -7622,6 +7631,7 @@ "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "dev": true, "requires": { "punycode": "^2.1.0" }, @@ -7629,7 +7639,8 @@ "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "dev": true } } }, @@ -7686,6 +7697,7 @@ "version": "1.10.0", "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "dev": true, "requires": { "assert-plus": "^1.0.0", "core-util-is": "1.0.2", @@ -7883,11 +7895,6 @@ "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==", "dev": true }, - "yallist": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", - "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" - }, "yaml": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.0.1.tgz", diff --git a/package.json b/package.json index 03643f7..47aaf30 100644 --- a/package.json +++ b/package.json @@ -29,8 +29,6 @@ "lrucache": "^1.0.3", "node-cache": "^4.2.0", "redis": "^2.6.0-2", - "request": "^2.88.2", - "request-etag": "^2.0.3", "semver": "^6.3.0", "tunnel": "0.0.6", "uuid": "^3.3.3", diff --git a/polling.js b/polling.js index 68a33a5..8f69518 100644 --- a/polling.js +++ b/polling.js @@ -16,7 +16,7 @@ function PollingProcessor(config, requestor) { const startTime = new Date().getTime(); config.logger.debug('Polling LaunchDarkly for feature flag updates'); - requestor.requestAllData((err, resp) => { + requestor.requestAllData((err, respBody) => { const elapsed = new Date().getTime() - startTime; const sleepFor = Math.max(config.pollInterval * 1000 - elapsed, 0); config.logger.debug('Elapsed: %d ms, sleeping for %d ms', elapsed, sleepFor); @@ -33,17 +33,24 @@ function PollingProcessor(config, requestor) { }, sleepFor); } } else { - const allData = JSON.parse(resp); - const initData = {}; - initData[dataKind.features.namespace] = allData.flags; - initData[dataKind.segments.namespace] = allData.segments; - featureStore.init(initData, () => { - cb(); - // Recursively call poll after the appropriate delay + if (respBody) { + const allData = JSON.parse(respBody); + const initData = {}; + initData[dataKind.features.namespace] = allData.flags; + initData[dataKind.segments.namespace] = allData.segments; + featureStore.init(initData, () => { + cb(); + // Recursively call poll after the appropriate delay + setTimeout(() => { + poll(cb); + }, sleepFor); + }); + } else { + // There wasn't an error but there wasn't any new data either, so just keep polling setTimeout(() => { poll(cb); }, sleepFor); - }); + } } }); } diff --git a/requestor.js b/requestor.js index b150180..5aeded5 100644 --- a/requestor.js +++ b/requestor.js @@ -1,4 +1,3 @@ -const ETagRequest = require('request-etag'); const httpUtils = require('./utils/httpUtils'); /** @@ -16,31 +15,14 @@ function Requestor(sdkKey, config) { const requestor = {}; const headers = httpUtils.getDefaultHeaders(sdkKey, config); + const requestWithETagCaching = httpUtils.httpWithETagCache(); - const cacheConfig = { - max: 100, - // LRUCache passes each cached item through the "length" function to determine how many units it should - // count for toward "max". We want our cache limit to be based on the number of responses, not their - // size; that is in fact the default behavior of LRUCache, but request-etag overrides it unless we do this: - length: function() { - return 1; - }, - }; - const requestWithETagCaching = new ETagRequest(cacheConfig); - - function makeRequest(resource) { - const requestParams = Object.assign({}, config.tlsParams, { - method: 'GET', - url: config.baseUri + resource, - headers, - timeout: config.timeout * 1000, - agent: config.proxyAgent, - }); - + function makeRequest(resource, useCache) { + const url = config.baseUri + resource; + const requestParams = { method: 'GET', headers }; return (cb, errCb) => { - requestWithETagCaching(requestParams, (err, resp, body) => { - // Note that when request-etag gives us a cached response, the body will only be in the "body" - // callback parameter -- not in resp.getBody(). For a fresh response, it'll be in both. + const requestFn = useCache ? requestWithETagCaching : httpUtils.httpRequest; + requestFn(url, requestParams, null, config, (err, resp, body) => { if (err) { errCb(err); } else { @@ -57,7 +39,7 @@ function Requestor(sdkKey, config) { err.status = response.statusCode; cb(err, null); } else { - cb(null, body); + cb(null, response.statusCode === 304 ? null : body); } }; } @@ -69,12 +51,14 @@ function Requestor(sdkKey, config) { } requestor.requestObject = (kind, key, cb) => { - const req = makeRequest(kind.requestPath + key); + const req = makeRequest(kind.requestPath + key, false); req(processResponse(cb), processErrorResponse(cb)); }; + // Note that requestAllData will pass (null, null) rather than (null, body) if it gets a 304 response; + // this is deliberate so that we don't keep updating the data store unnecessarily if there are no changes. requestor.requestAllData = cb => { - const req = makeRequest('/sdk/latest-all'); + const req = makeRequest('/sdk/latest-all', true); req(processResponse(cb), processErrorResponse(cb)); }; diff --git a/test/requestor-test.js b/test/requestor-test.js index f5f3628..310dc1b 100644 --- a/test/requestor-test.js +++ b/test/requestor-test.js @@ -79,5 +79,27 @@ describe('Requestor', () => { const req = promisify(r.requestAllData)(); await expect(req).rejects.toThrow(/bad-uri/); }); + + it('stores and sends etag', async () => { + const etag = "abc123"; + await withCloseable(TestHttpServer.start, async server => { + server.forMethodAndPath('get', '/sdk/latest-all', (req, res) => { + if (req.headers['if-none-match'] === etag) { + TestHttpHandlers.respond(304)(req, res); + } else { + TestHttpHandlers.respond(200, { 'content-type': 'application/json', etag }, JSON.stringify(allData))(req, res); + } + }); + const r = Requestor(sdkKey, { baseUri: server.url }); + const result1 = await promisify(r.requestAllData)(); + expect(JSON.parse(result1)).toEqual(allData); + const result2 = await promisify(r.requestAllData)(); + expect(result2).toEqual(null); + const req1 = await server.nextRequest(); + const req2 = await server.nextRequest(); + expect(req1.headers['if-none-match']).toBe(undefined); + expect(req2.headers['if-none-match']).toEqual(etag); + }) + }); }); }); diff --git a/utils/httpUtils.js b/utils/httpUtils.js index 2c14632..20e191b 100644 --- a/utils/httpUtils.js +++ b/utils/httpUtils.js @@ -1,8 +1,12 @@ +const http = require('http'); +const https = require('https'); +const url = require('url'); + const packageJson = require('../package.json'); const userAgent = 'NodeJSClient/' + packageJson.version; -module.exports.getDefaultHeaders = (sdkKey, config) => { +function getDefaultHeaders(sdkKey, config) { // Use lowercase header names for convenience in our test code, where we may be checking for headers in a // real HTTP request that will be lowercased by the request API const ret = { @@ -15,4 +19,76 @@ module.exports.getDefaultHeaders = (sdkKey, config) => { : config.wrapperName; } return ret; +} + +// Convenience wrapper for making an HTTP/HTTPS request via Node's standard modules. Unlike http.request, +// the callback takes (error, response, body) parameters instead of just (response). +function httpRequest(requestUrl, options, body, config, callback) { + // Note: https.request allows a url parameter to be passed separately from options, but only in v10.9.0+, so + // we still have to parse the URL until our minimum Node version is increased. + const urlOpts = url.parse(requestUrl); + const isSecure = urlOpts.protocol === 'https:'; + const allOptions = Object.assign( + {}, + config && config.tlsParams, + urlOpts, + { + timeout: config && config.timeout ? config.timeout * 1000 : undefined, + agent: config && config.proxyAgent, + }, + options + ); + const req = (isSecure ? https : http).request(allOptions, resp => { + let body = ''; + resp.on('data', chunk => { + body += chunk; + }); + resp.on('end', () => { + callback(null, resp, body); + }); + }); + req.on('error', err => { + callback(err); + }); + if (body !== null && body !== undefined) { + req.write(body); + } + req.end(); +} + +// Creates an in-memory etag cache and returns a wrapper for httpRequest that uses the cache. This is a +// naive implementation that does not place a bound on the cache; the SDK will normally always be hitting +// the same URL (the only time we don't is if we get an "indirect/put" stream event, but in that case we +// deliberately do not use the cache). +function httpWithETagCache() { + const cache = {}; + return (requestUrl, options, body, config, callback) => { + const cacheEntry = cache[requestUrl]; + const cachedEtag = cacheEntry && cacheEntry.etag; + let newOptions = options; + if (cachedEtag) { + const newHeaders = Object.assign({}, options && options.headers, { 'if-none-match': cachedEtag }); + newOptions = Object.assign({}, options, { headers: newHeaders }); + } + return httpRequest(requestUrl, newOptions, body, config, (err, resp, body) => { + if (err) { + callback(err); + } else { + if (resp.statusCode === 304 && cacheEntry) { + callback(null, resp, cacheEntry.body); + } else { + if (resp.headers['etag']) { + cache[requestUrl] = { etag: resp.headers['etag'], body }; + } + callback(null, resp, body); + } + } + }); + }; +} + +module.exports = { + getDefaultHeaders, + httpRequest, + httpWithETagCache, };