diff --git a/README.md b/README.md index dc78fb4..a938ad9 100644 --- a/README.md +++ b/README.md @@ -21,17 +21,8 @@ Number of minutes to backfill after connecting to the stream. Optional. Value sh Partition of the Firehose stream you want to connect to. Only required for Firehose streams. #### options.parser -Parser library for incoming JSON data. Optional, defaults to the native JSON parser. -Matching tag IDs are sent to us as big integers which can't be reliably parsed by the native JSON library in Node.js. When you rely on tag IDs you can use the excellent [json-bigint](https://www.npmjs.com/package/json-bigint) library: - -``` -var JSONbig = require('json-bigint'); -var stream = new Gnip.Stream({ - parser: JSONbig, - ... -}); -``` -More info on this issue can be found at [StackOverflow](http://stackoverflow.com/questions/8663298/json-transfer-of-bigint-12000000000002539-is-converted-to-12000000000002540) +Parser library for incoming JSON data. Optional, but defaults to the excellent [json-bigint](https://www.npmjs.com/package/json-bigint) library. +Matching tag IDs are sent to us as big integers which can't be reliably parsed by the native JSON library in Node.js. More info on this issue can be found at [StackOverflow](http://stackoverflow.com/questions/8663298/json-transfer-of-bigint-12000000000002539-is-converted-to-12000000000002540) ## API methods @@ -76,7 +67,7 @@ GNIP Rules endpoint url e.g. `https://gnip-api.twitter.com/rules/${streamType}/a The batch size used when adding/deleting rules in bulk. (Defaults to 5000) #### options.parser -Much like the `parser` option allowed in the [Gnip Stream](https://github.com/demian85/gnip#gnipstream) constructor, you can pass a custom parser handler/library for incoming JSON data. This is optional, and defaults to the native `JSON` parser. [More details](https://github.com/demian85/gnip#optionsparser). +Much like the `parser` option allowed in the [Gnip Stream](https://github.com/demian85/gnip#gnipstream) constructor, you can pass a custom parser handler/library for incoming JSON data. This is optional, and defaults to the [json-bigint](https://www.npmjs.com/package/json-bigint) library. [More details](https://github.com/demian85/gnip#optionsparser). ## API methods @@ -197,7 +188,6 @@ Installation Example Usage ==== var Gnip = require('gnip'); - var JSONBigInt = require('json-bigint'); var stream = new Gnip.Stream({ url : 'https://gnip-stream.twitter.com/stream/powertrack/accounts/xxx/publishers/twitter/prod.json', @@ -219,8 +209,7 @@ Example Usage url : 'https://gnip-api.twitter.com/rules/powertrack/accounts/xxx/publishers/twitter/prod.json', user : 'xxx', password : 'xxx', - batchSize: 1234, // not required, defaults to 5000 - parser: JSONBigInt // required in order to properly parse large integral rule IDs + batchSize: 1234 // not required, defaults to 5000 }); var newRules = [ diff --git a/lib/index.js b/lib/index.js index b7684ab..37c3ebd 100644 --- a/lib/index.js +++ b/lib/index.js @@ -4,6 +4,7 @@ var EventEmitter = require('events').EventEmitter, GnipRules = require('./rules'), GnipUsage = require('./usage'), GnipSearch = require('./search'), + JSONBigInt = require('json-bigint'), zlib = require('zlib'), _ = require('underscore'), querystring = require('querystring'), @@ -41,7 +42,7 @@ var GnipStream = function (options) { userAgent: null, url: null, debug: false, - parser: JSON + parser: JSONBigInt }, options || {}); self._req = null; diff --git a/lib/rules.js b/lib/rules.js index 5599911..01c91f7 100644 --- a/lib/rules.js +++ b/lib/rules.js @@ -2,7 +2,8 @@ var request = require('request'), crypto = require('crypto'), _ = require('underscore'), fs = require('fs'), - async = require('async'); + async = require('async'), + JSONBigInt = require('json-bigint'),; var LiveRules = function (endPoint, user, password, parser) { this._api = endPoint; @@ -123,7 +124,7 @@ var GnipRules = function (options) { url: null, debug: false, batchSize: 5000, - parser: JSON + parser: JSONBigInt }, options || {}); this._api = this.options.url; diff --git a/package-lock.json b/package-lock.json index eb8ef8d..be1f3bd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "gnip", - "version": "1.4.0", + "version": "1.4.1", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -172,6 +172,11 @@ "tweetnacl": "0.14.5" } }, + "bignumber.js": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-4.1.0.tgz", + "integrity": "sha512-eJzYkFYy9L4JzXsbymsFn3p54D+llV27oTQ+ziJG7WFRheJcNZilgVXMG0LoZtlQSKBsJdWtLFqOD0u+U0jZKA==" + }, "boom": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/boom/-/boom-4.3.1.tgz", @@ -855,6 +860,14 @@ "integrity": "sha512-vE2hT1D0HLZCLLclfBSfkfTTedhVj0fubHpJBHKwwUWX0nSbhPAfk+SG9rTX95BYNmau8rGFfCeaT6T5OW1C2A==", "dev": true }, + "json-bigint": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.2.3.tgz", + "integrity": "sha1-EY1/b/HThlnxn5TPc+ZKdaP5iKg=", + "requires": { + "bignumber.js": "4.1.0" + } + }, "json-schema": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", @@ -1286,6 +1299,15 @@ "tweetnacl": "0.14.5" } }, + "string_decoder": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", + "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==", + "dev": true, + "requires": { + "safe-buffer": "5.1.1" + } + }, "string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", @@ -1296,15 +1318,6 @@ "strip-ansi": "4.0.0" } }, - "string_decoder": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", - "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==", - "dev": true, - "requires": { - "safe-buffer": "5.1.1" - } - }, "stringstream": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", diff --git a/package.json b/package.json index ea1833e..a3fe147 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,10 @@ { "name": "Dirk Bonhomme", "email": "dirk@bytelogic.be" + }, + { + "name": "Tom Zellman", + "email": "tzellman@gmail.com" } ], "name": "gnip", @@ -30,6 +34,7 @@ }, "dependencies": { "async": "0.1.x", + "json-bigint": "^0.2.3", "limiter": "^1.1.0", "request": "2.x", "underscore": "^1.3.0"