diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index 2e8bd53..0ea8439 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -34,8 +34,10 @@ import * as transform from "./transformutil"; import * as fs from "fs"; import * as tmp from "tmp"; import * as summary from "./summary"; +import { deprecate } from "util"; +let cq = require("block-queue"); +const gsv = require("geojson-validation"); -const request = require("request"); let choiceList: { name: string, value: string}[] = []; const questions = [ { @@ -95,14 +97,15 @@ async function execInternal( json: isJson, headers: { Authorization: "Bearer " + token, - "Content-Type": contentType + "Content-Type": contentType, + "App-Name": "HereCLI" }, body: method === "GET" ? undefined : data }; const { response, body } = await requestAsync(reqJson); if (response.statusCode < 200 || response.statusCode > 210) - throw new Error("Invalid response"); + throw new Error("Invalid response - " + response.body); return body; } @@ -122,7 +125,8 @@ async function execInternalGzip( method: string, contentType: string, data: any, - token: string + token: string, + retry : number=3 ) { const zippedData = await gzip(data); const isJson = contentType == "application/json" ? true : false; @@ -141,9 +145,15 @@ async function execInternalGzip( body: method === "GET" ? undefined : zippedData }; - const { response, body } = await requestAsync(reqJson); - if (response.statusCode < 200 || response.statusCode > 210) - throw new Error("Invalid response"); + let { response, body } = await requestAsync(reqJson); + if (response.statusCode < 200 || response.statusCode > 210){ + if(response.statusCode>=500){ + await new Promise(done => setTimeout(done, 1000)); + body = execInternalGzip(uri,method,contentType,data,token,retry--); + }else{ + throw new Error("Invalid response :"+response.statusCode); + } + } return body; } @@ -566,12 +576,98 @@ program "option to enforce uniqueness to the id by creating a hash of feature and use that as id" ) .option("-o, --override", "override the data even if it share same id") + .option("-s, --stream", "streaming data support for large file uploads") .action(function (id, options) { uploadToXyzSpace(id, options); }); -function uploadToXyzSpace(id: string, options: any){ - (async () => { +function collate(result:Array){ + return result.reduce((features: any, feature: any) => { + if (feature.type === "Feature") { + features.push(feature); + } else if (feature.type === "FeatureCollection") { + features = features.concat(feature.features); + } else { + console.log("Unknown type" + feature.type); + } + return features + }, []); +} + +function streamingQueue(){ + let queue = cq(10,function (task:any,done:Function) { + uploadData(task.id, task.options, task.tags, task.fc, + true, task.options.ptag, task.options.file, task.options.id) + .then(x=>{ + queue.uploadCount += task.fc.features.length; + process.stdout.write("\ruploaded feature count :"+queue.uploadCount+", failed feature count :"+queue.failedCount); + queue.chunksize--; + done(); + }).catch((err) => { + queue.failedCount += task.fc.features.length; + process.stdout.write("\ruploaded feature count :"+queue.uploadCount+", failed feature count :"+queue.failedCount); + queue.chunksize--; + done(); + }); + }); + queue.uploadCount=0; + queue.chunksize=0; + queue.failedCount=0; + queue.send= async function(obj:any){ + while(this.chunksize>25){ + await new Promise(done => setTimeout(done, 1000)); + } + this.push(obj); + this.chunksize++; + } + queue.shutdown =async ()=>{ + queue.shutdown=true; + while(queue.chunksize!=0){ + await new Promise(done => setTimeout(done, 1000)); + } + return true; + } + return queue; +} + +function taskQueue(size:number=8,totalTaskSize:number){ + let queue = cq(size,function (task:any,done:Function) { + iterateChunk(task.chunk,task.url) + .then(x=>{ + queue.uploadCount += 1; + queue.chunksize--; + console.log("uploaded " + ((queue.uploadCount / totalTaskSize) * 100).toFixed(2) + "%"); + done(); + }).catch((err) => { + queue.failedCount += 1; + queue.chunksize--; + console.log("failed features " + ((queue.failedCount / totalTaskSize) * 100).toFixed(2) + "%"); + done(); + }); + }); + queue.uploadCount=0; + queue.chunksize=0; + queue.failedCount=0; + queue.send= async function(obj:any){ + queue.push(obj); + queue.chunksize++; + while(queue.chunksize>25){ + await new Promise(done => setTimeout(done, 1000)); + } + } + queue.shutdown =async ()=>{ + queue.shutdown=true; + while(queue.chunksize!=0){ + await new Promise(done => setTimeout(done, 1000)); + } + return true; + } + return queue; +} + + +async function uploadToXyzSpace(id: string, options: any){ + //(async () => { let tags = ""; if (options.tags) { tags = options.tags; @@ -590,27 +686,38 @@ function uploadToXyzSpace(id: string, options: any){ options.unique = true; } + if(options.assign && options.stream){ + console.log( + "conflicting options together. You cannot choose assign mode while selecting streaming option" + ); + process.exit(1); + } + if (options.file) { const fs = require("fs"); if (options.file.indexOf(".geojsonl") != -1) { - transform.readLineFromFile(options.file, 100).then((result: any) => { - const totalFeatures = result.reduce((features: any, feature: any) => { - if (feature.type === "Feature") { - features.push(feature); - } else if (feature.type === "FeatureCollection") { - features = features.concat(feature.features); - } else { - console.log("Unknown type" + feature.type); - } - return features - }, []); - uploadData(id, options, tags, { type: "FeatureCollection", features: totalFeatures }, true, options.ptag, options.file, options.id); - }); + if(!options.stream){ + transform.readLineFromFile(options.file, 100).then((result: any) => { + uploadData(id, options, tags, { type: "FeatureCollection", features: collate(result) }, true, options.ptag, options.file, options.id); + }); + }else{ + let queue = streamingQueue(); + transform.readLineAsChunks(options.file, options.chunk?options.chunk:1000,function(result:any){ + return new Promise((res,rej)=>{ + ( async()=>{ + if(result.length>0){ + await queue.send({id:id,options:options,tags:tags,fc:{ type: "FeatureCollection", features: collate(result) },retryCount:3}); + } + res(queue); + })(); + }); + }); + } } else if (options.file.indexOf(".shp") != -1) { let result = await transform.readShapeFile( options.file, ); - uploadData( + await uploadData( id, options, tags, @@ -621,44 +728,83 @@ function uploadToXyzSpace(id: string, options: any){ options.id ); } else if (options.file.indexOf(".csv") != -1) { - let result = await transform.read( - options.file, - true - ); - const object = { - features: transform.transform( - result, - options.lat, - options.lon, - options.alt - ), - type: "FeatureCollection" - }; - uploadData( + if(!options.stream){ + let result = await transform.read( + options.file, + true + ); + const object = { + features: transform.transform( + result, + options.lat, + options.lon, + options.alt + ), + type: "FeatureCollection" + }; + await uploadData( + id, + options, + tags, + object, + true, + options.ptag, + options.file, + options.id + ); + }else{ + let queue = streamingQueue(); + transform.readCSVAsChunks(options.file, options.chunk?options.chunk:1000,function(result:any){ + return new Promise((res,rej)=>{ + ( async()=>{ + if(result.length>0){ + const fc = { + features: transform.transform( + result, + options.lat, + options.lon, + options.alt + ), + type: "FeatureCollection" + }; + await queue.send({id:id,options:options,tags:tags,fc:fc,retryCount:3}); + res(queue); + } + })(); + }); + + }); + } + } else { + if(!options.stream){ + let result = await transform.read( + options.file, + false + ); + await uploadData( id, options, tags, - object, + JSON.parse(result), true, options.ptag, options.file, options.id - ); - } else { - let result = await transform.read( - options.file, - false - ); - uploadData( - id, - options, - tags, - JSON.parse(result), - true, - options.ptag, - options.file, - options.id - ); + ); + }else{ + let queue = streamingQueue(); + let c=0; + await transform.readGeoJsonAsChunks(options.file, options.chunk?options.chunk:1000,async function(result:any){ + if(result.length>0){ + const fc = { + features: result, + type: "FeatureCollection" + }; + await queue.send({id:id,options:options,tags:tags,fc:fc,retryCount:3}); + } + return queue; + }); + } } } else { const getStdin = require("get-stdin"); @@ -682,7 +828,7 @@ function uploadToXyzSpace(id: string, options: any){ } }); } - })(); + //})(); } function createQuestionsList(object: any) { @@ -718,24 +864,39 @@ function uploadData( fileName: string | null, uid: string ) { - if (object.type == "Feature") { - object = { features: [object], type: "FeatureCollection" }; - } - if (options.assign) { - //console.log("assign mode on"); - const questions = createQuestionsList(object); - inquirer.prompt(questions).then((answers: any) => { - if (options.ptag === undefined) { - options.ptag = ""; - } - options.ptag = options.ptag + answers.tagChoices; - if (options.id === undefined) { - options.id = ""; - } - options.id = options.id + answers.idChoice; - //console.log(options.ptag); - //console.log("unique key - " + options.id); - //Need to be inside if, else this will be executed before user choice is inserted as its async + return new Promise((resolve, reject) => { + + if (object.type == "Feature") { + object = { features: [object], type: "FeatureCollection" }; + } + if (options.assign) { + //console.log("assign mode on"); + const questions = createQuestionsList(object); + inquirer.prompt(questions).then((answers: any) => { + if (options.ptag === undefined) { + options.ptag = ""; + } + options.ptag = options.ptag + answers.tagChoices; + if (options.id === undefined) { + options.id = ""; + } + options.id = options.id + answers.idChoice; + //console.log(options.ptag); + //console.log("unique key - " + options.id); + //Need to be inside if, else this will be executed before user choice is inserted as its async + uploadDataToSpaceWithTags( + id, + options, + tags, + object, + false, + options.ptag, + fileName, + options.id + ).then(x=>resolve(x)).catch((error) => reject(error)); + + }); + } else { uploadDataToSpaceWithTags( id, options, @@ -745,23 +906,14 @@ function uploadData( options.ptag, fileName, options.id - ); - }); - } else { - uploadDataToSpaceWithTags( - id, - options, - tags, - object, - false, - options.ptag, - fileName, - options.id - ); - } + ).then(x=>resolve(x)).catch((error) => reject(error)); + } + + }); + } -function uploadDataToSpaceWithTags( +async function uploadDataToSpaceWithTags( id: string, options: any, tags: any, @@ -771,46 +923,59 @@ function uploadDataToSpaceWithTags( fileName: string | null, uid: string ) { - const gsv = require("geojson-validation"); - gsv.valid(object, async function (valid: boolean, errs: any) { - if (!valid) { - console.log(errs); - return; - } - const featureOut = await mergeAllTags( - object.features, - tags, - tagProperties, - fileName, - uid, - options - ); - - const chunks = options.chunk - ? chunkify(featureOut, parseInt(options.chunk)) - : [featureOut]; - const chunkSize = chunks.length; - const index = 0; - await iterateChunks( - chunks, - "/hub/spaces/" + id + "/features", - index, - chunkSize, - ); - if (isFile) - console.log( - "'" + - options.file + - "' uploaded to xyzspace '" + - id + - "' successfully" - ); - else - console.log( - "data upload to xyzspace '" + id + "' completed successfully" + return new Promise((resolve, reject) => { + gsv.valid(object, async function (valid: boolean, errs: any) { + if (!valid) { + console.log(errs); + reject(errs); + return; + } + const featureOut = await mergeAllTags( + object.features, + tags, + tagProperties, + fileName, + uid, + options ); - summary.summarize(featureOut, id, true); + try{ + if(options.stream){ + await iterateChunks([featureOut],"/hub/spaces/" + id + "/features",0,1,options.token); + }else{ + const chunks = options.chunk + ? chunkify(featureOut, parseInt(options.chunk)) + : [featureOut]; + await iterateChunks(chunks,"/hub/spaces/" + id + "/features",0,chunks.length,options.token); + // let tq = taskQueue(8,chunks.length); + // chunks.forEach(chunk=>{ + // tq.send({chunk:chunk,url:"/hub/spaces/" + id + "/features"}); + // }); + // await tq.shutdown(); + } + }catch(e){ + reject(e); + return; + } + if(!options.stream){ + if (isFile) + console.log( + "'" + + options.file + + "' uploaded to xyzspace '" + + id + + "' successfully" + ); + else + console.log( + "data upload to xyzspace '" + id + "' completed successfully" + ); + + summary.summarize(featureOut, id, true); + + } + resolve(true); + }); }); } @@ -982,7 +1147,7 @@ function getFileName(fileName: string) { } } -async function iterateChunks(chunks: any, url: string, index: number, chunkSize: number) { +async function iterateChunks(chunks: any, url: string, index: number, chunkSize: number, token: string) { const item = chunks.shift(); const fc = { type: "FeatureCollection", features: item }; const body = await execute( @@ -990,7 +1155,7 @@ async function iterateChunks(chunks: any, url: string, index: number, chunkSize: "PUT", "application/geo+json", JSON.stringify(fc), - null, + token, true ); @@ -1000,7 +1165,19 @@ async function iterateChunks(chunks: any, url: string, index: number, chunkSize: } console.log("uploaded " + ((index / chunkSize) * 100).toFixed(2) + "%"); - await iterateChunks(chunks, url, index, chunkSize); + await iterateChunks(chunks, url, index, chunkSize, token); +} +async function iterateChunk(chunk: any, url: string) { + const fc = { type: "FeatureCollection", features: chunk }; + const body = await execute( + url, + "PUT", + "application/geo+json", + JSON.stringify(fc), + null, + true + ); + return body; } function chunkify(data: any[], chunksize: number) { diff --git a/bin/transformutil.ts b/bin/transformutil.ts index 1ceb86e..71863c2 100644 --- a/bin/transformutil.ts +++ b/bin/transformutil.ts @@ -30,6 +30,7 @@ import * as tmp from "tmp"; import * as request from "request"; import * as readline from "readline"; import { requestAsync } from "./requestAsync"; +import { deprecate } from "util"; const latArray = ["y", "ycoord", "ycoordinate", "coordy", "coordinatey", "latitude", "lat"]; const lonArray = ["x", "xcoord", "xcoordinate", "coordx", "coordinatex", "longitude", "lon"]; @@ -227,3 +228,98 @@ export function readLineFromFile(incomingPath: string, chunckSize = 100) { }); } + +export function readLineAsChunks(incomingPath: string, chunckSize:number,streamFuntion:Function) { + return readData(incomingPath, 'geojsonl').then(path => { + return new Promise((resolve, reject) => { + let dataArray = new Array(); + var LineByLineReader = require('line-by-line'), + lr = new LineByLineReader(path); + lr.on('error', function (err:any) { + console.log(err); + throw err; + }); + lr.on('line', async function (line:any) { + dataArray.push(JSON.parse(line)); + if(dataArray.length>=chunckSize){ + lr.pause(); + await streamFuntion(dataArray); + lr.resume(); + dataArray=new Array(); + } + }); + lr.on('end', function () { + (async()=>{ + const queue = await streamFuntion(dataArray); + await queue.shutdown(); + console.log(""); + })(); + }); + }); + }); +} + + +export function readCSVAsChunks(incomingPath: string, chunckSize:number,streamFuntion:Function) { + return readData(incomingPath, 'csv').then(path => { + return new Promise((resolve, reject) => { + let dataArray = new Array(); + var csv = require("fast-csv"); + var stream = fs.createReadStream(path); + let csvstream = csv.fromStream(stream, {headers : true}).on("data", function(data:any){ + dataArray.push(data); + if(dataArray.length >=chunckSize){ + //console.log('dataArray '+chunckSize); + csvstream.pause(); + (async()=>{ + await streamFuntion(dataArray); + csvstream.resume(); + dataArray=new Array(); + })(); + } + }).on("end", function(){ + (async()=>{ + const queue = await streamFuntion(dataArray); + await queue.shutdown(); + console.log(""); + })(); + }); + }); + }); +} + + + +export function readGeoJsonAsChunks(incomingPath: string, chunckSize:number,streamFuntion:Function) { + return readData(incomingPath, 'geojson').then(path => { + return new Promise((resolve, reject) => { + let dataArray = new Array(); + const JSONStream = require('JSONStream'); + const es = require('event-stream'); + const fileStream = fs.createReadStream(path, {encoding: 'utf8'}); + let stream = fileStream.pipe(JSONStream.parse('features.*')); + stream.pipe(es.through(async function (data:any) { + dataArray.push(data); + if(dataArray.length >=chunckSize){ + stream.pause(); + fileStream.pause(); + await streamFuntion(dataArray); + dataArray=new Array(); + stream.resume(); + fileStream.resume(); + } + return data; + },function end () { + if(dataArray.length >0){ + (async()=>{ + const queue = await streamFuntion(dataArray); + await queue.shutdown(); + console.log(""); + dataArray=new Array(); + })(); + } + })); + }); + }); +} + \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 8f85c76..06a414d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -230,6 +230,15 @@ "integrity": "sha512-MDQLxNFRLasqS4UlkWMSACMKeSm1x4Q3TxzUC7KQUsh6RK1ZrQ0VEyE3yzXcBu+K8ejVj4wuX32eUG02yNp+YQ==", "dev": true }, + "JSONStream": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz", + "integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==", + "requires": { + "jsonparse": "^1.2.0", + "through": ">=2.2.7 <3" + } + }, "ajv": { "version": "5.5.2", "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", @@ -325,6 +334,25 @@ "sprintf-js": "~1.0.2" } }, + "arguments-extended": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/arguments-extended/-/arguments-extended-0.0.3.tgz", + "integrity": "sha1-YQfkkX0OtvCk3WYyD8Fa/HLvSUY=", + "requires": { + "extended": "~0.0.3", + "is-extended": "~0.0.8" + } + }, + "array-extended": { + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/array-extended/-/array-extended-0.0.11.tgz", + "integrity": "sha1-1xRK50jek8pybxIQCdv/FibRZL0=", + "requires": { + "arguments-extended": "~0.0.3", + "extended": "~0.0.3", + "is-extended": "~0.0.3" + } + }, "array-find-index": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/array-find-index/-/array-find-index-1.0.2.tgz", @@ -586,6 +614,11 @@ "tweetnacl": "^0.14.3" } }, + "block-queue": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/block-queue/-/block-queue-0.0.2.tgz", + "integrity": "sha1-HTjGN2yxmAGHOdnTFAw+BNxKzG8=" + }, "bluebird": { "version": "3.5.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.2.tgz", @@ -1013,6 +1046,16 @@ "assert-plus": "^1.0.0" } }, + "date-extended": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/date-extended/-/date-extended-0.0.6.tgz", + "integrity": "sha1-I4AtV90b94GIE/4MMuhRqG2iZ8k=", + "requires": { + "array-extended": "~0.0.3", + "extended": "~0.0.3", + "is-extended": "~0.0.3" + } + }, "debug": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", @@ -1026,6 +1069,11 @@ "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" }, + "declare.js": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/declare.js/-/declare.js-0.0.8.tgz", + "integrity": "sha1-BHit/5VkwAT1Hfc9i8E0AZ0o3N4=" + }, "decode-uri-component": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", @@ -1124,6 +1172,11 @@ "is-obj": "^1.0.0" } }, + "duplexer": { + "version": "0.1.1", + "resolved": "http://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz", + "integrity": "sha1-rOb/gIwc5mtX0ev5eXessCM0z8E=" + }, "duplexer3": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", @@ -1200,6 +1253,20 @@ "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=" }, + "event-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-4.0.1.tgz", + "integrity": "sha512-qACXdu/9VHPBzcyhdOWR5/IahhGMf0roTeZJfzz077GwylcDd90yOHLouhmv7GJ5XzPi6ekaQWd8AvPP2nOvpA==", + "requires": { + "duplexer": "^0.1.1", + "from": "^0.1.7", + "map-stream": "0.0.7", + "pause-stream": "^0.0.11", + "split": "^1.0.1", + "stream-combiner": "^0.2.2", + "through": "^2.3.8" + } + }, "execa": { "version": "0.10.0", "resolved": "https://registry.npmjs.org/execa/-/execa-0.10.0.tgz", @@ -1232,6 +1299,22 @@ "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, + "extended": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/extended/-/extended-0.0.6.tgz", + "integrity": "sha1-f7i/e52uOXWG5IVwrP1kLHjlBmk=", + "requires": { + "extender": "~0.0.5" + } + }, + "extender": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/extender/-/extender-0.0.10.tgz", + "integrity": "sha1-WJwHSCvmGhRgttgfnCSqZ+jzJM0=", + "requires": { + "declare.js": "~0.0.4" + } + }, "external-editor": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.0.3.tgz", @@ -1262,6 +1345,17 @@ "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", "integrity": "sha1-Ys8SAjTGg3hdkCNIqADvPgzCC8A=" }, + "fast-csv": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-2.4.1.tgz", + "integrity": "sha1-vX3SaDkfcpNntZRFuN0K0CaIGyY=", + "requires": { + "extended": "0.0.6", + "is-extended": "0.0.10", + "object-extended": "0.0.7", + "string-extended": "0.0.8" + } + }, "fast-deep-equal": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", @@ -1329,6 +1423,11 @@ } } }, + "from": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", + "integrity": "sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4=" + }, "from2": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", @@ -1805,6 +1904,14 @@ "resolved": "https://registry.npmjs.org/is-es2016-keyword/-/is-es2016-keyword-1.0.0.tgz", "integrity": "sha1-9uVOEQxeT40mXmnS7Q6vjPX0dxg=" }, + "is-extended": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/is-extended/-/is-extended-0.0.10.tgz", + "integrity": "sha1-JE4UDfdbscmjEG9BL/GC+1NKbWI=", + "requires": { + "extended": "~0.0.3" + } + }, "is-finite": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-finite/-/is-finite-1.0.2.tgz", @@ -1975,6 +2082,11 @@ "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, + "jsonparse": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", + "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=" + }, "jsprim": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", @@ -2065,6 +2177,11 @@ "invert-kv": "^1.0.0" } }, + "line-by-line": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/line-by-line/-/line-by-line-0.1.6.tgz", + "integrity": "sha512-MmwVPfOyp0lWnEZ3fBA8Ah4pMFvxO6WgWovqZNu7Y4J0TNnGcsV4S1LzECHbdgqk1hoHc2mFP1Axc37YUqwafg==" + }, "load-json-file": { "version": "2.0.0", "resolved": "http://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", @@ -2179,6 +2296,11 @@ "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz", "integrity": "sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=" }, + "map-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz", + "integrity": "sha1-ih8HiW2CsQkmvTdEokIACfiJdKg=" + }, "mem": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/mem/-/mem-1.1.0.tgz", @@ -2700,6 +2822,16 @@ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" }, + "object-extended": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/object-extended/-/object-extended-0.0.7.tgz", + "integrity": "sha1-hP0j9WsVWCrrPoiwXLVdJDLWijM=", + "requires": { + "array-extended": "~0.0.4", + "extended": "~0.0.3", + "is-extended": "~0.0.3" + } + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -2964,6 +3096,14 @@ } } }, + "pause-stream": { + "version": "0.0.11", + "resolved": "http://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", + "integrity": "sha1-/lo0sMvOErWqaitAPuLnO2AvFEU=", + "requires": { + "through": "~2.3" + } + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -3491,6 +3631,14 @@ "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.1.tgz", "integrity": "sha512-TfOfPcYGBB5sDuPn3deByxPhmfegAhpDYKSOXZQN81Oyrrif8ZCodOLzK3AesELnCx03kikhyDwh0pfvvQvF8w==" }, + "split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "requires": { + "through": "2" + } + }, "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -3530,6 +3678,15 @@ "resolved": "https://registry.npmjs.org/stackframe/-/stackframe-0.3.1.tgz", "integrity": "sha1-M6qE8Rd6VUjIk1Uzy/6zQgl19aQ=" }, + "stream-combiner": { + "version": "0.2.2", + "resolved": "http://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz", + "integrity": "sha1-rsjLrBd7Vrb0+kec7YwZEs7lKFg=", + "requires": { + "duplexer": "~0.1.1", + "through": "~2.3.4" + } + }, "stream-source": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/stream-source/-/stream-source-0.3.5.tgz", @@ -3540,6 +3697,17 @@ "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz", "integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=" }, + "string-extended": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/string-extended/-/string-extended-0.0.8.tgz", + "integrity": "sha1-dBlX3/SHsCcqee7FpE8jnubxfM0=", + "requires": { + "array-extended": "~0.0.5", + "date-extended": "~0.0.3", + "extended": "~0.0.3", + "is-extended": "~0.0.3" + } + }, "string-width": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", diff --git a/package.json b/package.json index 244998b..f8e8f37 100644 --- a/package.json +++ b/package.json @@ -12,17 +12,22 @@ }, "license": "MIT", "dependencies": { + "JSONStream": "^1.3.5", "available-versions": "^0.13.3", + "block-queue": "0.0.2", "colors": "1.3.2", "commander": "^2.14.1", "console.table": "0.10.0", "crypto-js": "3.1.9-1", "csvjson": "5.1.0", + "event-stream": "^4.0.1", + "fast-csv": "^2.4.1", "geojson-validation": "^0.2.1", "get-stdin": "6.0.0", "getmac": "1.4.3", "inquirer": "6.0.0", "latest-version": "4.0.0", + "line-by-line": "^0.1.6", "npm-check": "^5.7.1", "open": "0.0.5", "project-version": "1.2.0",