From 542ed3e205aa928ee7a6026dba040877941e5b90 Mon Sep 17 00:00:00 2001 From: "Nisar, Naitik" Date: Tue, 23 Oct 2018 18:49:11 +0530 Subject: [PATCH 01/23] creation of hexbin grid from space data creation of hexbin grid from space data --- bin/here-xyz.ts | 44 +++++++++++++++++++++++++++++++++++++++++++- package.json | 1 + 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index 2e8bd53..bec03ae 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -35,6 +35,8 @@ import * as fs from "fs"; import * as tmp from "tmp"; import * as summary from "./summary"; +//let hexbin = require('/Users/nisar/OneDrive - HERE Global B.V/home/github/hexbin'); +let hexbin = require('hexbin'); const request = require("request"); let choiceList: { name: string, value: string}[] = []; const questions = [ @@ -343,6 +345,45 @@ program })(); }); +program + .command('hexbin ') + .description('create hexgrid out of xyz space data and upload it to space') + .option("-c, --cellsize ", "size of hexgrid cell in meters") + .option("-i, --ids", "add ids of features as array inside property of created hexagon feature") + .action(function (id,options) { + (async () => { + try{ + options.totalRecords = Number.MAX_SAFE_INTEGER; + //options.token = 'Ef87rh2BTh29U-tyUx9NxQ'; + var features = await getSpaceDataFromXyz(id,options); + if (!options.cellsize) { + options.cellsize = 2000; + } + console.log("Creating hexbins for the space data"); + var hexFeatures = hexbin.calculateHexGrids(features, options.cellsize, options.ids); + console.log("uploading the hexagon grids to space"); + /* + fs.writeFile('out.json', JSON.stringify({type:"FeatureCollection",features:hexFeatures}), (err) => { + if (err) throw err; + }); + */ + + + var tmpObj = tmp.fileSync({ mode: 0o644, prefix: 'hex', postfix: '.json' }); + fs.writeFileSync(tmpObj.name, JSON.stringify({type:"FeatureCollection",features:hexFeatures})); + options.tags = 'hexbin'; + options.file = tmpObj.name; + options.override = true; + uploadToXyzSpace(id,options); + + + } catch (error) { + console.error(`hexbin creation failed: ${error}`); + process.exit(1); + } + })(); + }); + program .command("show ") .description("shows the content of the given [id]") @@ -1039,7 +1080,8 @@ common.validate( "describe", "clear", "token", - "analyze" + "analyze", + "hexbin" ], [process.argv[2]], program diff --git a/package.json b/package.json index 244998b..76c271f 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "geojson-validation": "^0.2.1", "get-stdin": "6.0.0", "getmac": "1.4.3", + "hexbin": "1.0.3", "inquirer": "6.0.0", "latest-version": "4.0.0", "npm-check": "^5.7.1", From 6d4613eb379e22ad53e05a7aadf98c7fdffaa7dc Mon Sep 17 00:00:00 2001 From: dhaneesh Date: Wed, 24 Oct 2018 19:42:32 +0530 Subject: [PATCH 02/23] streaming support for CSV and geojsonl streaming support for geojsonl and csv upload. Upload supportted with 10 simulaneous upload chunks --- bin/here-xyz.ts | 277 +++++++++++++++++++++++++++---------------- bin/transformutil.ts | 46 +++++++ package-lock.json | 95 +++++++++++++++ package.json | 2 + 4 files changed, 315 insertions(+), 105 deletions(-) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index 2e8bd53..ba59cef 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -34,6 +34,7 @@ import * as transform from "./transformutil"; import * as fs from "fs"; import * as tmp from "tmp"; import * as summary from "./summary"; +let cq = require("block-queue"); const request = require("request"); let choiceList: { name: string, value: string}[] = []; @@ -566,10 +567,39 @@ program "option to enforce uniqueness to the id by creating a hash of feature and use that as id" ) .option("-o, --override", "override the data even if it share same id") + .option("-s, --stream", "streaming data support for large file uploads") .action(function (id, options) { uploadToXyzSpace(id, options); }); +function collate(result:Array){ + return result.reduce((features: any, feature: any) => { + if (feature.type === "Feature") { + features.push(feature); + } else if (feature.type === "FeatureCollection") { + features = features.concat(feature.features); + } else { + console.log("Unknown type" + feature.type); + } + return features + }, []); +} + +function streamingQueue(){ + let queue = cq(10,function (task:any,done:Function) { + uploadData(task.id, task.options, task.tags, task.fc, + true, task.options.ptag, task.options.file, task.options.id) + .then(x=>{ + queue.uploadCount += task.fc.features.length; + console.log("uploaded feature count :"+queue.uploadCount); + done(); + }); + }); + queue.uploadCount=0; + return queue; +} + + function uploadToXyzSpace(id: string, options: any){ (async () => { let tags = ""; @@ -590,22 +620,27 @@ function uploadToXyzSpace(id: string, options: any){ options.unique = true; } + if(options.assign && options.stream){ + console.log( + "conflicting options together. You cannot choose assign mode while selecting streaming option" + ); + process.exit(1); + } + if (options.file) { const fs = require("fs"); if (options.file.indexOf(".geojsonl") != -1) { - transform.readLineFromFile(options.file, 100).then((result: any) => { - const totalFeatures = result.reduce((features: any, feature: any) => { - if (feature.type === "Feature") { - features.push(feature); - } else if (feature.type === "FeatureCollection") { - features = features.concat(feature.features); - } else { - console.log("Unknown type" + feature.type); - } - return features - }, []); - uploadData(id, options, tags, { type: "FeatureCollection", features: totalFeatures }, true, options.ptag, options.file, options.id); - }); + if(!options.stream){ + transform.readLineFromFile(options.file, 100).then((result: any) => { + uploadData(id, options, tags, { type: "FeatureCollection", features: collate(result) }, true, options.ptag, options.file, options.id); + }); + }else{ + let queue = streamingQueue(); + transform.readLineAsChunks(options.file, options.chunk?options.chunk:1000,function(result:any){ + if(result.length>0) + queue.push({id:id,options:options,tags:tags,fc:{ type: "FeatureCollection", features: collate(result) }}); + }); + } } else if (options.file.indexOf(".shp") != -1) { let result = await transform.readShapeFile( options.file, @@ -621,29 +656,48 @@ function uploadToXyzSpace(id: string, options: any){ options.id ); } else if (options.file.indexOf(".csv") != -1) { - let result = await transform.read( - options.file, - true - ); - const object = { - features: transform.transform( - result, - options.lat, - options.lon, - options.alt - ), - type: "FeatureCollection" - }; - uploadData( - id, - options, - tags, - object, - true, - options.ptag, + if(!options.stream){ + let result = await transform.read( options.file, - options.id - ); + true + ); + const object = { + features: transform.transform( + result, + options.lat, + options.lon, + options.alt + ), + type: "FeatureCollection" + }; + uploadData( + id, + options, + tags, + object, + true, + options.ptag, + options.file, + options.id + ); + }else{ + let queue = streamingQueue(); + transform.readCSVAsChunks(options.file, options.chunk?options.chunk:1000,function(result:any){ + if(result.length>0){ + const fc = { + features: transform.transform( + result, + options.lat, + options.lon, + options.alt + ), + type: "FeatureCollection" + }; + + queue.push({id:id,options:options,tags:tags,fc:fc}); + } + }); + } } else { let result = await transform.read( options.file, @@ -718,24 +772,39 @@ function uploadData( fileName: string | null, uid: string ) { - if (object.type == "Feature") { - object = { features: [object], type: "FeatureCollection" }; - } - if (options.assign) { - //console.log("assign mode on"); - const questions = createQuestionsList(object); - inquirer.prompt(questions).then((answers: any) => { - if (options.ptag === undefined) { - options.ptag = ""; - } - options.ptag = options.ptag + answers.tagChoices; - if (options.id === undefined) { - options.id = ""; - } - options.id = options.id + answers.idChoice; - //console.log(options.ptag); - //console.log("unique key - " + options.id); - //Need to be inside if, else this will be executed before user choice is inserted as its async + return new Promise((resolve, reject) => { + + if (object.type == "Feature") { + object = { features: [object], type: "FeatureCollection" }; + } + if (options.assign) { + //console.log("assign mode on"); + const questions = createQuestionsList(object); + inquirer.prompt(questions).then((answers: any) => { + if (options.ptag === undefined) { + options.ptag = ""; + } + options.ptag = options.ptag + answers.tagChoices; + if (options.id === undefined) { + options.id = ""; + } + options.id = options.id + answers.idChoice; + //console.log(options.ptag); + //console.log("unique key - " + options.id); + //Need to be inside if, else this will be executed before user choice is inserted as its async + uploadDataToSpaceWithTags( + id, + options, + tags, + object, + false, + options.ptag, + fileName, + options.id + ).then(x=>resolve(x)); + + }); + } else { uploadDataToSpaceWithTags( id, options, @@ -745,23 +814,14 @@ function uploadData( options.ptag, fileName, options.id - ); - }); - } else { - uploadDataToSpaceWithTags( - id, - options, - tags, - object, - false, - options.ptag, - fileName, - options.id - ); - } + ).then(x=>resolve(x)); + } + + }); + } -function uploadDataToSpaceWithTags( +async function uploadDataToSpaceWithTags( id: string, options: any, tags: any, @@ -771,46 +831,53 @@ function uploadDataToSpaceWithTags( fileName: string | null, uid: string ) { - const gsv = require("geojson-validation"); - gsv.valid(object, async function (valid: boolean, errs: any) { - if (!valid) { - console.log(errs); - return; - } - const featureOut = await mergeAllTags( - object.features, - tags, - tagProperties, - fileName, - uid, - options - ); - - const chunks = options.chunk - ? chunkify(featureOut, parseInt(options.chunk)) - : [featureOut]; - const chunkSize = chunks.length; - const index = 0; - await iterateChunks( - chunks, - "/hub/spaces/" + id + "/features", - index, - chunkSize, - ); - if (isFile) - console.log( - "'" + - options.file + - "' uploaded to xyzspace '" + - id + - "' successfully" + return new Promise((resolve, reject) => { + const gsv = require("geojson-validation"); + gsv.valid(object, async function (valid: boolean, errs: any) { + if (!valid) { + console.log(errs); + reject(errs); + return; + } + const featureOut = await mergeAllTags( + object.features, + tags, + tagProperties, + fileName, + uid, + options ); - else - console.log( - "data upload to xyzspace '" + id + "' completed successfully" + + const chunks = options.chunk + ? chunkify(featureOut, parseInt(options.chunk)) + : [featureOut]; + const chunkSize = chunks.length; + const index = 0; + await iterateChunks( + chunks, + "/hub/spaces/" + id + "/features", + index, + chunkSize, ); + if(!options.stream){ + if (isFile) + console.log( + "'" + + options.file + + "' uploaded to xyzspace '" + + id + + "' successfully" + ); + else + console.log( + "data upload to xyzspace '" + id + "' completed successfully" + ); - summary.summarize(featureOut, id, true); + summary.summarize(featureOut, id, true); + + } + resolve(true); + }); }); } diff --git a/bin/transformutil.ts b/bin/transformutil.ts index 1ceb86e..67e3fbe 100644 --- a/bin/transformutil.ts +++ b/bin/transformutil.ts @@ -30,6 +30,7 @@ import * as tmp from "tmp"; import * as request from "request"; import * as readline from "readline"; import { requestAsync } from "./requestAsync"; +import { deprecate } from "util"; const latArray = ["y", "ycoord", "ycoordinate", "coordy", "coordinatey", "latitude", "lat"]; const lonArray = ["x", "xcoord", "xcoordinate", "coordx", "coordinatex", "longitude", "lon"]; @@ -227,3 +228,48 @@ export function readLineFromFile(incomingPath: string, chunckSize = 100) { }); } + +export function readLineAsChunks(incomingPath: string, chunckSize:number,streamFuntion:Function) { + + return readData(incomingPath, 'geojsonl').then(path => { + return new Promise((resolve, reject) => { + let dataArray = new Array(); + const instream = fs.createReadStream(path); + const outstream = new (require('stream'))(); + const rl = readline.createInterface(instream, outstream); + + rl.on('line', (line: string) => { + dataArray.push(JSON.parse(line)); + if(dataArray.length>=chunckSize){ + streamFuntion(dataArray) + dataArray=new Array(); + } + }); + rl.on("error", err => console.log(err)); + rl.on('close', () => streamFuntion(dataArray)); + }); + }); +} + + +export function readCSVAsChunks(incomingPath: string, chunckSize:number,streamFuntion:Function) { + return readData(incomingPath, 'geojsonl').then(path => { + return new Promise((resolve, reject) => { + let dataArray = new Array(); + var csv = require("fast-csv"); + var stream = fs.createReadStream(incomingPath); + csv.fromStream(stream, {headers : true}).on("data", function(data:any){ + dataArray.push(data); + if(dataArray.length>=chunckSize){ + streamFuntion(dataArray) + dataArray=new Array(); + } + }).on("end", function(){ + streamFuntion(dataArray) + }); + }); + }); +} + + + diff --git a/package-lock.json b/package-lock.json index 8f85c76..6acd494 100644 --- a/package-lock.json +++ b/package-lock.json @@ -325,6 +325,25 @@ "sprintf-js": "~1.0.2" } }, + "arguments-extended": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/arguments-extended/-/arguments-extended-0.0.3.tgz", + "integrity": "sha1-YQfkkX0OtvCk3WYyD8Fa/HLvSUY=", + "requires": { + "extended": "~0.0.3", + "is-extended": "~0.0.8" + } + }, + "array-extended": { + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/array-extended/-/array-extended-0.0.11.tgz", + "integrity": "sha1-1xRK50jek8pybxIQCdv/FibRZL0=", + "requires": { + "arguments-extended": "~0.0.3", + "extended": "~0.0.3", + "is-extended": "~0.0.3" + } + }, "array-find-index": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/array-find-index/-/array-find-index-1.0.2.tgz", @@ -586,6 +605,11 @@ "tweetnacl": "^0.14.3" } }, + "block-queue": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/block-queue/-/block-queue-0.0.2.tgz", + "integrity": "sha1-HTjGN2yxmAGHOdnTFAw+BNxKzG8=" + }, "bluebird": { "version": "3.5.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.2.tgz", @@ -1013,6 +1037,16 @@ "assert-plus": "^1.0.0" } }, + "date-extended": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/date-extended/-/date-extended-0.0.6.tgz", + "integrity": "sha1-I4AtV90b94GIE/4MMuhRqG2iZ8k=", + "requires": { + "array-extended": "~0.0.3", + "extended": "~0.0.3", + "is-extended": "~0.0.3" + } + }, "debug": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", @@ -1026,6 +1060,11 @@ "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" }, + "declare.js": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/declare.js/-/declare.js-0.0.8.tgz", + "integrity": "sha1-BHit/5VkwAT1Hfc9i8E0AZ0o3N4=" + }, "decode-uri-component": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", @@ -1232,6 +1271,22 @@ "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, + "extended": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/extended/-/extended-0.0.6.tgz", + "integrity": "sha1-f7i/e52uOXWG5IVwrP1kLHjlBmk=", + "requires": { + "extender": "~0.0.5" + } + }, + "extender": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/extender/-/extender-0.0.10.tgz", + "integrity": "sha1-WJwHSCvmGhRgttgfnCSqZ+jzJM0=", + "requires": { + "declare.js": "~0.0.4" + } + }, "external-editor": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.0.3.tgz", @@ -1262,6 +1317,17 @@ "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", "integrity": "sha1-Ys8SAjTGg3hdkCNIqADvPgzCC8A=" }, + "fast-csv": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-2.4.1.tgz", + "integrity": "sha1-vX3SaDkfcpNntZRFuN0K0CaIGyY=", + "requires": { + "extended": "0.0.6", + "is-extended": "0.0.10", + "object-extended": "0.0.7", + "string-extended": "0.0.8" + } + }, "fast-deep-equal": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", @@ -1805,6 +1871,14 @@ "resolved": "https://registry.npmjs.org/is-es2016-keyword/-/is-es2016-keyword-1.0.0.tgz", "integrity": "sha1-9uVOEQxeT40mXmnS7Q6vjPX0dxg=" }, + "is-extended": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/is-extended/-/is-extended-0.0.10.tgz", + "integrity": "sha1-JE4UDfdbscmjEG9BL/GC+1NKbWI=", + "requires": { + "extended": "~0.0.3" + } + }, "is-finite": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-finite/-/is-finite-1.0.2.tgz", @@ -2700,6 +2774,16 @@ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" }, + "object-extended": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/object-extended/-/object-extended-0.0.7.tgz", + "integrity": "sha1-hP0j9WsVWCrrPoiwXLVdJDLWijM=", + "requires": { + "array-extended": "~0.0.4", + "extended": "~0.0.3", + "is-extended": "~0.0.3" + } + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -3540,6 +3624,17 @@ "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz", "integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=" }, + "string-extended": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/string-extended/-/string-extended-0.0.8.tgz", + "integrity": "sha1-dBlX3/SHsCcqee7FpE8jnubxfM0=", + "requires": { + "array-extended": "~0.0.5", + "date-extended": "~0.0.3", + "extended": "~0.0.3", + "is-extended": "~0.0.3" + } + }, "string-width": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", diff --git a/package.json b/package.json index 244998b..8856668 100644 --- a/package.json +++ b/package.json @@ -13,11 +13,13 @@ "license": "MIT", "dependencies": { "available-versions": "^0.13.3", + "block-queue": "0.0.2", "colors": "1.3.2", "commander": "^2.14.1", "console.table": "0.10.0", "crypto-js": "3.1.9-1", "csvjson": "5.1.0", + "fast-csv": "^2.4.1", "geojson-validation": "^0.2.1", "get-stdin": "6.0.0", "getmac": "1.4.3", From 86599a5037121041f95df755be012d079c15cfe6 Mon Sep 17 00:00:00 2001 From: dhaneesh Date: Wed, 24 Oct 2018 20:39:48 +0530 Subject: [PATCH 03/23] changed type from geojsonl to csv --- bin/transformutil.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/transformutil.ts b/bin/transformutil.ts index 67e3fbe..6596615 100644 --- a/bin/transformutil.ts +++ b/bin/transformutil.ts @@ -253,7 +253,7 @@ export function readLineAsChunks(incomingPath: string, chunckSize:number,streamF export function readCSVAsChunks(incomingPath: string, chunckSize:number,streamFuntion:Function) { - return readData(incomingPath, 'geojsonl').then(path => { + return readData(incomingPath, 'csv').then(path => { return new Promise((resolve, reject) => { let dataArray = new Array(); var csv = require("fast-csv"); From a8f4262b9618dc99ceace379de093649db77242b Mon Sep 17 00:00:00 2001 From: "Nisar, Naitik" Date: Thu, 25 Oct 2018 18:00:15 +0530 Subject: [PATCH 04/23] added cellsize as well in the tags added cellsize as well in the tags --- bin/here-xyz.ts | 5 ++--- package.json | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index bec03ae..7b413a9 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -363,20 +363,19 @@ program var hexFeatures = hexbin.calculateHexGrids(features, options.cellsize, options.ids); console.log("uploading the hexagon grids to space"); /* + /* fs.writeFile('out.json', JSON.stringify({type:"FeatureCollection",features:hexFeatures}), (err) => { if (err) throw err; }); */ - var tmpObj = tmp.fileSync({ mode: 0o644, prefix: 'hex', postfix: '.json' }); fs.writeFileSync(tmpObj.name, JSON.stringify({type:"FeatureCollection",features:hexFeatures})); - options.tags = 'hexbin'; + options.tags = 'hexbin_'+options.cellsize; options.file = tmpObj.name; options.override = true; uploadToXyzSpace(id,options); - } catch (error) { console.error(`hexbin creation failed: ${error}`); process.exit(1); diff --git a/package.json b/package.json index 76c271f..8d87157 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "geojson-validation": "^0.2.1", "get-stdin": "6.0.0", "getmac": "1.4.3", - "hexbin": "1.0.3", + "hexbin": "1.0.4", "inquirer": "6.0.0", "latest-version": "4.0.0", "npm-check": "^5.7.1", From f50dfde07a37fc58a1fa9bbdcee681374605f5f8 Mon Sep 17 00:00:00 2001 From: "Nisar, Naitik" Date: Fri, 26 Oct 2018 22:40:58 +0530 Subject: [PATCH 05/23] multiple fixes and enhancemnets 1. supporting creation of multiple hexbins at once 2. added centroid feature as a seperate tag 3. counts based on subproperty groupBy --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8d87157..ace368c 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "geojson-validation": "^0.2.1", "get-stdin": "6.0.0", "getmac": "1.4.3", - "hexbin": "1.0.4", + "hexbin": "1.0.6", "inquirer": "6.0.0", "latest-version": "4.0.0", "npm-check": "^5.7.1", From e896b42f372c40fc2c138fa8deba7d5342662d05 Mon Sep 17 00:00:00 2001 From: "Nisar, Naitik" Date: Fri, 26 Oct 2018 22:45:35 +0530 Subject: [PATCH 06/23] adding missing file adding missing file --- bin/here-xyz.ts | 76 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 22 deletions(-) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index 7b413a9..3606808 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -37,6 +37,7 @@ import * as summary from "./summary"; //let hexbin = require('/Users/nisar/OneDrive - HERE Global B.V/home/github/hexbin'); let hexbin = require('hexbin'); +const md5 = require('md5'); const request = require("request"); let choiceList: { name: string, value: string}[] = []; const questions = [ @@ -350,32 +351,63 @@ program .description('create hexgrid out of xyz space data and upload it to space') .option("-c, --cellsize ", "size of hexgrid cell in meters") .option("-i, --ids", "add ids of features as array inside property of created hexagon feature") + .option("-p, --groupBy ", "Name of the Property using which hexbin counts will be further grouped") .action(function (id,options) { (async () => { try{ - options.totalRecords = Number.MAX_SAFE_INTEGER; - //options.token = 'Ef87rh2BTh29U-tyUx9NxQ'; - var features = await getSpaceDataFromXyz(id,options); - if (!options.cellsize) { - options.cellsize = 2000; - } - console.log("Creating hexbins for the space data"); - var hexFeatures = hexbin.calculateHexGrids(features, options.cellsize, options.ids); - console.log("uploading the hexagon grids to space"); - /* - /* - fs.writeFile('out.json', JSON.stringify({type:"FeatureCollection",features:hexFeatures}), (err) => { - if (err) throw err; + options.totalRecords = Number.MAX_SAFE_INTEGER; + //options.token = 'Ef87rh2BTh29U-tyUx9NxQ'; + const features = await getSpaceDataFromXyz(id,options); + let cellSizes:number[] = []; + if (!options.cellsize) { + cellSizes.push(2000); + } else { + options.cellsize.split(",").forEach(function (item : string) { + if (item && item != "") { + let number = parseInt(item.toLowerCase()); + if (isNaN(number)){ + console.error(`hexbin creation failed: cellsize input "${item}" is not a valid number`); + process.exit(1); + } + cellSizes.push(number); + } + }); + } + + cellSizes.forEach(function (cellsize : number) { + // (async () => { + console.log("Creating hexbins for the space data with size " + cellsize); + let hexFeatures = hexbin.calculateHexGrids(features, cellsize, options.ids, options.groupBy); + console.log("uploading the hexagon grids to space with size " + cellsize); + + let centroidFeatures:any[] = []; + hexFeatures.forEach(function (hexFeature : any) { + let geometry = {"type":"Point","coordinates":hexFeature.properties.centroid}; + let hashId = md5(JSON.stringify(geometry)); + centroidFeatures.push({type:"Feature","geometry":geometry,"properties":hexFeature.properties,"id":hashId}); + }); + //hexFeatures = hexFeatures.concat(centroidFeatures); + /* + fs.writeFile('out.json', JSON.stringify({type:"FeatureCollection",features:hexFeatures}), (err) => { + if (err) throw err; + }); + */ + + let tmpObj = tmp.fileSync({ mode: 0o644, prefix: 'hex', postfix: '.json' }); + fs.writeFileSync(tmpObj.name, JSON.stringify({type:"FeatureCollection",features:hexFeatures})); + options.tags = 'hexbin_'+cellsize; + options.file = tmpObj.name; + options.override = true; + uploadToXyzSpace(id,options); + + tmpObj = tmp.fileSync({ mode: 0o644, prefix: 'hex', postfix: '.json' }); + fs.writeFileSync(tmpObj.name, JSON.stringify({type:"FeatureCollection",features:centroidFeatures})); + options.tags = 'centroid_'+cellsize+',hexbin_'+cellsize; + options.file = tmpObj.name; + options.override = true; + uploadToXyzSpace(id,options); + //}); }); - */ - - var tmpObj = tmp.fileSync({ mode: 0o644, prefix: 'hex', postfix: '.json' }); - fs.writeFileSync(tmpObj.name, JSON.stringify({type:"FeatureCollection",features:hexFeatures})); - options.tags = 'hexbin_'+options.cellsize; - options.file = tmpObj.name; - options.override = true; - uploadToXyzSpace(id,options); - } catch (error) { console.error(`hexbin creation failed: ${error}`); process.exit(1); From 59ed432709fee9b5534e908346f06e3b3bb18af7 Mon Sep 17 00:00:00 2001 From: dhaneesh Date: Sat, 27 Oct 2018 04:41:44 +0530 Subject: [PATCH 07/23] Backpressure implementation while streaming to XYZ --- bin/here-xyz.ts | 51 ++++++++++++++++++++++++++++++-------------- bin/transformutil.ts | 24 ++++++++++++--------- package-lock.json | 5 +++++ package.json | 1 + 4 files changed, 55 insertions(+), 26 deletions(-) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index ba59cef..5c1c122 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -592,10 +592,19 @@ function streamingQueue(){ .then(x=>{ queue.uploadCount += task.fc.features.length; console.log("uploaded feature count :"+queue.uploadCount); + queue.chunksize--; done(); }); }); queue.uploadCount=0; + queue.chunksize=0; + queue.send= async function(obj:any){ + while(this.chunksize>25){ + await new Promise(done => setTimeout(done, 1000)); + } + this.push(obj); + this.chunksize++; + } return queue; } @@ -637,8 +646,14 @@ function uploadToXyzSpace(id: string, options: any){ }else{ let queue = streamingQueue(); transform.readLineAsChunks(options.file, options.chunk?options.chunk:1000,function(result:any){ - if(result.length>0) - queue.push({id:id,options:options,tags:tags,fc:{ type: "FeatureCollection", features: collate(result) }}); + return new Promise((res,rej)=>{ + ( async()=>{ + if(result.length>0){ + await queue.send({id:id,options:options,tags:tags,fc:{ type: "FeatureCollection", features: collate(result) }}); + } + res(); + })(); + }); }); } } else if (options.file.indexOf(".shp") != -1) { @@ -683,19 +698,24 @@ function uploadToXyzSpace(id: string, options: any){ }else{ let queue = streamingQueue(); transform.readCSVAsChunks(options.file, options.chunk?options.chunk:1000,function(result:any){ - if(result.length>0){ - const fc = { - features: transform.transform( - result, - options.lat, - options.lon, - options.alt - ), - type: "FeatureCollection" - }; - - queue.push({id:id,options:options,tags:tags,fc:fc}); - } + return new Promise((res,rej)=>{ + ( async()=>{ + if(result.length>0){ + const fc = { + features: transform.transform( + result, + options.lat, + options.lon, + options.alt + ), + type: "FeatureCollection" + }; + await queue.send({id:id,options:options,tags:tags,fc:fc}); + res(); + } + })(); + }); + }); } } else { @@ -832,7 +852,6 @@ async function uploadDataToSpaceWithTags( uid: string ) { return new Promise((resolve, reject) => { - const gsv = require("geojson-validation"); gsv.valid(object, async function (valid: boolean, errs: any) { if (!valid) { console.log(errs); diff --git a/bin/transformutil.ts b/bin/transformutil.ts index 6596615..fd85f3a 100644 --- a/bin/transformutil.ts +++ b/bin/transformutil.ts @@ -230,23 +230,27 @@ export function readLineFromFile(incomingPath: string, chunckSize = 100) { export function readLineAsChunks(incomingPath: string, chunckSize:number,streamFuntion:Function) { - return readData(incomingPath, 'geojsonl').then(path => { return new Promise((resolve, reject) => { let dataArray = new Array(); - const instream = fs.createReadStream(path); - const outstream = new (require('stream'))(); - const rl = readline.createInterface(instream, outstream); - - rl.on('line', (line: string) => { + var LineByLineReader = require('line-by-line'), + lr = new LineByLineReader(path); + lr.on('error', function (err:any) { + console.log(err); + throw err; + }); + lr.on('line', async function (line:any) { dataArray.push(JSON.parse(line)); if(dataArray.length>=chunckSize){ - streamFuntion(dataArray) + lr.pause(); + await streamFuntion(dataArray); + lr.resume(); dataArray=new Array(); } }); - rl.on("error", err => console.log(err)); - rl.on('close', () => streamFuntion(dataArray)); + lr.on('end', function () { + streamFuntion(dataArray) + }); }); }); } @@ -258,7 +262,7 @@ export function readCSVAsChunks(incomingPath: string, chunckSize:number,streamFu let dataArray = new Array(); var csv = require("fast-csv"); var stream = fs.createReadStream(incomingPath); - csv.fromStream(stream, {headers : true}).on("data", function(data:any){ + let csvstream = csv.fromStream(stream, {headers : true}).on("data", function(data:any){ dataArray.push(data); if(dataArray.length>=chunckSize){ streamFuntion(dataArray) diff --git a/package-lock.json b/package-lock.json index 6acd494..5947280 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2139,6 +2139,11 @@ "invert-kv": "^1.0.0" } }, + "line-by-line": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/line-by-line/-/line-by-line-0.1.6.tgz", + "integrity": "sha512-MmwVPfOyp0lWnEZ3fBA8Ah4pMFvxO6WgWovqZNu7Y4J0TNnGcsV4S1LzECHbdgqk1hoHc2mFP1Axc37YUqwafg==" + }, "load-json-file": { "version": "2.0.0", "resolved": "http://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", diff --git a/package.json b/package.json index 8856668..c334f2f 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "getmac": "1.4.3", "inquirer": "6.0.0", "latest-version": "4.0.0", + "line-by-line": "^0.1.6", "npm-check": "^5.7.1", "open": "0.0.5", "project-version": "1.2.0", From c78186d7d902577ca00989d784972a25ebc6a083 Mon Sep 17 00:00:00 2001 From: dhaneesh Date: Sat, 27 Oct 2018 04:55:40 +0530 Subject: [PATCH 08/23] minor updates --- bin/here-xyz.ts | 1 + bin/transformutil.ts | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index 5c1c122..e9ec48c 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -35,6 +35,7 @@ import * as fs from "fs"; import * as tmp from "tmp"; import * as summary from "./summary"; let cq = require("block-queue"); +const gsv = require("geojson-validation"); const request = require("request"); let choiceList: { name: string, value: string}[] = []; diff --git a/bin/transformutil.ts b/bin/transformutil.ts index fd85f3a..4e38bf1 100644 --- a/bin/transformutil.ts +++ b/bin/transformutil.ts @@ -264,9 +264,14 @@ export function readCSVAsChunks(incomingPath: string, chunckSize:number,streamFu var stream = fs.createReadStream(incomingPath); let csvstream = csv.fromStream(stream, {headers : true}).on("data", function(data:any){ dataArray.push(data); - if(dataArray.length>=chunckSize){ - streamFuntion(dataArray) - dataArray=new Array(); + if(dataArray.length >=chunckSize){ + //console.log('dataArray '+chunckSize); + csvstream.pause(); + (async()=>{ + await streamFuntion(dataArray); + csvstream.resume(); + dataArray=new Array(); + })(); } }).on("end", function(){ streamFuntion(dataArray) From 82a90638ad24f71b107f2b86f60d371835b29818 Mon Sep 17 00:00:00 2001 From: "Nisar, Naitik" Date: Thu, 1 Nov 2018 18:47:28 +0530 Subject: [PATCH 09/23] adding cellsize as well while creating hash id of centroid point adding cellsize as well while creating hash id of centroid point --- bin/here-xyz.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index 3606808..6a6ee93 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -383,7 +383,7 @@ program let centroidFeatures:any[] = []; hexFeatures.forEach(function (hexFeature : any) { let geometry = {"type":"Point","coordinates":hexFeature.properties.centroid}; - let hashId = md5(JSON.stringify(geometry)); + let hashId = md5(JSON.stringify(geometry)+'_'+cellsize); centroidFeatures.push({type:"Feature","geometry":geometry,"properties":hexFeature.properties,"id":hashId}); }); //hexFeatures = hexFeatures.concat(centroidFeatures); From 31951c04cb3115293e2a54d9f417e3c854e83ad0 Mon Sep 17 00:00:00 2001 From: "Nisar, Naitik" Date: Thu, 1 Nov 2018 19:32:30 +0530 Subject: [PATCH 10/23] updated the async behavior uploadToXyzSpace so that hexbins are uploaded in controlled manner updated the async behavior uploadToXyzSpace so that hexbins are uploaded in controlled manner --- bin/here-xyz.ts | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index 28d3591..49f52e3 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -376,8 +376,9 @@ program }); } - cellSizes.forEach(function (cellsize : number) { - // (async () => { + //cellSizes.forEach(function (cellsize : number) { + for(const cellsize of cellSizes){ + //(async () => { console.log("Creating hexbins for the space data with size " + cellsize); let hexFeatures = hexbin.calculateHexGrids(features, cellsize, options.ids, options.groupBy); console.log("uploading the hexagon grids to space with size " + cellsize); @@ -400,16 +401,17 @@ program options.tags = 'hexbin_'+cellsize; options.file = tmpObj.name; options.override = true; - uploadToXyzSpace(id,options); + await uploadToXyzSpace(id,options); tmpObj = tmp.fileSync({ mode: 0o644, prefix: 'hex', postfix: '.json' }); fs.writeFileSync(tmpObj.name, JSON.stringify({type:"FeatureCollection",features:centroidFeatures})); options.tags = 'centroid_'+cellsize+',hexbin_'+cellsize; options.file = tmpObj.name; options.override = true; - uploadToXyzSpace(id,options); + await uploadToXyzSpace(id,options); + //}); + } //}); - }); } catch (error) { console.error(`hexbin creation failed: ${error}`); process.exit(1); @@ -682,8 +684,8 @@ function streamingQueue(){ } -function uploadToXyzSpace(id: string, options: any){ - (async () => { +async function uploadToXyzSpace(id: string, options: any){ + //(async () => { let tags = ""; if (options.tags) { tags = options.tags; @@ -733,7 +735,7 @@ function uploadToXyzSpace(id: string, options: any){ let result = await transform.readShapeFile( options.file, ); - uploadData( + await uploadData( id, options, tags, @@ -758,7 +760,7 @@ function uploadToXyzSpace(id: string, options: any){ ), type: "FeatureCollection" }; - uploadData( + await uploadData( id, options, tags, @@ -796,7 +798,7 @@ function uploadToXyzSpace(id: string, options: any){ options.file, false ); - uploadData( + await uploadData( id, options, tags, @@ -829,7 +831,7 @@ function uploadToXyzSpace(id: string, options: any){ } }); } - })(); + //})(); } function createQuestionsList(object: any) { From f01a792dd282317d504a37eac383c8554edeb069 Mon Sep 17 00:00:00 2001 From: dhaneesh Date: Fri, 2 Nov 2018 15:56:17 +0530 Subject: [PATCH 11/23] streaming changes with retry --- bin/here-xyz.ts | 48 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index e9ec48c..2f97ce6 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -124,7 +124,8 @@ async function execInternalGzip( method: string, contentType: string, data: any, - token: string + token: string, + retry : number=3 ) { const zippedData = await gzip(data); const isJson = contentType == "application/json" ? true : false; @@ -143,9 +144,15 @@ async function execInternalGzip( body: method === "GET" ? undefined : zippedData }; - const { response, body } = await requestAsync(reqJson); - if (response.statusCode < 200 || response.statusCode > 210) - throw new Error("Invalid response"); + let { response, body } = await requestAsync(reqJson); + if (response.statusCode < 200 || response.statusCode > 210){ + if(response.statusCode>=500){ + await new Promise(done => setTimeout(done, 1000)); + body = execInternalGzip(uri,method,contentType,data,token,retry--); + }else{ + throw new Error("Invalid response :"+response.statusCode); + } + } return body; } @@ -595,10 +602,16 @@ function streamingQueue(){ console.log("uploaded feature count :"+queue.uploadCount); queue.chunksize--; done(); - }); + }).catch((err) => { + queue.failedCount += task.fc.features.length; + console.log("failed feature count :"+queue.failedCount); + queue.chunksize--; + done(); + }); }); queue.uploadCount=0; queue.chunksize=0; + queue.failedCount=0; queue.send= async function(obj:any){ while(this.chunksize>25){ await new Promise(done => setTimeout(done, 1000)); @@ -650,7 +663,7 @@ function uploadToXyzSpace(id: string, options: any){ return new Promise((res,rej)=>{ ( async()=>{ if(result.length>0){ - await queue.send({id:id,options:options,tags:tags,fc:{ type: "FeatureCollection", features: collate(result) }}); + await queue.send({id:id,options:options,tags:tags,fc:{ type: "FeatureCollection", features: collate(result) },retryCount:3}); } res(); })(); @@ -711,7 +724,7 @@ function uploadToXyzSpace(id: string, options: any){ ), type: "FeatureCollection" }; - await queue.send({id:id,options:options,tags:tags,fc:fc}); + await queue.send({id:id,options:options,tags:tags,fc:fc,retryCount:3}); res(); } })(); @@ -822,7 +835,7 @@ function uploadData( options.ptag, fileName, options.id - ).then(x=>resolve(x)); + ).then(x=>resolve(x)).catch((error) => reject(error)); }); } else { @@ -835,7 +848,7 @@ function uploadData( options.ptag, fileName, options.id - ).then(x=>resolve(x)); + ).then(x=>resolve(x)).catch((error) => reject(error)); } }); @@ -873,12 +886,17 @@ async function uploadDataToSpaceWithTags( : [featureOut]; const chunkSize = chunks.length; const index = 0; - await iterateChunks( - chunks, - "/hub/spaces/" + id + "/features", - index, - chunkSize, - ); + try{ + await iterateChunks( + chunks, + "/hub/spaces/" + id + "/features", + index, + chunkSize, + ); + }catch(e){ + reject(e); + return; + } if(!options.stream){ if (isFile) console.log( From fbb663b112bab7abd950999caea5b88e92a74aa5 Mon Sep 17 00:00:00 2001 From: "Nisar, Naitik" Date: Fri, 2 Nov 2018 18:58:21 +0530 Subject: [PATCH 12/23] supporting readToken,writeToken,targetSpace and fixing message in failure scenario supporting readToken,writeToken,targetSpace and fixing message in failure scenario --- bin/here-xyz.ts | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index 49f52e3..280b687 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -107,7 +107,7 @@ async function execInternal( const { response, body } = await requestAsync(reqJson); if (response.statusCode < 200 || response.statusCode > 210) - throw new Error("Invalid response"); + throw new Error("Invalid response - " + response.body); return body; } @@ -148,7 +148,7 @@ async function execInternalGzip( const { response, body } = await requestAsync(reqJson); if (response.statusCode < 200 || response.statusCode > 210) - throw new Error("Invalid response"); + throw new Error("Invalid response - " + response.body); return body; } @@ -354,11 +354,17 @@ program .option("-c, --cellsize ", "size of hexgrid cell in meters") .option("-i, --ids", "add ids of features as array inside property of created hexagon feature") .option("-p, --groupBy ", "Name of the Property using which hexbin counts will be further grouped") + .option("-r, --readToken ", "Token to access source space") + .option("-w, --writeToken ", "Token to access Target space where hexbins will be written") + .option("-t, --targetSpace ", "Target Space name where hexbins and centroids will be uploaded") .action(function (id,options) { (async () => { try{ options.totalRecords = Number.MAX_SAFE_INTEGER; //options.token = 'Ef87rh2BTh29U-tyUx9NxQ'; + if(options.readToken){ + options.token = options.readToken; + } const features = await getSpaceDataFromXyz(id,options); let cellSizes:number[] = []; if (!options.cellsize) { @@ -375,7 +381,13 @@ program } }); } - + options.token = null; + if(options.writeToken){ + options.token = options.writeToken; + } + if(options.targetSpace){ + id = options.targetSpace; + } //cellSizes.forEach(function (cellsize : number) { for(const cellsize of cellSizes){ //(async () => { @@ -398,14 +410,14 @@ program let tmpObj = tmp.fileSync({ mode: 0o644, prefix: 'hex', postfix: '.json' }); fs.writeFileSync(tmpObj.name, JSON.stringify({type:"FeatureCollection",features:hexFeatures})); - options.tags = 'hexbin_'+cellsize; + options.tags = 'hexbin_'+cellsize+',cell_'+cellsize+',hexbin'; options.file = tmpObj.name; options.override = true; await uploadToXyzSpace(id,options); tmpObj = tmp.fileSync({ mode: 0o644, prefix: 'hex', postfix: '.json' }); fs.writeFileSync(tmpObj.name, JSON.stringify({type:"FeatureCollection",features:centroidFeatures})); - options.tags = 'centroid_'+cellsize+',hexbin_'+cellsize; + options.tags = 'centroid_'+cellsize+',cell_'+cellsize+',centroid'; options.file = tmpObj.name; options.override = true; await uploadToXyzSpace(id,options); @@ -952,6 +964,7 @@ async function uploadDataToSpaceWithTags( "/hub/spaces/" + id + "/features", index, chunkSize, + options.token ); if(!options.stream){ if (isFile) @@ -1143,7 +1156,7 @@ function getFileName(fileName: string) { } } -async function iterateChunks(chunks: any, url: string, index: number, chunkSize: number) { +async function iterateChunks(chunks: any, url: string, index: number, chunkSize: number, token: string) { const item = chunks.shift(); const fc = { type: "FeatureCollection", features: item }; const body = await execute( @@ -1151,7 +1164,7 @@ async function iterateChunks(chunks: any, url: string, index: number, chunkSize: "PUT", "application/geo+json", JSON.stringify(fc), - null, + token, true ); @@ -1161,7 +1174,7 @@ async function iterateChunks(chunks: any, url: string, index: number, chunkSize: } console.log("uploaded " + ((index / chunkSize) * 100).toFixed(2) + "%"); - await iterateChunks(chunks, url, index, chunkSize); + await iterateChunks(chunks, url, index, chunkSize, token); } function chunkify(data: any[], chunksize: number) { From cd4ae3edec94fe3e5d65b8895e9796f43b91846d Mon Sep 17 00:00:00 2001 From: "Nisar, Naitik" Date: Sat, 3 Nov 2018 21:42:16 +0530 Subject: [PATCH 13/23] Fixing issue of upload failure when hexbin created are out of permissable lat long range Fixing issue of upload failure when hexbin created are out of permissable lat long range --- bin/here-xyz.ts | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index 280b687..d5dabe2 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -396,11 +396,25 @@ program console.log("uploading the hexagon grids to space with size " + cellsize); let centroidFeatures:any[] = []; - hexFeatures.forEach(function (hexFeature : any) { - let geometry = {"type":"Point","coordinates":hexFeature.properties.centroid}; - let hashId = md5(JSON.stringify(geometry)+'_'+cellsize); - centroidFeatures.push({type:"Feature","geometry":geometry,"properties":hexFeature.properties,"id":hashId}); - }); + var i = hexFeatures.length; + while (i--) { + let isValidHexagon = true; + hexFeatures[i].geometry.coordinates[0].forEach(function (coordinate : Array) { + if(coordinate[0] < -180 || coordinate[0] > 180 || coordinate[1] > 90 || coordinate[1] < -90){ + isValidHexagon = false; + console.log("Invalid hexagon which is created outside of permissable range - " + coordinate); + } + }); + if (isValidHexagon) { + let geometry = {"type":"Point","coordinates":hexFeatures[i].properties.centroid}; + let hashId = md5(JSON.stringify(geometry)+'_'+cellsize); + centroidFeatures.push({type:"Feature","geometry":geometry,"properties":hexFeatures[i].properties,"id":hashId}); + } else { + console.log("Invalid hexagon is getting created, ignoring this - " + JSON.stringify(hexFeatures[i])); + hexFeatures.splice(i, 1); + } + } + //hexFeatures = hexFeatures.concat(centroidFeatures); /* fs.writeFile('out.json', JSON.stringify({type:"FeatureCollection",features:hexFeatures}), (err) => { From 9c1aa5079477faa9b312d693d20c1ff797cb9e5b Mon Sep 17 00:00:00 2001 From: "Nisar, Naitik" Date: Sat, 3 Nov 2018 21:51:37 +0530 Subject: [PATCH 14/23] changing the option -t to -d for destaination space id changing the option -t to -d for destaination space id --- bin/here-xyz.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index d5dabe2..4e11362 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -356,7 +356,7 @@ program .option("-p, --groupBy ", "Name of the Property using which hexbin counts will be further grouped") .option("-r, --readToken ", "Token to access source space") .option("-w, --writeToken ", "Token to access Target space where hexbins will be written") - .option("-t, --targetSpace ", "Target Space name where hexbins and centroids will be uploaded") + .option("-d, --destSpace ", "Destination Space name where hexbins and centroids will be uploaded") .action(function (id,options) { (async () => { try{ @@ -385,8 +385,8 @@ program if(options.writeToken){ options.token = options.writeToken; } - if(options.targetSpace){ - id = options.targetSpace; + if(options.destSpace){ + id = options.destSpace; } //cellSizes.forEach(function (cellsize : number) { for(const cellsize of cellSizes){ From a649f5d5dc6c8eb6c0628b0a09e32a1298085b26 Mon Sep 17 00:00:00 2001 From: "Nisar, Naitik" Date: Sat, 3 Nov 2018 22:53:56 +0530 Subject: [PATCH 15/23] give support of reading source space with tags give support of reading source space with tags --- bin/here-xyz.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index 4e11362..9655a11 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -357,6 +357,7 @@ program .option("-r, --readToken ", "Token to access source space") .option("-w, --writeToken ", "Token to access Target space where hexbins will be written") .option("-d, --destSpace ", "Destination Space name where hexbins and centroids will be uploaded") + .option("-t, --tags ", "Hexbins will be created for those records only which matches the tag value in source space ") .action(function (id,options) { (async () => { try{ @@ -366,6 +367,10 @@ program options.token = options.readToken; } const features = await getSpaceDataFromXyz(id,options); + if(features.length === 0){ + console.log("No features is available to create hexbins"); + process.exit(); + } let cellSizes:number[] = []; if (!options.cellsize) { cellSizes.push(2000); From eaee51711d494e47f235ae65ee715bf9bd9cec58 Mon Sep 17 00:00:00 2001 From: "Nisar, Naitik" Date: Mon, 5 Nov 2018 14:23:22 +0530 Subject: [PATCH 16/23] adding source space ID as a tag when writing hexbins to different space adding source space ID as a tag when writing hexbins to different space --- bin/here-xyz.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index 9655a11..ca44fe4 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -361,6 +361,7 @@ program .action(function (id,options) { (async () => { try{ + const sourceId = id; options.totalRecords = Number.MAX_SAFE_INTEGER; //options.token = 'Ef87rh2BTh29U-tyUx9NxQ'; if(options.readToken){ @@ -430,6 +431,9 @@ program let tmpObj = tmp.fileSync({ mode: 0o644, prefix: 'hex', postfix: '.json' }); fs.writeFileSync(tmpObj.name, JSON.stringify({type:"FeatureCollection",features:hexFeatures})); options.tags = 'hexbin_'+cellsize+',cell_'+cellsize+',hexbin'; + if(options.destSpace){ + options.tags += ','+sourceId; + } options.file = tmpObj.name; options.override = true; await uploadToXyzSpace(id,options); @@ -437,6 +441,9 @@ program tmpObj = tmp.fileSync({ mode: 0o644, prefix: 'hex', postfix: '.json' }); fs.writeFileSync(tmpObj.name, JSON.stringify({type:"FeatureCollection",features:centroidFeatures})); options.tags = 'centroid_'+cellsize+',cell_'+cellsize+',centroid'; + if(options.destSpace){ + options.tags += ','+sourceId; + } options.file = tmpObj.name; options.override = true; await uploadToXyzSpace(id,options); From 2e449e335aac6c3a652b78a4519cabc138974817 Mon Sep 17 00:00:00 2001 From: dhaneesh Date: Tue, 13 Nov 2018 13:27:18 +0530 Subject: [PATCH 17/23] changed for streaming geojson file --- bin/here-xyz.ts | 46 +++++++++++++++++++++--------- bin/transformutil.ts | 33 ++++++++++++++++++++- package-lock.json | 68 ++++++++++++++++++++++++++++++++++++++++++++ package.json | 2 ++ 4 files changed, 134 insertions(+), 15 deletions(-) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index 2f97ce6..60ba0c0 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -733,20 +733,38 @@ function uploadToXyzSpace(id: string, options: any){ }); } } else { - let result = await transform.read( - options.file, - false - ); - uploadData( - id, - options, - tags, - JSON.parse(result), - true, - options.ptag, - options.file, - options.id - ); + if(!options.stream){ + let result = await transform.read( + options.file, + false + ); + uploadData( + id, + options, + tags, + JSON.parse(result), + true, + options.ptag, + options.file, + options.id + ); + }else{ + let queue = streamingQueue(); + transform.readGeoJsonAsChunks(options.file, options.chunk?options.chunk:1000,function(result:any){ + return new Promise((res,rej)=>{ + ( async()=>{ + if(result.length>0){ + const fc = { + features: result, + type: "FeatureCollection" + }; + await queue.send({id:id,options:options,tags:tags,fc:fc,retryCount:3}); + res(); + } + })(); + }); + }); + } } } else { const getStdin = require("get-stdin"); diff --git a/bin/transformutil.ts b/bin/transformutil.ts index 4e38bf1..8758137 100644 --- a/bin/transformutil.ts +++ b/bin/transformutil.ts @@ -261,7 +261,7 @@ export function readCSVAsChunks(incomingPath: string, chunckSize:number,streamFu return new Promise((resolve, reject) => { let dataArray = new Array(); var csv = require("fast-csv"); - var stream = fs.createReadStream(incomingPath); + var stream = fs.createReadStream(path); let csvstream = csv.fromStream(stream, {headers : true}).on("data", function(data:any){ dataArray.push(data); if(dataArray.length >=chunckSize){ @@ -282,3 +282,34 @@ export function readCSVAsChunks(incomingPath: string, chunckSize:number,streamFu +export function readGeoJsonAsChunks(incomingPath: string, chunckSize:number,streamFuntion:Function) { + return readData(incomingPath, 'geojson').then(path => { + return new Promise((resolve, reject) => { + let dataArray = new Array(); + const JSONStream = require('JSONStream'); + const es = require('event-stream'); + const fileStream = fs.createReadStream(path, {encoding: 'utf8'}); + let stream = fileStream.pipe(JSONStream.parse('features.*')); + stream.pipe(es.through(async function (data:any) { + if(dataArray.length >=chunckSize){ + stream.pause(); + fileStream.pause(); + await streamFuntion(dataArray); + stream.resume(); + fileStream.resume(); + dataArray=new Array(); + } + dataArray.push(data); + return data; + },function end () { + if(dataArray.length >0){ + (async()=>{ + await streamFuntion(dataArray); + dataArray=new Array(); + })(); + } + })); + }); + }); +} + \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 5947280..06a414d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -230,6 +230,15 @@ "integrity": "sha512-MDQLxNFRLasqS4UlkWMSACMKeSm1x4Q3TxzUC7KQUsh6RK1ZrQ0VEyE3yzXcBu+K8ejVj4wuX32eUG02yNp+YQ==", "dev": true }, + "JSONStream": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz", + "integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==", + "requires": { + "jsonparse": "^1.2.0", + "through": ">=2.2.7 <3" + } + }, "ajv": { "version": "5.5.2", "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", @@ -1163,6 +1172,11 @@ "is-obj": "^1.0.0" } }, + "duplexer": { + "version": "0.1.1", + "resolved": "http://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz", + "integrity": "sha1-rOb/gIwc5mtX0ev5eXessCM0z8E=" + }, "duplexer3": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", @@ -1239,6 +1253,20 @@ "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=" }, + "event-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-4.0.1.tgz", + "integrity": "sha512-qACXdu/9VHPBzcyhdOWR5/IahhGMf0roTeZJfzz077GwylcDd90yOHLouhmv7GJ5XzPi6ekaQWd8AvPP2nOvpA==", + "requires": { + "duplexer": "^0.1.1", + "from": "^0.1.7", + "map-stream": "0.0.7", + "pause-stream": "^0.0.11", + "split": "^1.0.1", + "stream-combiner": "^0.2.2", + "through": "^2.3.8" + } + }, "execa": { "version": "0.10.0", "resolved": "https://registry.npmjs.org/execa/-/execa-0.10.0.tgz", @@ -1395,6 +1423,11 @@ } } }, + "from": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", + "integrity": "sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4=" + }, "from2": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", @@ -2049,6 +2082,11 @@ "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, + "jsonparse": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", + "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=" + }, "jsprim": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", @@ -2258,6 +2296,11 @@ "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz", "integrity": "sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=" }, + "map-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz", + "integrity": "sha1-ih8HiW2CsQkmvTdEokIACfiJdKg=" + }, "mem": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/mem/-/mem-1.1.0.tgz", @@ -3053,6 +3096,14 @@ } } }, + "pause-stream": { + "version": "0.0.11", + "resolved": "http://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", + "integrity": "sha1-/lo0sMvOErWqaitAPuLnO2AvFEU=", + "requires": { + "through": "~2.3" + } + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -3580,6 +3631,14 @@ "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.1.tgz", "integrity": "sha512-TfOfPcYGBB5sDuPn3deByxPhmfegAhpDYKSOXZQN81Oyrrif8ZCodOLzK3AesELnCx03kikhyDwh0pfvvQvF8w==" }, + "split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "requires": { + "through": "2" + } + }, "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -3619,6 +3678,15 @@ "resolved": "https://registry.npmjs.org/stackframe/-/stackframe-0.3.1.tgz", "integrity": "sha1-M6qE8Rd6VUjIk1Uzy/6zQgl19aQ=" }, + "stream-combiner": { + "version": "0.2.2", + "resolved": "http://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz", + "integrity": "sha1-rsjLrBd7Vrb0+kec7YwZEs7lKFg=", + "requires": { + "duplexer": "~0.1.1", + "through": "~2.3.4" + } + }, "stream-source": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/stream-source/-/stream-source-0.3.5.tgz", diff --git a/package.json b/package.json index c334f2f..f8e8f37 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ }, "license": "MIT", "dependencies": { + "JSONStream": "^1.3.5", "available-versions": "^0.13.3", "block-queue": "0.0.2", "colors": "1.3.2", @@ -19,6 +20,7 @@ "console.table": "0.10.0", "crypto-js": "3.1.9-1", "csvjson": "5.1.0", + "event-stream": "^4.0.1", "fast-csv": "^2.4.1", "geojson-validation": "^0.2.1", "get-stdin": "6.0.0", From b90152ff7e2bd0830588f79ecae476a5e6f673ef Mon Sep 17 00:00:00 2001 From: dhaneesh Date: Tue, 13 Nov 2018 18:30:36 +0530 Subject: [PATCH 18/23] Parallel uploads for all type of uploads --- bin/here-xyz.ts | 83 ++++++++++++++++++++++++++++++++++---------- bin/transformutil.ts | 4 +-- 2 files changed, 66 insertions(+), 21 deletions(-) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index 60ba0c0..1d27e39 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -34,6 +34,7 @@ import * as transform from "./transformutil"; import * as fs from "fs"; import * as tmp from "tmp"; import * as summary from "./summary"; +import { deprecate } from "util"; let cq = require("block-queue"); const gsv = require("geojson-validation"); @@ -594,14 +595,14 @@ function collate(result:Array){ } function streamingQueue(){ - let queue = cq(10,function (task:any,done:Function) { + let queue = cq(10,function (task:any,done:Function) { uploadData(task.id, task.options, task.tags, task.fc, true, task.options.ptag, task.options.file, task.options.id) .then(x=>{ queue.uploadCount += task.fc.features.length; console.log("uploaded feature count :"+queue.uploadCount); queue.chunksize--; - done(); + done(); }).catch((err) => { queue.failedCount += task.fc.features.length; console.log("failed feature count :"+queue.failedCount); @@ -622,6 +623,41 @@ function streamingQueue(){ return queue; } +function taskQueue(size:number=8,totalTaskSize:number){ + let queue = cq(size,function (task:any,done:Function) { + iterateChunk(task.chunk,task.url) + .then(x=>{ + queue.uploadCount += 1; + queue.chunksize--; + console.log("uploaded " + ((queue.uploadCount / totalTaskSize) * 100).toFixed(2) + "%"); + done(); + }).catch((err) => { + queue.failedCount += 1; + queue.chunksize--; + console.log("failed features " + ((queue.failedCount / totalTaskSize) * 100).toFixed(2) + "%"); + done(); + }); + }); + queue.uploadCount=0; + queue.chunksize=0; + queue.failedCount=0; + queue.send= async function(obj:any){ + queue.push(obj); + queue.chunksize++; + while(queue.chunksize>25){ + await new Promise(done => setTimeout(done, 1000)); + } + } + queue.shutdown =async ()=>{ + queue.shutdown=true; + while(queue.chunksize!=0){ + await new Promise(done => setTimeout(done, 1000)); + } + return true; + } + return queue; +} + function uploadToXyzSpace(id: string, options: any){ (async () => { @@ -750,19 +786,15 @@ function uploadToXyzSpace(id: string, options: any){ ); }else{ let queue = streamingQueue(); - transform.readGeoJsonAsChunks(options.file, options.chunk?options.chunk:1000,function(result:any){ - return new Promise((res,rej)=>{ - ( async()=>{ + let c=0; + transform.readGeoJsonAsChunks(options.file, options.chunk?options.chunk:1000,async function(result:any){ if(result.length>0){ const fc = { features: result, type: "FeatureCollection" }; await queue.send({id:id,options:options,tags:tags,fc:fc,retryCount:3}); - res(); } - })(); - }); }); } } @@ -899,18 +931,19 @@ async function uploadDataToSpaceWithTags( options ); - const chunks = options.chunk - ? chunkify(featureOut, parseInt(options.chunk)) - : [featureOut]; - const chunkSize = chunks.length; - const index = 0; try{ - await iterateChunks( - chunks, - "/hub/spaces/" + id + "/features", - index, - chunkSize, - ); + if(options.stream){ + await iterateChunks([featureOut],"/hub/spaces/" + id + "/features",0,1); + }else{ + const chunks = options.chunk + ? chunkify(featureOut, parseInt(options.chunk)) + : [featureOut]; + let tq = taskQueue(8,chunks.length); + chunks.forEach(chunk=>{ + tq.send({chunk:chunk,url:"/hub/spaces/" + id + "/features"}); + }); + await tq.shutdown(); + } }catch(e){ reject(e); return; @@ -1125,6 +1158,18 @@ async function iterateChunks(chunks: any, url: string, index: number, chunkSize: console.log("uploaded " + ((index / chunkSize) * 100).toFixed(2) + "%"); await iterateChunks(chunks, url, index, chunkSize); } +async function iterateChunk(chunk: any, url: string) { + const fc = { type: "FeatureCollection", features: chunk }; + const body = await execute( + url, + "PUT", + "application/geo+json", + JSON.stringify(fc), + null, + true + ); + return body; +} function chunkify(data: any[], chunksize: number) { let chunks: any[] = []; diff --git a/bin/transformutil.ts b/bin/transformutil.ts index 8758137..3d6dc13 100644 --- a/bin/transformutil.ts +++ b/bin/transformutil.ts @@ -291,15 +291,15 @@ export function readGeoJsonAsChunks(incomingPath: string, chunckSize:number,stre const fileStream = fs.createReadStream(path, {encoding: 'utf8'}); let stream = fileStream.pipe(JSONStream.parse('features.*')); stream.pipe(es.through(async function (data:any) { + dataArray.push(data); if(dataArray.length >=chunckSize){ stream.pause(); fileStream.pause(); await streamFuntion(dataArray); + dataArray=new Array(); stream.resume(); fileStream.resume(); - dataArray=new Array(); } - dataArray.push(data); return data; },function end () { if(dataArray.length >0){ From 9ef7352003d1ed5104284b2368c373081735c144 Mon Sep 17 00:00:00 2001 From: dhaneesh Date: Wed, 14 Nov 2018 12:57:48 +0530 Subject: [PATCH 19/23] adjusted streaming messges to single line --- bin/here-xyz.ts | 16 ++++++++++++---- bin/transformutil.ts | 16 +++++++++++++--- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index 1d27e39..1c86cb2 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -600,12 +600,12 @@ function streamingQueue(){ true, task.options.ptag, task.options.file, task.options.id) .then(x=>{ queue.uploadCount += task.fc.features.length; - console.log("uploaded feature count :"+queue.uploadCount); + process.stdout.write("\ruploaded feature count :"+queue.uploadCount+", failed feature count :"+queue.failedCount); queue.chunksize--; done(); }).catch((err) => { queue.failedCount += task.fc.features.length; - console.log("failed feature count :"+queue.failedCount); + process.stdout.write("\ruploaded feature count :"+queue.uploadCount+", failed feature count :"+queue.failedCount); queue.chunksize--; done(); }); @@ -620,6 +620,13 @@ function streamingQueue(){ this.push(obj); this.chunksize++; } + queue.shutdown =async ()=>{ + queue.shutdown=true; + while(queue.chunksize!=0){ + await new Promise(done => setTimeout(done, 1000)); + } + return true; + } return queue; } @@ -701,7 +708,7 @@ function uploadToXyzSpace(id: string, options: any){ if(result.length>0){ await queue.send({id:id,options:options,tags:tags,fc:{ type: "FeatureCollection", features: collate(result) },retryCount:3}); } - res(); + res(queue); })(); }); }); @@ -761,7 +768,7 @@ function uploadToXyzSpace(id: string, options: any){ type: "FeatureCollection" }; await queue.send({id:id,options:options,tags:tags,fc:fc,retryCount:3}); - res(); + res(queue); } })(); }); @@ -795,6 +802,7 @@ function uploadToXyzSpace(id: string, options: any){ }; await queue.send({id:id,options:options,tags:tags,fc:fc,retryCount:3}); } + return queue; }); } } diff --git a/bin/transformutil.ts b/bin/transformutil.ts index 3d6dc13..71863c2 100644 --- a/bin/transformutil.ts +++ b/bin/transformutil.ts @@ -249,7 +249,11 @@ export function readLineAsChunks(incomingPath: string, chunckSize:number,streamF } }); lr.on('end', function () { - streamFuntion(dataArray) + (async()=>{ + const queue = await streamFuntion(dataArray); + await queue.shutdown(); + console.log(""); + })(); }); }); }); @@ -274,7 +278,11 @@ export function readCSVAsChunks(incomingPath: string, chunckSize:number,streamFu })(); } }).on("end", function(){ - streamFuntion(dataArray) + (async()=>{ + const queue = await streamFuntion(dataArray); + await queue.shutdown(); + console.log(""); + })(); }); }); }); @@ -304,7 +312,9 @@ export function readGeoJsonAsChunks(incomingPath: string, chunckSize:number,stre },function end () { if(dataArray.length >0){ (async()=>{ - await streamFuntion(dataArray); + const queue = await streamFuntion(dataArray); + await queue.shutdown(); + console.log(""); dataArray=new Array(); })(); } From 7478fb893ae84189627caea65b3caba8ee93d5ac Mon Sep 17 00:00:00 2001 From: "Nisar, Naitik" Date: Tue, 27 Nov 2018 12:31:00 +0530 Subject: [PATCH 20/23] Fixing miner bugs Fixing miner bugs --- bin/here-xyz.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index 355ca14..02be8c5 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -359,7 +359,7 @@ program program .command('hexbin ') .description('create hexgrid out of xyz space data and upload it to space') - .option("-c, --cellsize ", "size of hexgrid cell in meters") + .option("-c, --cellsize ", "size of hexgrid cells in meters, you can give multiple values in comma separated way") .option("-i, --ids", "add ids of features as array inside property of created hexagon feature") .option("-p, --groupBy ", "Name of the Property using which hexbin counts will be further grouped") .option("-r, --readToken ", "Token to access source space") @@ -1053,7 +1053,7 @@ async function uploadDataToSpaceWithTags( try{ if(options.stream){ - await iterateChunks([featureOut],"/hub/spaces/" + id + "/features",0,1); + await iterateChunks([featureOut],"/hub/spaces/" + id + "/features",0,1,options.token); }else{ const chunks = options.chunk ? chunkify(featureOut, parseInt(options.chunk)) From 56a84dc5cef3103e7fd9e57531d380b164a0729d Mon Sep 17 00:00:00 2001 From: dhaneesh Date: Tue, 15 Jan 2019 15:45:41 +0530 Subject: [PATCH 21/23] removing hexbin dependancy --- bin/here-xyz.ts | 116 +----------------------------------------------- package.json | 1 - 2 files changed, 1 insertion(+), 116 deletions(-) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index 02be8c5..6246844 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -38,10 +38,6 @@ import { deprecate } from "util"; let cq = require("block-queue"); const gsv = require("geojson-validation"); -//let hexbin = require('/Users/nisar/OneDrive - HERE Global B.V/home/github/hexbin'); -let hexbin = require('hexbin'); -const md5 = require('md5'); -const request = require("request"); let choiceList: { name: string, value: string}[] = []; const questions = [ { @@ -356,115 +352,6 @@ program })(); }); -program - .command('hexbin ') - .description('create hexgrid out of xyz space data and upload it to space') - .option("-c, --cellsize ", "size of hexgrid cells in meters, you can give multiple values in comma separated way") - .option("-i, --ids", "add ids of features as array inside property of created hexagon feature") - .option("-p, --groupBy ", "Name of the Property using which hexbin counts will be further grouped") - .option("-r, --readToken ", "Token to access source space") - .option("-w, --writeToken ", "Token to access Target space where hexbins will be written") - .option("-d, --destSpace ", "Destination Space name where hexbins and centroids will be uploaded") - .option("-t, --tags ", "Hexbins will be created for those records only which matches the tag value in source space ") - .action(function (id,options) { - (async () => { - try{ - const sourceId = id; - options.totalRecords = Number.MAX_SAFE_INTEGER; - //options.token = 'Ef87rh2BTh29U-tyUx9NxQ'; - if(options.readToken){ - options.token = options.readToken; - } - const features = await getSpaceDataFromXyz(id,options); - if(features.length === 0){ - console.log("No features is available to create hexbins"); - process.exit(); - } - let cellSizes:number[] = []; - if (!options.cellsize) { - cellSizes.push(2000); - } else { - options.cellsize.split(",").forEach(function (item : string) { - if (item && item != "") { - let number = parseInt(item.toLowerCase()); - if (isNaN(number)){ - console.error(`hexbin creation failed: cellsize input "${item}" is not a valid number`); - process.exit(1); - } - cellSizes.push(number); - } - }); - } - options.token = null; - if(options.writeToken){ - options.token = options.writeToken; - } - if(options.destSpace){ - id = options.destSpace; - } - //cellSizes.forEach(function (cellsize : number) { - for(const cellsize of cellSizes){ - //(async () => { - console.log("Creating hexbins for the space data with size " + cellsize); - let hexFeatures = hexbin.calculateHexGrids(features, cellsize, options.ids, options.groupBy); - console.log("uploading the hexagon grids to space with size " + cellsize); - - let centroidFeatures:any[] = []; - var i = hexFeatures.length; - while (i--) { - let isValidHexagon = true; - hexFeatures[i].geometry.coordinates[0].forEach(function (coordinate : Array) { - if(coordinate[0] < -180 || coordinate[0] > 180 || coordinate[1] > 90 || coordinate[1] < -90){ - isValidHexagon = false; - console.log("Invalid hexagon which is created outside of permissable range - " + coordinate); - } - }); - if (isValidHexagon) { - let geometry = {"type":"Point","coordinates":hexFeatures[i].properties.centroid}; - let hashId = md5(JSON.stringify(geometry)+'_'+cellsize); - centroidFeatures.push({type:"Feature","geometry":geometry,"properties":hexFeatures[i].properties,"id":hashId}); - } else { - console.log("Invalid hexagon is getting created, ignoring this - " + JSON.stringify(hexFeatures[i])); - hexFeatures.splice(i, 1); - } - } - - //hexFeatures = hexFeatures.concat(centroidFeatures); - /* - fs.writeFile('out.json', JSON.stringify({type:"FeatureCollection",features:hexFeatures}), (err) => { - if (err) throw err; - }); - */ - - let tmpObj = tmp.fileSync({ mode: 0o644, prefix: 'hex', postfix: '.json' }); - fs.writeFileSync(tmpObj.name, JSON.stringify({type:"FeatureCollection",features:hexFeatures})); - options.tags = 'hexbin_'+cellsize+',cell_'+cellsize+',hexbin'; - if(options.destSpace){ - options.tags += ','+sourceId; - } - options.file = tmpObj.name; - options.override = true; - await uploadToXyzSpace(id,options); - - tmpObj = tmp.fileSync({ mode: 0o644, prefix: 'hex', postfix: '.json' }); - fs.writeFileSync(tmpObj.name, JSON.stringify({type:"FeatureCollection",features:centroidFeatures})); - options.tags = 'centroid_'+cellsize+',cell_'+cellsize+',centroid'; - if(options.destSpace){ - options.tags += ','+sourceId; - } - options.file = tmpObj.name; - options.override = true; - await uploadToXyzSpace(id,options); - //}); - } - //}); - } catch (error) { - console.error(`hexbin creation failed: ${error}`); - process.exit(1); - } - })(); - }); - program .command("show ") .description("shows the content of the given [id]") @@ -1327,8 +1214,7 @@ common.validate( "describe", "clear", "token", - "analyze", - "hexbin" + "analyze" ], [process.argv[2]], program diff --git a/package.json b/package.json index 2d32c33..f8e8f37 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,6 @@ "geojson-validation": "^0.2.1", "get-stdin": "6.0.0", "getmac": "1.4.3", - "hexbin": "1.0.6", "inquirer": "6.0.0", "latest-version": "4.0.0", "line-by-line": "^0.1.6", From ca52ac3e414b02c71c983e167b3fc0259fdf073b Mon Sep 17 00:00:00 2001 From: dhaneesh Date: Tue, 15 Jan 2019 16:10:23 +0530 Subject: [PATCH 22/23] Minor modifications on review --- bin/here-xyz.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index 6246844..109ba43 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -945,11 +945,12 @@ async function uploadDataToSpaceWithTags( const chunks = options.chunk ? chunkify(featureOut, parseInt(options.chunk)) : [featureOut]; - let tq = taskQueue(8,chunks.length); - chunks.forEach(chunk=>{ - tq.send({chunk:chunk,url:"/hub/spaces/" + id + "/features"}); - }); - await tq.shutdown(); + await iterateChunks(chunks,"/hub/spaces/" + id + "/features",0,chunks.length,options.token); + // let tq = taskQueue(8,chunks.length); + // chunks.forEach(chunk=>{ + // tq.send({chunk:chunk,url:"/hub/spaces/" + id + "/features"}); + // }); + // await tq.shutdown(); } }catch(e){ reject(e); From d66caf9ea969e162346b5a457ff26df5e92ee19d Mon Sep 17 00:00:00 2001 From: dhaneesh Date: Tue, 15 Jan 2019 16:38:51 +0530 Subject: [PATCH 23/23] added header app-name --- bin/here-xyz.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/here-xyz.ts b/bin/here-xyz.ts index 109ba43..0ea8439 100644 --- a/bin/here-xyz.ts +++ b/bin/here-xyz.ts @@ -97,7 +97,8 @@ async function execInternal( json: isJson, headers: { Authorization: "Bearer " + token, - "Content-Type": contentType + "Content-Type": contentType, + "App-Name": "HereCLI" }, body: method === "GET" ? undefined : data };