From 9393cd0b2f538ba1f8f1fa02e83aae21832be99f Mon Sep 17 00:00:00 2001
From: dhaneesh <dhaneesh.tb@here.com>
Date: Tue, 9 Oct 2018 00:04:09 +0530
Subject: [PATCH] changes for supportting geojsonl

---
 bin/here-xyz.js      | 16 +++++++++++++-
 bin/transformutil.js | 52 +++++++++++++++++++++++++++++++++++++++++++-
 package-lock.json    |  5 +++++
 package.json         |  1 +
 4 files changed, 72 insertions(+), 2 deletions(-)

diff --git a/bin/here-xyz.js b/bin/here-xyz.js
index cbdd495..5564326 100755
--- a/bin/here-xyz.js
+++ b/bin/here-xyz.js
@@ -512,7 +512,21 @@ program
 
     if (options.file) {
       var fs = require('fs');
-      if(options.file.indexOf(".shp")!=-1){
+      if(options.file.indexOf(".geojsonl")!=-1){
+        transform.readLineFromFile(options.file,function(result,isCompleted){
+          const totalFeatures = result.reduce(function (features, feature) {
+              if(feature.type=="Feature"){
+                features.push(feature);
+              }else if(feature.type=="FeatureCollection"){
+                features=features.concat(feature.features);
+              }else{
+                console.log("Unknown type"+feature.type);
+              }
+              return features
+          }, []);
+          uploadData(id,options,tags,{type:"FeatureCollection",features:totalFeatures},true,options.ptag,options.file,options.id);
+        },100);
+      }else if(options.file.indexOf(".shp")!=-1){
         transform.readShapeFile(options.file,function(result){
           uploadData(id,options,tags,result,true,options.ptag,options.file,options.id);
         },true);
diff --git a/bin/transformutil.js b/bin/transformutil.js
index 36cd678..db20f12 100644
--- a/bin/transformutil.js
+++ b/bin/transformutil.js
@@ -96,6 +96,53 @@ function readDataFromFile(path,callBack,needConversion){
         callBack(file_data);
 }
 
+
+function readData(path,postfix){
+    return new Promise((resolve,reject)=>{
+        if(path.indexOf("http://")!=-1 || path.indexOf("https://")!=-1){
+            const request = require('request');
+            const fs = require('fs');
+            const tmp = require('tmp');
+            tmp.file({ mode: 0644, prefix: '', postfix: postfix }, function _tempFileCreated(err, tempFilePath, fd) {
+                if (err) throw err;
+                const dest = fs.createWriteStream(tempFilePath);
+                dest.on('finish', function (e) {
+                            resolve(tempFilePath);
+                        });
+                request.get(path)
+                    .on('error', function(err) {
+                            console.log(err)
+                    }).pipe(dest);
+            });
+        }else{
+            resolve(path);
+        }
+    });
+	
+}
+
+/*
+chunckSize should be used later to stream data
+*/
+function readLineFromFile(incomingPath,callBack,chunckSize=100) {
+    readData(incomingPath,'geojsonl').then(path=>{
+        const dataArray=new Array();
+        const fs = require('fs'),
+            readline = require('readline'),
+            instream = fs.createReadStream(path),
+            outstream = new (require('stream'))(),
+            rl = readline.createInterface(instream, outstream);
+        
+        rl.on('line', function (line) {
+            dataArray.push(JSON.parse(line));
+        });
+        
+        rl.on('close', function (line) {
+            callBack(dataArray,true);
+        });
+    });
+}
+
 function dataToJson(file_data){
     var csvjson = require('csvjson');
     var options = {
@@ -186,4 +233,7 @@ function isLon(k){
 }
 module.exports.read = read;
 module.exports.transform = transform;
-module.exports.readShapeFile = readShapeFile
\ No newline at end of file
+module.exports.readShapeFile = readShapeFile
+module.exports.readLineFromFile = readLineFromFile
+
+ 
\ No newline at end of file
diff --git a/package-lock.json b/package-lock.json
index 35e6175..cf46e97 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -2942,6 +2942,11 @@
         "util-deprecate": "~1.0.1"
       }
     },
+    "readline": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/readline/-/readline-1.3.0.tgz",
+      "integrity": "sha1-xYDXfvLPyHUrEySYBg3JeTp6wBw="
+    },
     "readline2": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/readline2/-/readline2-1.0.1.tgz",
diff --git a/package.json b/package.json
index c35d418..60a371a 100644
--- a/package.json
+++ b/package.json
@@ -24,6 +24,7 @@
     "open": "0.0.5",
     "project-version": "1.2.0",
     "prompt": "^1.0.0",
+    "readline": "1.3.0",
     "request": "^2.83.0",
     "shapefile": "0.6.6",
     "tmp": "0.0.33",