From 461e4436d30e2a4ed89abb5683eb49c28c0fb9ec Mon Sep 17 00:00:00 2001 From: Nate Goldman Date: Sun, 22 Mar 2015 12:27:02 -0700 Subject: [PATCH 01/16] update readme --- README.md | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e0fc9e2..eb2c7dd 100644 --- a/README.md +++ b/README.md @@ -1,43 +1,41 @@ ## Socrata Provider for [Koop](https://github.com/Esri/koop) ------------ This provider makes it possible to access [Socrata's JSON API](http://dev.socrata.com/docs/formats/json.html) as either GeoJSON or an Esri FeatureService. This is particular useful for making maps and doing analysis on the web. -## Installation +## Install To install/use this provider you first need a working installation of [Koop](https://github.com/Esri/koop). Then from within the koop directory you'll need to run the following: - ``` - npm install https://github.com/chelm/koop-socrata/tarball/master - ``` +``` +npm install https://github.com/chelm/koop-socrata/tarball/master +``` ## Register Socrata Hosts -Once this provider's been installed you need to "register" a particular instance of Socrate with your Koop instance. To do this you make `POST` request to the `/socrata` endpoint like so: +Once this provider's been installed you need to "register" a particular instance of Socrata with your Koop instance. To do this you make `POST` request to the `/socrata` endpoint like so: - ``` - curl --data "host=https://data.nola.gov&id=nola" localhost:1337/socrata - ``` +``` +curl --data "host=https://data.nola.gov&id=nola" localhost:1337/socrata +``` -What you'll need for that request to work is an ID and a the URL of the Socrata instance. The ID is what you'll use to reference datasets that come from Socrata in Koop. +What you'll need for that request to work is an ID and the URL of the Socrata instance. The ID is what you'll use to reference datasets that come from Socrata in Koop. To make sure this works you can visit: http://localhost:1337/socrata and you should see all of the register hosts. ## Access Socrata Data -To access a dataset hosted in Socrata you'll need a "resource id" from Socrata. Datasets in Socrata can be accessed as raw JSON like this: +To access a dataset hosted in Socrata you'll need a "Resource ID" from Socrata. Datasets in Socrata can be accessed as raw JSON like this: * [https://data.nola.gov/Geographic-Reference/NOLA-Short-Term-Rentals-Map/psp3-bvzw](https://data.nola.gov/Geographic-Reference/NOLA-Short-Term-Rentals-Map/psp3-bvzw) translates into -> https://data.nola.gov/resource/psp3-bvzw.json -And then the ID `psp3-bvzw` can be referenced in Koop like so: - -[http://koop.dc.esri.com/socrata/nola/psp3-bvzw](http://koop.dc.esri.com/socrata/nola/psp3-bvzw) +And then the ID `psp3-bvzw` can be referenced in Koop like so: +http://koop.dc.esri.com/socrata/nola/psp3-bvzw ## Examples -Here's a few examples of data hosted in Socrata and accessed via Koop +Here are a few examples of data hosted in Socrata and accessed via Koop. -* GeoJSON [http://koop.dc.esri.com/socrata/nola/psp3-bvzw](http://koop.dc.esri.com/socrata/nola/psp3-bvzw) -* FeatureService [http://koop.dc.esri.com/socrata/nola/psp3-bvzw/FeatureServer/0] -* All of the publicly registered Socrata instances [http://koop.dc.esri.com/socrata](http://koop.dc.esri.com/socrata) +* GeoJSON: http://koop.dc.esri.com/socrata/nola/psp3-bvzw +* FeatureService: http://koop.dc.esri.com/socrata/nola/psp3-bvzw/FeatureServer/0 +* All publicly registered Socrata instances: http://koop.dc.esri.com/socrata From 285255845e6e38fe04518865c7e78932e5b1f77d Mon Sep 17 00:00:00 2001 From: Sam Libby Date: Fri, 3 Apr 2015 13:46:50 -0400 Subject: [PATCH 02/16] Update Readme.MD Updated links to new repo under koopjs Updated NOLA links to a valid dataset (previous one was removed). --- README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index eb2c7dd..76e8219 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This provider makes it possible to access [Socrata's JSON API](http://dev.socrat To install/use this provider you first need a working installation of [Koop](https://github.com/Esri/koop). Then from within the koop directory you'll need to run the following: ``` -npm install https://github.com/chelm/koop-socrata/tarball/master +npm install https://github.com/koopjs/koop-socrata/tarball/master ``` ## Register Socrata Hosts @@ -17,6 +17,7 @@ Once this provider's been installed you need to "register" a particular instance ``` curl --data "host=https://data.nola.gov&id=nola" localhost:1337/socrata ``` +*for Windows users, download cURL from http://curl.haxx.se/download.html or use a tool of your choice to generate the POST request* What you'll need for that request to work is an ID and the URL of the Socrata instance. The ID is what you'll use to reference datasets that come from Socrata in Koop. @@ -26,16 +27,16 @@ To make sure this works you can visit: http://localhost:1337/socrata and you sho To access a dataset hosted in Socrata you'll need a "Resource ID" from Socrata. Datasets in Socrata can be accessed as raw JSON like this: -* [https://data.nola.gov/Geographic-Reference/NOLA-Short-Term-Rentals-Map/psp3-bvzw](https://data.nola.gov/Geographic-Reference/NOLA-Short-Term-Rentals-Map/psp3-bvzw) translates into -> https://data.nola.gov/resource/psp3-bvzw.json +* [https://data.nola.gov/Health-Education-and-Social-Services/NOLA-Grocery-Stores/fwm6-d78i](https://data.nola.gov/Health-Education-and-Social-Services/NOLA-Grocery-Stores/fwm6-d78i) translates into -> https://data.nola.gov/resource/fwm6-d78i.json -And then the ID `psp3-bvzw` can be referenced in Koop like so: +And then the ID `fwm6-d78i` can be referenced in Koop like so: -http://koop.dc.esri.com/socrata/nola/psp3-bvzw +http://koop.dc.esri.com/socrata/nola/fwm6-d78i ## Examples Here are a few examples of data hosted in Socrata and accessed via Koop. -* GeoJSON: http://koop.dc.esri.com/socrata/nola/psp3-bvzw -* FeatureService: http://koop.dc.esri.com/socrata/nola/psp3-bvzw/FeatureServer/0 +* GeoJSON: http://koop.dc.esri.com/socrata/nola/fwm6-d78i +* FeatureService: http://koop.dc.esri.com/socrata/nola/fwm6-d78i/FeatureServer/0 * All publicly registered Socrata instances: http://koop.dc.esri.com/socrata From 596c01852b66ee82c7cf44b801227d74df4a4c42 Mon Sep 17 00:00:00 2001 From: pholleran Date: Thu, 9 Apr 2015 22:57:11 -0500 Subject: [PATCH 03/16] Handle Multiple Location Columns in Socrata If multiple locations columns exist in Socrata, use the syntax url/socrata/id!desiredColumn to ensure the desired column is used --- models/Socrata.js | 207 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 204 insertions(+), 3 deletions(-) diff --git a/models/Socrata.js b/models/Socrata.js index 6374f96..f5d8d30 100644 --- a/models/Socrata.js +++ b/models/Socrata.js @@ -1,5 +1,208 @@ var request = require('request'); +var Socrata = function( koop ){ + + var socrata = {}; + socrata.__proto__ = koop.BaseModel( koop ); + + // adds a service to the koop.Cache.db + // needs a host, generates an id + socrata.register = function( id, host, callback ){ + var type = 'socrata:services'; + koop.Cache.db.serviceCount( type, function(error, count){ + id = id || count++; + koop.Cache.db.serviceRegister( type, {'id': id, 'host': host}, function( err, success ){ + callback( err, id ); + }); + }); + }; + + socrata.remove = function( id, callback ){ + koop.Cache.db.serviceRemove( 'socrata:services', parseInt(id) || id, callback); + }; + + // get service by id, no id == return all + socrata.find = function( id, callback ){ + koop.Cache.db.serviceGet( 'socrata:services', parseInt(id) || id, function(err, res){ + if (err){ + callback('No datastores have been registered with this provider yet. Try POSTing {"host":"url", "id":"yourId"} to /socrata', null); + } + else{ + callback(null, res); + } + }); + }; + + socrata.socrata_path = '/resource/'; + socrata.socrata_view_path = '/resource/'; + + // got the service and get the item + socrata.getResource = function( host, hostId, id, options, callback ){ + var self = this, + type = 'Socrata', + key = id, + locFieldName, + urlid; + + // test id for '!' character indicating presence of a column name and handle + if (id.indexOf("!") != -1){ + locFieldName = id.substring(id.indexOf("!") + 1,id.length); + urlid = id.substring(0, id.indexOf("!")); + } + else{ + urlid = id; + } + + koop.Cache.get( type, key, options, function(err, entry ){ + if ( err ){ + var url = host + self.socrata_path + urlid + '.json'; + var meta_url = host + self.socrata_view_path + urlid + '.json'; + //dmf: have to make a request to the views endpoint in order to get metadata + var name; + request.get(meta_url, function(err, data, response){ + if (err){ + callback(err, null); + } else { + try { + name = JSON.parse( data.body ).name; + } catch( e ){ + callback(e, null); + } + } + + request.get(url, function(err, data, response ){ + if (err) { + callback(err, null); + } else { + try { + var types = JSON.parse( data.headers['x-soda2-types'] ); + fields = JSON.parse( data.headers['x-soda2-fields'] ); + var locationField; + if (locFieldName){ + locationField = locFieldName; + } + else { + types.forEach(function(t,i){ + if (t == 'location'){ + locationField = fields[i]; + } + }); + } + + self.toGeojson( JSON.parse( data.body ), locationField, function(err, geojson){ + geojson.updated_at = new Date(data.headers['last-modified']).getTime(); + geojson.name = id; + geojson.host = { + id: hostId, + url: host + }; + koop.Cache.insert( type, key, geojson, 0, function( err, success){ + if ( success ) callback( null, [geojson] ); + }); + }); + } catch(e){ + console.log('shit?', e); + koop.log.error('Unable to parse response %s', url); + callback(e, null); + } + } + }); + }); + } else { + callback( null, entry ); + } + }); + + }; + + socrata.toGeojson = function(json, locationField, callback){ + if (!json || !json.length){ + callback('Error converting data to geojson', null); + } else { + var geojson = {type: 'FeatureCollection', features: []}; + var geojsonFeature; + json.forEach(function(feature, i){ + geojsonFeature = {type: 'Feature', geometry: {}, id: i+1}; + if (feature && locationField){ + if (feature[locationField] && feature[locationField].latitude && feature[locationField].longitude){ + geojsonFeature.geometry.coordinates = [parseFloat(feature[locationField].longitude), parseFloat(feature[locationField].latitude)]; + geojsonFeature.geometry.type = 'Point'; + delete feature.location; + geojsonFeature.properties = feature; + geojson.features.push( geojsonFeature ); + } + } else if ( feature && feature.latitude && feature.longitude ){ + geojsonFeature.geometry.coordinates = [parseFloat(feature.longitude), parseFloat(feature.latitude)]; + geojsonFeature.geometry.type = 'Point'; + geojsonFeature.properties = feature; + geojson.features.push( geojsonFeature ); + } else { + geojsonFeature.geometry = null; + geojsonFeature.properties = feature; + geojson.features.push( geojsonFeature ); + } + }); + callback(null, geojson); + } + }; + + // compares the sha on the cached data and the hosted data + // this method name is special reserved name that will get called by the cache model + socrata.checkCache = function(key, data, options, callback){ + var self = this; + url = data.host + this.socrata_path + key + '.json'; + + var lapsed = (new Date().getTime() - data.updated_at); + if (typeof(data.updated_at) == "undefined" || (lapsed > (1000*60*60))){ + callback(null, false); + } else { + request.get(url, function( err, data, response ){ + if (err) { + callback( err, null ); + } else { + var types = JSON.parse( data.headers['x-soda2-types'] ); + var fields = JSON.parse( data.headers['x-soda2-fields'] ); + var locationField; + types.forEach(function(t,i){ + if (t == 'location'){ + locationField = fields[i]; + } + }); + self.toGeojson( JSON.parse( data.body ), locationField, function( error, geojson ){ + geojson.updated_at = new Date(data.headers['last-modified']).getTime(); + geojson.name = data.name || key; + geojson.host = data.host; + callback( error, [geojson] ); + }); + } + }); + } + + }; + + // drops the item from the cache + socrata.dropItem = function( host, itemId, options, callback ){ + var dir = [ 'socrata', host, itemId].join(':'); + koop.Cache.remove('Socrata', itemId, options, function(err, res){ + koop.files.removeDir( 'files/' + dir, function(err, res){ + koop.files.removeDir( 'tiles/'+ dir, function(err, res){ + koop.files.removeDir( 'thumbs/'+ dir, function(err, res){ + callback(err, true); + }); + }); + }); + }); + }; + + return socrata; + +}; + + +module.exports = Socrata; + +var request = require('request'); + var Socrata = function( koop ){ var socrata = {}; @@ -182,6 +385,4 @@ var Socrata = function( koop ){ }; - -module.exports = Socrata; - +module.exports = Socrata; \ No newline at end of file From 387e6cbaff39ce2fdd0c9e095c66b50ff7c4219a Mon Sep 17 00:00:00 2001 From: pholleran Date: Thu, 9 Apr 2015 23:13:43 -0500 Subject: [PATCH 04/16] Handle multiple location columns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If multiple location columns are present in Socrata use syntax ‘socrata/id!desiredColumn’ to indicate desired column to use. --- models/Socrata.js | 185 ---------------------------------------------- 1 file changed, 185 deletions(-) diff --git a/models/Socrata.js b/models/Socrata.js index f5d8d30..31c99bb 100644 --- a/models/Socrata.js +++ b/models/Socrata.js @@ -201,188 +201,3 @@ var Socrata = function( koop ){ module.exports = Socrata; -var request = require('request'); - -var Socrata = function( koop ){ - - var socrata = {}; - socrata.__proto__ = koop.BaseModel( koop ); - - // adds a service to the koop.Cache.db - // needs a host, generates an id - socrata.register = function( id, host, callback ){ - var type = 'socrata:services'; - koop.Cache.db.serviceCount( type, function(error, count){ - id = id || count++; - koop.Cache.db.serviceRegister( type, {'id': id, 'host': host}, function( err, success ){ - callback( err, id ); - }); - }); - }; - - socrata.remove = function( id, callback ){ - koop.Cache.db.serviceRemove( 'socrata:services', parseInt(id) || id, callback); - }; - - // get service by id, no id == return all - socrata.find = function( id, callback ){ - koop.Cache.db.serviceGet( 'socrata:services', parseInt(id) || id, function(err, res){ - if (err){ - callback('No datastores have been registered with this provider yet. Try POSTing {"host":"url", "id":"yourId"} to /socrata', null); - } - else{ - callback(null, res); - } - }); - }; - - socrata.socrata_path = '/resource/'; - socrata.socrata_view_path = '/resource/'; - - // got the service and get the item - socrata.getResource = function( host, hostId, id, options, callback ){ - var self = this, - type = 'Socrata', - key = id; - - koop.Cache.get( type, key, options, function(err, entry ){ - if ( err ){ - var url = host + self.socrata_path + id + '.json'; - var meta_url = host + self.socrata_view_path + id + '.json'; - //dmf: have to make a request to the views endpoint in order to get metadata - var name; - request.get(meta_url, function(err, data, response){ - if (err){ - callback(err, null); - } else { - try { - name = JSON.parse( data.body ).name; - } catch( e ){ - callback(e, null); - } - } - - request.get(url, function(err, data, response ){ - if (err) { - callback(err, null); - } else { - try { - var types = JSON.parse( data.headers['x-soda2-types'] ); - fields = JSON.parse( data.headers['x-soda2-fields'] ); - var locationField; - types.forEach(function(t,i){ - if (t == 'location'){ - locationField = fields[i]; - } - }); - - self.toGeojson( JSON.parse( data.body ), locationField, function(err, geojson){ - geojson.updated_at = new Date(data.headers['last-modified']).getTime(); - geojson.name = id; - geojson.host = { - id: hostId, - url: host - }; - koop.Cache.insert( type, key, geojson, 0, function( err, success){ - if ( success ) callback( null, [geojson] ); - }); - }); - } catch(e){ - console.log('shit?', e); - koop.log.error('Unable to parse response %s', url); - callback(e, null); - } - } - }); - }); - } else { - callback( null, entry ); - } - }); - - }; - - socrata.toGeojson = function(json, locationField, callback){ - if (!json || !json.length){ - callback('Error converting data to geojson', null); - } else { - var geojson = {type: 'FeatureCollection', features: []}; - var geojsonFeature; - json.forEach(function(feature, i){ - geojsonFeature = {type: 'Feature', geometry: {}, id: i+1}; - if (feature && locationField){ - if (feature[locationField] && feature[locationField].latitude && feature[locationField].longitude){ - geojsonFeature.geometry.coordinates = [parseFloat(feature[locationField].longitude), parseFloat(feature[locationField].latitude)]; - geojsonFeature.geometry.type = 'Point'; - delete feature.location; - geojsonFeature.properties = feature; - geojson.features.push( geojsonFeature ); - } - } else if ( feature && feature.latitude && feature.longitude ){ - geojsonFeature.geometry.coordinates = [parseFloat(feature.longitude), parseFloat(feature.latitude)]; - geojsonFeature.geometry.type = 'Point'; - geojsonFeature.properties = feature; - geojson.features.push( geojsonFeature ); - } else { - geojsonFeature.geometry = null; - geojsonFeature.properties = feature; - geojson.features.push( geojsonFeature ); - } - }); - callback(null, geojson); - } - }; - - // compares the sha on the cached data and the hosted data - // this method name is special reserved name that will get called by the cache model - socrata.checkCache = function(key, data, options, callback){ - var self = this; - url = data.host + this.socrata_path + key + '.json'; - - var lapsed = (new Date().getTime() - data.updated_at); - if (typeof(data.updated_at) == "undefined" || (lapsed > (1000*60*60))){ - callback(null, false); - } else { - request.get(url, function( err, data, response ){ - if (err) { - callback( err, null ); - } else { - var types = JSON.parse( data.headers['x-soda2-types'] ); - var fields = JSON.parse( data.headers['x-soda2-fields'] ); - var locationField; - types.forEach(function(t,i){ - if (t == 'location'){ - locationField = fields[i]; - } - }); - self.toGeojson( JSON.parse( data.body ), locationField, function( error, geojson ){ - geojson.updated_at = new Date(data.headers['last-modified']).getTime(); - geojson.name = data.name || key; - geojson.host = data.host; - callback( error, [geojson] ); - }); - } - }); - } - - }; - - // drops the item from the cache - socrata.dropItem = function( host, itemId, options, callback ){ - var dir = [ 'socrata', host, itemId].join(':'); - koop.Cache.remove('Socrata', itemId, options, function(err, res){ - koop.files.removeDir( 'files/' + dir, function(err, res){ - koop.files.removeDir( 'tiles/'+ dir, function(err, res){ - koop.files.removeDir( 'thumbs/'+ dir, function(err, res){ - callback(err, true); - }); - }); - }); - }); - }; - - return socrata; - -}; - -module.exports = Socrata; \ No newline at end of file From e28d7bdb9710e66d8aebc3c86395133b7374c60f Mon Sep 17 00:00:00 2001 From: pholleran Date: Thu, 16 Apr 2015 15:40:32 -0500 Subject: [PATCH 05/16] Handle Large Datasets Modified Socrata provider to page through requests to large datasets. --- models/Socrata.js | 152 +++++++++++++++++++++++++++++++--------------- 1 file changed, 102 insertions(+), 50 deletions(-) diff --git a/models/Socrata.js b/models/Socrata.js index 31c99bb..b537a2b 100644 --- a/models/Socrata.js +++ b/models/Socrata.js @@ -27,14 +27,13 @@ var Socrata = function( koop ){ if (err){ callback('No datastores have been registered with this provider yet. Try POSTing {"host":"url", "id":"yourId"} to /socrata', null); } - else{ + else { callback(null, res); } }); }; socrata.socrata_path = '/resource/'; - socrata.socrata_view_path = '/resource/'; // got the service and get the item socrata.getResource = function( host, hostId, id, options, callback ){ @@ -42,71 +41,124 @@ var Socrata = function( koop ){ type = 'Socrata', key = id, locFieldName, - urlid; + urlid, + paging = false, + limit = 1000; // test id for '!' character indicating presence of a column name and handle if (id.indexOf("!") != -1){ locFieldName = id.substring(id.indexOf("!") + 1,id.length); urlid = id.substring(0, id.indexOf("!")); } - else{ + else { urlid = id; } + // attempt to load from cache, if error perform new request and get first page koop.Cache.get( type, key, options, function(err, entry ){ if ( err ){ - var url = host + self.socrata_path + urlid + '.json'; - var meta_url = host + self.socrata_view_path + urlid + '.json'; - //dmf: have to make a request to the views endpoint in order to get metadata - var name; - request.get(meta_url, function(err, data, response){ - if (err){ + var url = host + self.socrata_path + urlid + '.json?$order=:id&$limit=' + limit; + request.get(url, function(err, data, response ){ + if (err) { callback(err, null); - } else { - try { - name = JSON.parse( data.body ).name; - } catch( e ){ - callback(e, null); + } else { + + // test to see if paging will be needed later + if (Object.keys(JSON.parse(data.body)).length == limit){ + paging = true; } - } - request.get(url, function(err, data, response ){ - if (err) { - callback(err, null); - } else { - try { + // get name of location field + try { + var locationField; + if (locFieldName){ + locationField = locFieldName; + } + else { var types = JSON.parse( data.headers['x-soda2-types'] ); - fields = JSON.parse( data.headers['x-soda2-fields'] ); - var locationField; - if (locFieldName){ - locationField = locFieldName; - } - else { - types.forEach(function(t,i){ - if (t == 'location'){ - locationField = fields[i]; - } - }); - } - - self.toGeojson( JSON.parse( data.body ), locationField, function(err, geojson){ - geojson.updated_at = new Date(data.headers['last-modified']).getTime(); - geojson.name = id; - geojson.host = { - id: hostId, - url: host - }; - koop.Cache.insert( type, key, geojson, 0, function( err, success){ - if ( success ) callback( null, [geojson] ); - }); + var fields = JSON.parse( data.headers['x-soda2-fields'] ); + types.forEach(function(t,i){ + if (t == 'location'){ + locationField = fields[i]; + } }); - } catch(e){ - console.log('shit?', e); - koop.log.error('Unable to parse response %s', url); - callback(e, null); } + + // parse first page to geoJSON and insert + self.toGeojson( JSON.parse( data.body ), locationField, function(err, geojson){ + geojson.updated_at = new Date(data.headers['last-modified']).getTime(); + geojson.name = id; + geojson.host = { + id: hostId, + url: host + }; + koop.Cache.insert( type, key, geojson, 0, function( err, success){ + if ( success ) { + // check to see if paging is needed + if (paging === false){ + callback( null, [geojson] ); + } + else { + // create GeoJSON return object + retGeoJSON = geojson; + // detrmine count of table and needed pages + var count, pages; + var pagesComplete = 0; + var countUrl = host + self.socrata_path + urlid + '.json?$select=count(*)'; + request.get(countUrl, function(err, data, response){ + count = parseInt(JSON.parse(data.body)[0].count,10); + if ((count/limit) % 1 === 0){ + pages = (count/limit - 1); + } + else { + pages = Math.floor(count/limit); + } + // page through data + for (var p = 1; p <= pages; p++){ + var pUrl = host + self.socrata_path + urlid + '.json?$order=:id&$limit=' + limit + '&$offset=' + (p*limit); + request.get(pUrl,function(err, data, response){ + // parse pages to GeoJSON and insert partial + self.toGeojson( JSON.parse( data.body ), locationField, function(err, geojson){ + geojson.updated_at = new Date(data.headers['last-modified']).getTime(); + geojson.name = id; + geojson.host = { + id: hostId, + url: host + }; + koop.Cache.insertPartial( type, key, geojson, 0, function( err, success){ + if ( success ) { + // append geojson to return object + for (f = 0; f < geojson.features.length; f++){ + retGeoJSON.features.push(geojson.features[f]); + } + // update pages completed and check for completion of pages + pagesComplete++; + checkDone(); + } + }); + }); + }); + } + + // function to check completion of pages + var checkDone = function(){ + if (pagesComplete == pages){ + callback( null, [retGeoJSON]) + } + else { + } + }; + }); + } + } + }); + }); + } catch (e){ + console.log('Error?', e); + koop.log.error('Unable to parse response %s', url); + callback(e, null); } - }); + } }); } else { callback( null, entry ); @@ -150,7 +202,7 @@ var Socrata = function( koop ){ // this method name is special reserved name that will get called by the cache model socrata.checkCache = function(key, data, options, callback){ var self = this; - url = data.host + this.socrata_path + key + '.json'; + var url = data.host + this.socrata_path + key + '.json'; var lapsed = (new Date().getTime() - data.updated_at); if (typeof(data.updated_at) == "undefined" || (lapsed > (1000*60*60))){ From 7aed5bc96ff34ab4481989a856d607f9dcd8d57c Mon Sep 17 00:00:00 2001 From: pholleran Date: Fri, 17 Apr 2015 08:31:16 -0500 Subject: [PATCH 06/16] Update Readme.md --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 76e8219..fa59ddd 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,14 @@ And then the ID `fwm6-d78i` can be referenced in Koop like so: http://koop.dc.esri.com/socrata/nola/fwm6-d78i +If your Socrata data has more than one location column, you can specify the desired location column in the http request like this: + +https:///socrata//! + +## Handle Large Datasets + +The Socrata API defaults to 1000 results per request, but can be set to return up to 50,000. Koop will page through large datasets to capture all the points. To change the number of results per request, modify the variable in the socrata.getResoruce function in models/Socrata.js. + ## Examples Here are a few examples of data hosted in Socrata and accessed via Koop. From 21663c5642895db7fa8eec4fdf66a2ea88e3214f Mon Sep 17 00:00:00 2001 From: pholleran Date: Fri, 17 Apr 2015 08:33:03 -0500 Subject: [PATCH 07/16] Update README.md formatting changes --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fa59ddd..ae421b9 100644 --- a/README.md +++ b/README.md @@ -35,11 +35,11 @@ http://koop.dc.esri.com/socrata/nola/fwm6-d78i If your Socrata data has more than one location column, you can specify the desired location column in the http request like this: -https:///socrata//! +https://path_to_koop/socrata/socrataProvider/dataSetID!spatialColumn ## Handle Large Datasets -The Socrata API defaults to 1000 results per request, but can be set to return up to 50,000. Koop will page through large datasets to capture all the points. To change the number of results per request, modify the variable in the socrata.getResoruce function in models/Socrata.js. +The Socrata API defaults to 1000 results per request, but can be set to return up to 50,000. Koop will page through large datasets to capture all the points. To change the number of results per request, modify the 'limit' variable in the socrata.getResoruce function in models/Socrata.js. ## Examples From 05dc4bfbcbf7e68622beff0d4c789da307f45788 Mon Sep 17 00:00:00 2001 From: pholleran Date: Mon, 20 Apr 2015 20:27:01 -0500 Subject: [PATCH 08/16] Handle out-of-range LAT/LON Handle Lat/Long values reported above 180 --- models/Socrata.js | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/models/Socrata.js b/models/Socrata.js index b537a2b..e4920c1 100644 --- a/models/Socrata.js +++ b/models/Socrata.js @@ -176,18 +176,30 @@ var Socrata = function( koop ){ json.forEach(function(feature, i){ geojsonFeature = {type: 'Feature', geometry: {}, id: i+1}; if (feature && locationField){ - if (feature[locationField] && feature[locationField].latitude && feature[locationField].longitude){ + if (feature[locationField] && (parseFloat(feature[locationField].latitude) <= 90) && (parseFloat(feature[locationField].longitude) <= 180)){ geojsonFeature.geometry.coordinates = [parseFloat(feature[locationField].longitude), parseFloat(feature[locationField].latitude)]; geojsonFeature.geometry.type = 'Point'; delete feature.location; geojsonFeature.properties = feature; geojson.features.push( geojsonFeature ); } + else { + geojsonFeature.geometry = null; + geojsonFeature.properties = feature; + geojson.features.push( geojsonFeature ); + } } else if ( feature && feature.latitude && feature.longitude ){ - geojsonFeature.geometry.coordinates = [parseFloat(feature.longitude), parseFloat(feature.latitude)]; - geojsonFeature.geometry.type = 'Point'; - geojsonFeature.properties = feature; - geojson.features.push( geojsonFeature ); + if ((parseFloat(feature.latitude) <= 90) && (parseFloat(feature.longitude) <= 180)){ + geojsonFeature.geometry.coordinates = [parseFloat(feature.longitude), parseFloat(feature.latitude)]; + geojsonFeature.geometry.type = 'Point'; + geojsonFeature.properties = feature; + geojson.features.push( geojsonFeature ); + } + else { + geojsonFeature.geometry = null; + geojsonFeature.properties = feature; + geojson.features.push( geojsonFeature ); + } } else { geojsonFeature.geometry = null; geojsonFeature.properties = feature; From 41e4b8c19889519b57609201f16b4ff2935b3b1e Mon Sep 17 00:00:00 2001 From: pholleran Date: Mon, 20 Apr 2015 22:54:50 -0500 Subject: [PATCH 09/16] Handle Null Geometry Handle lat/long out of range --- models/Socrata.js | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/models/Socrata.js b/models/Socrata.js index e4920c1..170a3f4 100644 --- a/models/Socrata.js +++ b/models/Socrata.js @@ -174,31 +174,36 @@ var Socrata = function( koop ){ var geojson = {type: 'FeatureCollection', features: []}; var geojsonFeature; json.forEach(function(feature, i){ + var lat, lon; geojsonFeature = {type: 'Feature', geometry: {}, id: i+1}; if (feature && locationField){ - if (feature[locationField] && (parseFloat(feature[locationField].latitude) <= 90) && (parseFloat(feature[locationField].longitude) <= 180)){ - geojsonFeature.geometry.coordinates = [parseFloat(feature[locationField].longitude), parseFloat(feature[locationField].latitude)]; - geojsonFeature.geometry.type = 'Point'; - delete feature.location; + lon = parseFloat(feature[locationField].longitude); + lat = parseFloat(feature[locationField].latitude); + if ((lon < -180 || lon > 180) || (lat < -90 || lat > 90)){ + geojsonFeature.geometry = null; geojsonFeature.properties = feature; geojson.features.push( geojsonFeature ); } else { - geojsonFeature.geometry = null; + geojsonFeature.geometry.coordinates = [lon, lat]; + geojsonFeature.geometry.type = 'Point'; + delete feature.location; geojsonFeature.properties = feature; geojson.features.push( geojsonFeature ); } } else if ( feature && feature.latitude && feature.longitude ){ - if ((parseFloat(feature.latitude) <= 90) && (parseFloat(feature.longitude) <= 180)){ - geojsonFeature.geometry.coordinates = [parseFloat(feature.longitude), parseFloat(feature.latitude)]; - geojsonFeature.geometry.type = 'Point'; + lon = parseFloat(feature.longitude); + lat = parseFloat(feature.latitude); + if ((lon < -180 || lon > 180) || (lat < -90 || lat > 90)){ + geojsonFeature.geometry = null; geojsonFeature.properties = feature; - geojson.features.push( geojsonFeature ); + geojson.features.push( geojsonFeature ); } else { - geojsonFeature.geometry = null; + geojsonFeature.geometry.coordinates = [lon, lat]; + geojsonFeature.geometry.type = 'Point'; geojsonFeature.properties = feature; - geojson.features.push( geojsonFeature ); + geojson.features.push( geojsonFeature ); } } else { geojsonFeature.geometry = null; From c9d9a75a8c30463be9fc3da3deef5379b7733cd1 Mon Sep 17 00:00:00 2001 From: chelm Date: Tue, 21 Apr 2015 10:01:46 -0600 Subject: [PATCH 10/16] creating v0.1.0; using standard format and better test example --- config/default.yml | 4 - controller/index.js | 390 ++++++++++++++++++++++---------------------- index.js | 10 +- models/Socrata.js | 348 ++++++++++++++++++++------------------- package.json | 14 +- test/model-test.js | 58 ------- test/routes-test.js | 122 -------------- 7 files changed, 385 insertions(+), 561 deletions(-) delete mode 100644 config/default.yml delete mode 100644 test/model-test.js delete mode 100644 test/routes-test.js diff --git a/config/default.yml b/config/default.yml deleted file mode 100644 index a50ac03..0000000 --- a/config/default.yml +++ /dev/null @@ -1,4 +0,0 @@ -# Default config -db: - postgis: - conn: "postgres://localhost/koopdev" diff --git a/controller/index.js b/controller/index.js index fec0ea0..32581c2 100644 --- a/controller/index.js +++ b/controller/index.js @@ -1,285 +1,289 @@ -var extend = require('node.extend'), - sm = require('sphericalmercator'), - merc = new sm({size:256}), +var SphericalMerc = require('sphericalmercator'), + merc = new SphericalMerc({ size: 256 }), fs = require('fs'), - crypto = require('crypto'); + crypto = require('crypto') // a function that is given an instance of Koop at init -var Controller = function( Socrata, BaseController ){ +var Controller = function (Socrata, BaseController) { + var controller = BaseController() - var controller = {}; - controller.__proto__ = BaseController(); - - // register a socrata instance - controller.register = function(req, res){ - if ( !req.body.host ){ - res.send('Must provide a host to register:', 500); - } else { - Socrata.register( req.body.id, req.body.host, function(err, id){ + // register a socrata instance + controller.register = function (req, res) { + if (!req.body.host) { + res.send('Must provide a host to register:', 500) + } else { + Socrata.register(req.body.id, req.body.host, function (err, id) { if (err) { - res.send( err, 500); + res.send(err, 500) } else { - res.json({ 'serviceId': id }); + res.json({ 'serviceId': id }) } - }); + }) } - }; + } - controller.list = function(req, res){ - Socrata.find(null, function(err, data){ + controller.list = function (req, res) { + Socrata.find(null, function (err, data) { if (err) { - res.send( err, 500); + res.send(err, 500) } else { - res.json( data ); + res.json(data) } - }); - }; + }) + } - controller.find = function(req, res){ - Socrata.find(req.params.id, function(err, data){ + controller.find = function (req, res) { + Socrata.find(req.params.id, function (err, data) { if (err) { - res.send( err, 404); + res.send(err, 404) } else { - res.json( data ); + res.json(data) } - }); - }; + }) + } // drops the cache for an item - controller.drop = function(req, res){ - Socrata.find(req.params.id, function(err, data){ + controller.drop = function (req, res) { + Socrata.find(req.params.id, function (err, data) { if (err) { - res.send( err, 500); + res.send(err, 500) } else { - // Get the item - Socrata.dropItem( data.host, req.params.item, req.query, function(error, itemJson){ + // Get the item + Socrata.dropItem(data.host, req.params.item, req.query, function (error, itemJson) { if (error) { - res.send( error, 500); + res.send(error, 500) } else { - res.json( itemJson ); + res.json(itemJson) } - }); + }) } - }); - }; + }) + } - controller.findResource = function(req, res){ - Socrata.find(req.params.id, function(err, data){ + controller.findResource = function (req, res) { + Socrata.find(req.params.id, function (err, data) { if (err) { - res.send( err, 500); + res.send(err, 500) } else { - // Get the item - Socrata.getResource( data.host, req.params.id, req.params.item, req.query, function(error, itemJson){ + // Get the item + Socrata.getResource(data.host, req.params.id, req.params.item, req.query, function (error, itemJson) { if (error) { - res.send( error, 500); - } else if ( req.params.format ) { + res.send(error, 500) + } else if (req.params.format) { // change geojson to json - req.params.format = req.params.format.replace('geojson', 'json'); - - var dir = ['socrata', req.params.id, req.params.item ].join(':'); - // build the file key as an MD5 hash that's a join on the paams and look for the file - var toHash = JSON.stringify( req.params ) + JSON.stringify( req.query ); - var key = crypto.createHash('md5').update( toHash ).digest('hex'); - - var path = ['files', dir].join('/'); - var fileName = key + '.' + req.params.format; - Socrata.files.exists( path, fileName, function( exists, path ){ - if ( exists ){ - if (path.substr(0, 4) == 'http'){ - res.redirect( path ); + req.params.format = req.params.format.replace('geojson', 'json') + + var dir = ['socrata', req.params.id, req.params.item ].join(':') + // build the file key as an MD5 hash that's a join on the paams and look for the file + var toHash = JSON.stringify(req.params) + JSON.stringify(req.query) + var key = crypto.createHash('md5').update(toHash).digest('hex') + + var path = ['files', dir].join('/') + var fileName = key + '.' + req.params.format + Socrata.files.exists(path, fileName, function (exists, path) { + if (exists) { + if (path.substr(0, 4) === 'http') { + res.redirect(path) } else { - res.sendfile( path ); + res.sendfile(path) } } else { - Socrata.exportToFormat( req.params.format, dir, key, itemJson[0], {}, function(err, file){ - if (err){ - res.send(err, 500); + Socrata.exportToFormat(req.params.format, dir, key, itemJson[0], {}, function (err, file) { + if (err) { + res.send(err, 500) } else { - res.sendfile( file ); + if (file.substr(0, 4) === 'http') { + res.redirect(file) + } else { + res.sendfile(file) + } } - }); + }) } - }); - } else { - res.json( itemJson[0] ); + }) + } else { + res.json(itemJson[0]) } - }); + }) } - }); - }; + }) + } - controller.del = function(req, res){ - if ( !req.params.id ){ - res.send( 'Must specify a service id', 500 ); - } else { - Socrata.remove(req.params.id, function(err, data){ + controller.del = function (req, res) { + if (!req.params.id) { + res.send('Must specify a service id', 500) + } else { + Socrata.remove(req.params.id, function (err, data) { if (err) { - res.send( err, 500); + res.send(err, 500) } else { - res.json( data ); + res.json(data) } - }); + }) } - }; - - controller.featureserver = function( req, res ){ - var callback = req.query.callback; - delete req.query.callback; - - for (var k in req.body){ - req.query[k] = req.body[k]; + } + + controller.featureserver = function (req, res) { + var callback = req.query.callback + delete req.query.callback + + for (var k in req.body) { + req.query[k] = req.body[k] } - Socrata.find(req.params.id, function(err, data){ + Socrata.find(req.params.id, function (err, data) { if (err) { - res.send( err, 500); + res.send(err, 500) } else { - // Get the item - Socrata.getResource( data.host, req.params.id, req.params.item, req.query, function(error, geojson){ + // Get the item + Socrata.getResource(data.host, req.params.id, req.params.item, req.query, function (error, geojson) { if (error) { - res.send( error, 500); + res.send(error, 500) } else { // pass to the shared logic for FeatureService routing - delete req.query.geometry; - delete req.query.where; - controller.processFeatureServer( req, res, err, geojson, callback); + delete req.query.geometry + delete req.query.where + controller.processFeatureServer(req, res, err, geojson, callback) } - }); + }) } - }); - - }; + }) + } - controller.tiles = function( req, res ){ - var callback = req.query.callback; - delete req.query.callback; + controller.tiles = function (req, res) { + var callback = req.query.callback + delete req.query.callback var key, - layer = req.params.layer || 0; + layer = req.params.layer || 0 - var _send = function( err, data ){ - req.params.key = key + ':' + layer; - if (req.query.style){ - req.params.style = req.query.style; + var _send = function (err, data) { + if (err) { + res.status(404).send(err) + return + } + req.params.key = key + ':' + layer + if (req.query.style) { + req.params.style = req.query.style + } + Socrata.tileGet(req.params, data[ layer ], function (err, tile) { + if (err) { + res.status(404).send(err) + return } - Socrata.tileGet( req.params, data[ layer ], function(err, tile){ - if ( req.params.format == 'png' || req.params.format == 'pbf'){ - res.sendfile( tile ); + if (req.params.format === 'png' || req.params.format === 'pbf') { + res.sendfile(tile) + } else { + if (callback) { + res.send(callback + '(' + JSON.stringify(tile) + ')') } else { - if ( callback ){ - res.send( callback + '(' + JSON.stringify( tile ) + ')' ); + if (typeof tile === 'string') { + res.sendfile(tile) } else { - if (typeof tile == 'string'){ - res.sendfile( tile ); - } else { - res.json( tile ); - } + res.json(tile) } } - }); + } + }) } - // build the geometry from z,x,y - var bounds = merc.bbox( req.params.x, req.params.y, req.params.z ); - + var bounds = merc.bbox(req.params.x, req.params.y, req.params.z) req.query.geometry = { - xmin: bounds[0], - ymin: bounds[1], - xmax: bounds[2], - ymax: bounds[3], - spatialReference: { wkid: 4326 } - }; + xmin: bounds[0], + ymin: bounds[1], + xmax: bounds[2], + ymax: bounds[3], + spatialReference: { wkid: 4326 } + } - var _sendImmediate = function( file ){ - if ( req.params.format == 'png' || req.params.format == 'pbf'){ - res.sendfile( file ); + var _sendImmediate = function (file) { + if (req.params.format === 'png' || req.params.format === 'pbf') { + res.sendfile(file) } else { - fs.readFile(file, function(err, data){ - if ( callback ){ - res.send( callback + '(' + JSON.parse(data) + ')' ); + fs.readFile(file, function (err, data) { + if (err) { + res.status(404).send(err) + return + } + if (callback) { + res.send(callback + '(' + JSON.parse(data) + ')') } else { - res.json( JSON.parse(data) ); + res.json(JSON.parse(data)) } }) } - }; - - var key = ['socrata', req.params.id, req.params.item].join(':'); - var file = Socrata.files.localDir + '/tiles/'; - file += key + '/' + req.params.format; - file += '/' + req.params.z + '/' + req.params.x + '/' + req.params.y + '.' + req.params.format; - - var jsonFile = file.replace(/png|pbf|utf/g, 'json'); + } + key = ['socrata', req.params.id, req.params.item].join(':') + var file = Socrata.files.localDir + '/tiles/' + file += key + '/' + req.params.format + file += '/' + req.params.z + '/' + req.params.x + '/' + req.params.y + '.' + req.params.format + var jsonFile = file.replace(/png|pbf|utf/g, 'json') // if the json file alreadty exists, dont hit the db, just send the data - if (fs.existsSync(jsonFile) && !fs.existsSync( file ) ){ - _send( null, fs.readFileSync( jsonFile ) ); - } else if ( !fs.existsSync( file ) ) { - Socrata.find(req.params.id, function(err, data){ + if (fs.existsSync(jsonFile) && !fs.existsSync(file)) { + _send(null, fs.readFileSync(jsonFile)) + } else if (!fs.existsSync(file)) { + Socrata.find(req.params.id, function (err, data) { if (err) { - res.send( err, 500); + res.send(err, 500) } else { - // Get the item - Socrata.getResource( data.host, req,params.id, req.params.item, req.query, _send ); + // Get the item + Socrata.getResource(data.host, req, req.params.id, req.params.item, req.query, _send) } - }); + }) } else { - _sendImmediate(file); + _sendImmediate(file) } - - }; - - - controller.thumbnail = function(req, res){ - - // check the image first and return if exists - var key = ['socrata', req.params.id, req.params.item].join(':'); - var dir = Socrata.files.localDir + '/thumbs/'; - req.query.width = parseInt( req.query.width ) || 150; - req.query.height = parseInt( req.query.height ) || 150; - req.query.f_base = dir + key + '/' + req.query.width + '::' + req.query.height; - - var fileName = Socrata.thumbnailExists(key, req.query); - if ( fileName ){ - res.sendfile( fileName ); + } + + // need a thumbnail? get it here... + controller.thumbnail = function (req, res) { + var key = ['socrata', req.params.id, req.params.item].join(':') + var dir = Socrata.files.localDir + '/thumbs/' + req.query.width = parseInt(req.query.width, 0) || 150 + req.query.height = parseInt(req.query.height, 0) || 150 + req.query.f_base = dir + key + '/' + req.query.width + '::' + req.query.height + var fileName = Socrata.thumbnailExists(key, req.query) + + if (fileName) { + res.sendfile(fileName) } else { - - Socrata.find(req.params.id, function(err, data){ + Socrata.find(req.params.id, function (err, data) { if (err) { - res.send( err, 500); + res.send(err, 500) } else { - // Get the item - Socrata.getResource( data.host, req.params.id, req.params.item, req.query, function(error, itemJson){ + // Get the item + Socrata.getResource(data.host, req.params.id, req.params.item, req.query, function (error, itemJson) { if (error) { - res.send( error, 500); + res.send(error, 500) } else { - var key = ['socrata', req.params.id, req.params.item].join(':'); - + var key = ['socrata', req.params.id, req.params.item].join(':') // generate a thumbnail - Socrata.thumbnailExists( itemJson[0], key, req.query, function(err, file){ - if (err){ - res.send(err, 500); + Socrata.thumbnailExists(itemJson[0], key, req.query, function (err, file) { + if (err) { + res.send(err, 500) } else { // send back image - res.sendfile( file ); + res.sendfile(file) } - }); - + }) } - }); + }) } - }); + }) } + } - }; - - - controller.preview = function(req, res){ - res.render(__dirname + '/../views/demo', { locals:{ host: req.params.id, item: req.params.item } }); - }; - - return controller; + controller.preview = function (req, res) { + res.render(__dirname + '/../views/demo', { + locals: { + host: req.params.id, + item: req.params.item + } + }) + } + return controller } -module.exports = Controller; +module.exports = Controller diff --git a/index.js b/index.js index 26b7797..ad48778 100644 --- a/index.js +++ b/index.js @@ -1,5 +1,5 @@ -exports.name = 'Socrata'; -exports.hosts = true; -exports.controller = require('./controller'); -exports.routes = require('./routes'); -exports.model = require('./models/Socrata.js'); +exports.name = 'Socrata' +exports.hosts = true +exports.controller = require('./controller') +exports.routes = require('./routes') +exports.model = require('./models/Socrata.js') diff --git a/models/Socrata.js b/models/Socrata.js index 170a3f4..2086b45 100644 --- a/models/Socrata.js +++ b/models/Socrata.js @@ -1,143 +1,146 @@ -var request = require('request'); +var request = require('request') -var Socrata = function( koop ){ +var Socrata = function (koop) { + var socrata = koop.BaseModel(koop) - var socrata = {}; - socrata.__proto__ = koop.BaseModel( koop ); + // wrap request into a central place + socrata.request = function (url, callback) { + request.get(url, callback) + } // adds a service to the koop.Cache.db // needs a host, generates an id - socrata.register = function( id, host, callback ){ - var type = 'socrata:services'; - koop.Cache.db.serviceCount( type, function(error, count){ - id = id || count++; - koop.Cache.db.serviceRegister( type, {'id': id, 'host': host}, function( err, success ){ - callback( err, id ); - }); - }); - }; + socrata.register = function (id, host, callback) { + var type = 'socrata:services' + koop.Cache.db.serviceCount(type, function (error, count) { + if (error) { + return callback(error, null) + } + id = id || count++ + koop.Cache.db.serviceRegister(type, {'id': id, 'host': host}, function (err, success) { + callback(err, id) + }) + }) + } - socrata.remove = function( id, callback ){ - koop.Cache.db.serviceRemove( 'socrata:services', parseInt(id) || id, callback); - }; + socrata.remove = function (id, callback) { + koop.Cache.db.serviceRemove('socrata:services', parseInt(id, 0) || id, callback) + } // get service by id, no id == return all - socrata.find = function( id, callback ){ - koop.Cache.db.serviceGet( 'socrata:services', parseInt(id) || id, function(err, res){ - if (err){ - callback('No datastores have been registered with this provider yet. Try POSTing {"host":"url", "id":"yourId"} to /socrata', null); - } - else { - callback(null, res); + socrata.find = function (id, callback) { + koop.Cache.db.serviceGet('socrata:services', parseInt(id, 0) || id, function (err, res) { + if (err) { + callback('No datastores have been registered with this provider yet. Try POSTing {"host":"url", "id":"yourId"} to /socrata', null) + } else { + callback(null, res) } - }); - }; + }) + } - socrata.socrata_path = '/resource/'; + socrata.socrata_path = '/resource/' // got the service and get the item - socrata.getResource = function( host, hostId, id, options, callback ){ - var self = this, - type = 'Socrata', + socrata.getResource = function ( host, hostId, id, options, callback ) { + var type = 'Socrata', key = id, locFieldName, urlid, paging = false, - limit = 1000; + limit = 1000 // test id for '!' character indicating presence of a column name and handle if (id.indexOf("!") != -1){ - locFieldName = id.substring(id.indexOf("!") + 1,id.length); - urlid = id.substring(0, id.indexOf("!")); + locFieldName = id.substring(id.indexOf("!") + 1, id.length) + urlid = id.substring(0, id.indexOf("!")) } else { - urlid = id; + urlid = id } // attempt to load from cache, if error perform new request and get first page - koop.Cache.get( type, key, options, function(err, entry ){ - if ( err ){ - var url = host + self.socrata_path + urlid + '.json?$order=:id&$limit=' + limit; - request.get(url, function(err, data, response ){ + koop.Cache.get( type, key, options, function (err, entry ) { + if ( err ) { + var url = host + socrata.socrata_path + urlid + '.json?$order=:id&$limit=' + limit + request.get(url, function(err, data, response ) { if (err) { - callback(err, null); + callback(err, null) } else { // test to see if paging will be needed later - if (Object.keys(JSON.parse(data.body)).length == limit){ - paging = true; + if (Object.keys(JSON.parse(data.body)).length === limit) { + paging = true } // get name of location field try { - var locationField; - if (locFieldName){ - locationField = locFieldName; + var locationField + if (locFieldName) { + locationField = locFieldName } else { - var types = JSON.parse( data.headers['x-soda2-types'] ); - var fields = JSON.parse( data.headers['x-soda2-fields'] ); - types.forEach(function(t,i){ - if (t == 'location'){ - locationField = fields[i]; + var types = JSON.parse( data.headers['x-soda2-types'] ) + var fields = JSON.parse( data.headers['x-soda2-fields'] ) + types.forEach(function (t,i) { + if (t === 'location') { + locationField = fields[i] } - }); } // parse first page to geoJSON and insert - self.toGeojson( JSON.parse( data.body ), locationField, function(err, geojson){ - geojson.updated_at = new Date(data.headers['last-modified']).getTime(); - geojson.name = id; + socrata.toGeojson( JSON.parse( data.body ), locationField, function (err, geojson) { + geojson.updated_at = new Date(data.headers['last-modified']).getTime() + geojson.name = id geojson.host = { id: hostId, url: host - }; - koop.Cache.insert( type, key, geojson, 0, function( err, success){ + } + koop.Cache.insert( type, key, geojson, 0, function ( err, success) { if ( success ) { // check to see if paging is needed - if (paging === false){ - callback( null, [geojson] ); + if (paging === false) { + callback( null, [geojson] ) } else { // create GeoJSON return object - retGeoJSON = geojson; + retGeoJSON = geojson // detrmine count of table and needed pages - var count, pages; - var pagesComplete = 0; - var countUrl = host + self.socrata_path + urlid + '.json?$select=count(*)'; - request.get(countUrl, function(err, data, response){ - count = parseInt(JSON.parse(data.body)[0].count,10); - if ((count/limit) % 1 === 0){ - pages = (count/limit - 1); + var count, pages + var pagesComplete = 0 + var countUrl = host + socrata.socrata_path + urlid + '.json?$select=count(*)' + request.get(countUrl, function (err, data, response) { + count = parseInt(JSON.parse(data.body)[0].count,10) + if ((count/limit) % 1 === 0) { + pages = (count/limit - 1) } else { - pages = Math.floor(count/limit); + pages = Math.floor(count/limit) } // page through data - for (var p = 1; p <= pages; p++){ - var pUrl = host + self.socrata_path + urlid + '.json?$order=:id&$limit=' + limit + '&$offset=' + (p*limit); - request.get(pUrl,function(err, data, response){ + for (var p = 1; p <= pages; p++) { + var pUrl = host + socrata.socrata_path + urlid + '.json?$order=:id&$limit=' + limit + '&$offset=' + (p*limit) + request.get(pUrl,function (err, data, response) { // parse pages to GeoJSON and insert partial - self.toGeojson( JSON.parse( data.body ), locationField, function(err, geojson){ - geojson.updated_at = new Date(data.headers['last-modified']).getTime(); - geojson.name = id; + socrata.toGeojson( JSON.parse( data.body ), locationField, function (err, geojson) { + geojson.updated_at = new Date(data.headers['last-modified']).getTime() + geojson.name = id geojson.host = { id: hostId, url: host }; - koop.Cache.insertPartial( type, key, geojson, 0, function( err, success){ + koop.Cache.insertPartial( type, key, geojson, 0, function ( err, success) { if ( success ) { // append geojson to return object - for (f = 0; f < geojson.features.length; f++){ - retGeoJSON.features.push(geojson.features[f]); + for (f = 0; f < geojson.features.length; f++) { + retGeoJSON.features.push(geojson.features[f]) } // update pages completed and check for completion of pages - pagesComplete++; - checkDone(); + pagesComplete++ + checkDone() } - }); - }); - }); + }) + }) + }) } // function to check completion of pages @@ -147,126 +150,127 @@ var Socrata = function( koop ){ } else { } - }; - }); + } + }) } } - }); - }); - } catch (e){ - console.log('Error?', e); - koop.log.error('Unable to parse response %s', url); - callback(e, null); + }) + }) + } catch (e) { + console.log('Error?', e) + koop.log.error('Unable to parse response %s', url) + callback(e, null) } - } - }); + }) + }) } else { - callback( null, entry ); + callback(null, entry) } - }); + }) + } - }; - - socrata.toGeojson = function(json, locationField, callback){ - if (!json || !json.length){ - callback('Error converting data to geojson', null); + socrata.toGeojson = function (json, locationField, callback) { + if (!json || !json.length) { + callback('Error converting data to geojson', null) } else { - var geojson = {type: 'FeatureCollection', features: []}; - var geojsonFeature; - json.forEach(function(feature, i){ - var lat, lon; - geojsonFeature = {type: 'Feature', geometry: {}, id: i+1}; - if (feature && locationField){ - lon = parseFloat(feature[locationField].longitude); - lat = parseFloat(feature[locationField].latitude); - if ((lon < -180 || lon > 180) || (lat < -90 || lat > 90)){ - geojsonFeature.geometry = null; - geojsonFeature.properties = feature; - geojson.features.push( geojsonFeature ); + var geojson = { type: 'FeatureCollection', features: [] } + var geojsonFeature + json.forEach(function (feature, i) { + var lat, lon + geojsonFeature = { type: 'Feature', geometry: {}, id: i+1 } + if (feature && locationField) { + lon = parseFloat(feature[locationField].longitude) + lat = parseFloat(feature[locationField].latitude) + if ((lon < -180 || lon > 180) || (lat < -90 || lat > 90)) { + geojsonFeature.geometry = null + geojsonFeature.properties = feature + geojson.features.push( geojsonFeature ) } else { - geojsonFeature.geometry.coordinates = [lon, lat]; - geojsonFeature.geometry.type = 'Point'; - delete feature.location; - geojsonFeature.properties = feature; - geojson.features.push( geojsonFeature ); + geojsonFeature.geometry.coordinates = [lon, lat] + geojsonFeature.geometry.type = 'Point' + delete feature.location + geojsonFeature.properties = feature + geojson.features.push( geojsonFeature ) } - } else if ( feature && feature.latitude && feature.longitude ){ - lon = parseFloat(feature.longitude); - lat = parseFloat(feature.latitude); - if ((lon < -180 || lon > 180) || (lat < -90 || lat > 90)){ - geojsonFeature.geometry = null; - geojsonFeature.properties = feature; - geojson.features.push( geojsonFeature ); + } + else if ( feature && feature.latitude && feature.longitude ) { + lon = parseFloat(feature.longitude) + lat = parseFloat(feature.latitude) + if ((lon < -180 || lon > 180) || (lat < -90 || lat > 90)) { + geojsonFeature.geometry = null + geojsonFeature.properties = feature + geojson.features.push( geojsonFeature ) } else { - geojsonFeature.geometry.coordinates = [lon, lat]; - geojsonFeature.geometry.type = 'Point'; - geojsonFeature.properties = feature; - geojson.features.push( geojsonFeature ); + geojsonFeature.geometry.coordinates = [lon, lat] + geojsonFeature.geometry.type = 'Point' + geojsonFeature.properties = feature + geojson.features.push( geojsonFeature ) } } else { - geojsonFeature.geometry = null; - geojsonFeature.properties = feature; - geojson.features.push( geojsonFeature ); + geojsonFeature.geometry = null + geojsonFeature.properties = feature + geojson.features.push(geojsonFeature) } - }); - callback(null, geojson); + }) + callback(null, geojson) } - }; + } // compares the sha on the cached data and the hosted data // this method name is special reserved name that will get called by the cache model - socrata.checkCache = function(key, data, options, callback){ - var self = this; - var url = data.host + this.socrata_path + key + '.json'; - - var lapsed = (new Date().getTime() - data.updated_at); - if (typeof(data.updated_at) == "undefined" || (lapsed > (1000*60*60))){ - callback(null, false); + socrata.checkCache = function (key, data, options, callback) { + var url = data.host + this.socrata_path + key + '.json' + var lapsed = (new Date().getTime() - data.updated_at) + if (typeof (data.updated_at) === 'undefined' || (lapsed > (1000 * 60 * 60))) { + callback(null, false) } else { - request.get(url, function( err, data, response ){ + socrata.request(url, function (err, data, response) { if (err) { - callback( err, null ); + callback(err, null) } else { - var types = JSON.parse( data.headers['x-soda2-types'] ); - var fields = JSON.parse( data.headers['x-soda2-fields'] ); - var locationField; - types.forEach(function(t,i){ - if (t == 'location'){ - locationField = fields[i]; + var types = JSON.parse(data.headers['x-soda2-types']) + var fields = JSON.parse(data.headers['x-soda2-fields']) + var locationField + types.forEach(function (t, i) { + if (t === 'location') { + locationField = fields[i] } - }); - self.toGeojson( JSON.parse( data.body ), locationField, function( error, geojson ){ - geojson.updated_at = new Date(data.headers['last-modified']).getTime(); - geojson.name = data.name || key; - geojson.host = data.host; - callback( error, [geojson] ); - }); + }) + socrata.toGeojson(JSON.parse(data.body), locationField, function (error, geojson) { + geojson.updated_at = new Date(data.headers['last-modified']).getTime() + geojson.name = data.name || key + geojson.host = data.host + callback(error, [geojson]) + }) } - }); + }) } + } - }; - - // drops the item from the cache - socrata.dropItem = function( host, itemId, options, callback ){ - var dir = [ 'socrata', host, itemId].join(':'); - koop.Cache.remove('Socrata', itemId, options, function(err, res){ - koop.files.removeDir( 'files/' + dir, function(err, res){ - koop.files.removeDir( 'tiles/'+ dir, function(err, res){ - koop.files.removeDir( 'thumbs/'+ dir, function(err, res){ - callback(err, true); - }); - }); - }); - }); - }; - - return socrata; + // drops the item from the cache + socrata.dropItem = function (host, itemId, options, callback) { + var dir = [ 'socrata', host, itemId].join(':'), + errors = [] -}; + koop.Cache.remove('Socrata', itemId, options, function (err, res) { + if (err) errors.push(err) + koop.files.removeDir('files/' + dir, function (err, res) { + if (err) errors.push(err) + koop.files.removeDir('tiles/' + dir, function (err, res) { + if (err) errors.push(err) + koop.files.removeDir('thumbs/' + dir, function (err, res) { + if (err) errors.push(err) + callback(errors.join(', '), true) + }) + }) + }) + }) + } + return socrata -module.exports = Socrata; +} +module.exports = Socrata diff --git a/package.json b/package.json index 7343ebc..0428a21 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,10 @@ { "name": "koop-socrata", - "version": "0.0.9", + "version": "0.1.0", "description": "A socrata wrapper for koop ", "main": "index.js", - "directories": { - "test": "test" - }, "scripts": { - "test": "mocha test/ -t 5000" + "test": "standard && node test/*.js | tap-spec" }, "dependencies": { "config": "~0.4.35", @@ -19,9 +16,12 @@ "author": "Chris Helm", "license": "BSD-2-Clause", "devDependencies": { + "koop": "~1.0.0", + "sinon": "^1.14.1", + "standard": "^3.3.0", "supertest": "~0.11.0", - "should": "~3.3.1", - "koop": "~1.0.0" + "tap-spec": "^2.2.2", + "tape": "^3.5.0" }, "repository": { "type": "git", diff --git a/test/model-test.js b/test/model-test.js deleted file mode 100644 index 845fef9..0000000 --- a/test/model-test.js +++ /dev/null @@ -1,58 +0,0 @@ -var should = require('should'), - config = require('config'), - koop = require('koop/lib'); - -var data = require('./fixtures/earthquakes.json'); - -before(function(done){ - // setup koop - koop.Cache.db = koop.PostGIS.connect( config.db.postgis.conn ); - var data_dir = __dirname + '/output/'; - koop.Cache.data_dir = data_dir; - Socrata = new require('../models/Socrata.js')( koop ); - done(); -}); - -describe('Socrata Model', function(){ - - afterEach(function(done){ - done(); - }); - - describe('socrata model methods', function() { - before(function(done ){ - done(); - }); - it('toGeoJSON should err when given no data', function(done) { - Socrata.toGeojson([], 'location', function(err, geojson){ - should.exist(err); - should.not.exist( geojson ); - return done(); - }); - }); - - it('toGeoJSON should return geojson', function(done) { - Socrata.toGeojson(data, 'location', function(err, geojson){ - should.not.exist(err); - should.exist( geojson ); - geojson.features.length.should.not.equal(0); - return done(); - }); - }); - - it('getResource should return geojson', function(done) { - Socrata.getResource('https://data.seattle.gov', 'seattle', '2tje-83f6', {}, function(err, geojson){ - should.not.exist(err); - should.exist( geojson ); - geojson[0].features.length.should.not.equal(0); - return done(); - }); - }); - - }); - -}); - - - - diff --git a/test/routes-test.js b/test/routes-test.js deleted file mode 100644 index 683a0f6..0000000 --- a/test/routes-test.js +++ /dev/null @@ -1,122 +0,0 @@ -var should = require('should'), - request = require('supertest'), - config = require('config'), - koop = require('koop-server')(config), - kooplib = require('koop-server/lib'); - -before(function(done){ - var provider = require('../index.js'); - agol = new provider.model( kooplib ); - controller = new provider.controller( agol ); - koop._bindRoutes( provider.routes, controller ); - done(); -}); - -var resource = 'f7f2-ggz5'; - -describe('Koop Routes', function(){ - - before(function(done){ - request(koop) - .post('/socrata') - .set('Content-Type', 'application/json') - .send({ - 'host': 'https://data.cityofchicago.org', - 'id': 'tester' - }) - .end(function(err, res){ - //res.should.have.status(200); - done(); - }); - }); - - after(function(done){ - request(koop) - .del('/socrata/tester') - .end(function(err, res){ - //res.should.have.status(200); - done(); - }); - }); - - - describe('/socrata routes', function() { - it('register should return 500 when POSTing w/o a host', function(done) { - request(koop) - .post('/socrata') - .end(function(err, res){ - res.should.have.status(500); - done(); - }); - }); - - it('should return 200 when GETing all registered providers', function(done) { - request(koop) - .get('/socrata') - .end(function(err, res){ - res.should.have.status( 200 ); - done(); - }); - }); - - it('should return 200 when GETing a registered provider', function(done) { - request(koop) - .get('/socrata/tester') - .end(function(err, res){ - res.should.have.status( 200 ); - done(); - }); - }); - - it('should return 404 when GETing an unknown provider/host', function(done) { - request(koop) - .get('/socrata/bogus') - .end(function(err, res){ - res.should.have.status( 404 ); - done(); - }); - }); - - it('should return 200 when accessing item data', function(done) { - request(koop) - .get('/socrata/tester/' + resource ) - .end(function(err, res){ - res.should.have.status( 200 ); - should.not.exist(err); - done(); - }); - }); - - it('should return 200 when accessing item as a featureservice', function(done) { - request(koop) - .get('/socrata/tester/' + resource + '/FeatureServer') - .end(function(err, res){ - res.should.have.status( 200 ); - should.not.exist(err); - done(); - }); - }); - - it('should return 200 when accessing item as a featureservice layer', function(done) { - request(koop) - .get('/socrata/tester/' + resource + '/FeatureServer/0') - .end(function(err, res){ - res.should.have.status( 200 ); - should.not.exist(err); - done(); - }); - }); - - it('should return 200 when accessing item as a featureservice query', function(done) { - request(koop) - .get('/socrata/tester/' + resource + '/FeatureServer/0/query') - .end(function(err, res){ - res.should.have.status( 200 ); - should.not.exist(err); - done(); - }); - }); - - }); - -}); From f3d3f682f9386ba135da9adb5af40a2764b548a9 Mon Sep 17 00:00:00 2001 From: chelm Date: Tue, 21 Apr 2015 10:03:36 -0600 Subject: [PATCH 11/16] adding tests --- test/controller.js | 103 +++++++++++++++++++++++++++++++++++++++++++++ test/model.js | 83 ++++++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 test/controller.js create mode 100644 test/model.js diff --git a/test/controller.js b/test/controller.js new file mode 100644 index 0000000..b382e17 --- /dev/null +++ b/test/controller.js @@ -0,0 +1,103 @@ +var koop = require('koop')({}), + kooplib = require('koop/lib'), + sinon = require('sinon'), + test = require('tape'), + request = require('supertest') + +// use Koop's local cache as a db for tests +kooplib.Cache = new kooplib.DataCache(koop) +kooplib.Cache.db = kooplib.LocalDB + +var provider = require('../index.js'), + model = provider.model(kooplib), + controller = provider.controller(model, kooplib.BaseController) + +koop._bindRoutes(provider.routes, controller) + +var sample_id = 'seattle', + sample_host = 'https://data.seattle.gov' + +// In the setup test we create several stubs that squash the +// normal behavoir of the model/controller methods. +// This allows us the only test the controller & routing and not the model here +test('setup', function (t) { + sinon.stub(model, 'register', function (id, host, callback) { + callback(null, id) + }) + sinon.stub(model, 'find', function (id, callback) { + callback(null, [{ 'id': sample_id, 'host': sample_host }]) + }) + sinon.stub(model, 'dropItem', function (host, item, options, callback) { + callback(null, true) + }) + sinon.stub(model, 'getResource', function (host, id, item, options, callback) { + callback(null, {}) + }) + sinon.stub(controller, 'processFeatureServer', function (req, res, err, geojson, callback) { + res.send({}) + }) + t.end() +}) + +test('register a socrata instance', function (t) { + request(koop) + .post('/socrata') + .set('Content-Type', 'application/json') + .send({ + 'host': sample_host, + 'id': sample_id + }) + .end(function () { + t.equals(model.register.called, true) + t.end() + }) +}) + +test('list the registered socrata instances', function (t) { + request(koop) + .get('/socrata') + .set('Content-Type', 'application/json') + .end(function () { + t.equals(model.find.called, true) + t.end() + }) +}) + +test('getting items calls the models find and findResource method', function (t) { + request(koop) + .get('/socrata/seattle/fake') + .end(function () { + t.equals(model.find.called, true) + t.equals(model.getResource.called, true) + t.end() + }) +}) + +test('dropping items calls the models dropItem method', function (t) { + request(koop) + .get('/socrata/seattle/fake/drop') + .end(function () { + t.equals(model.dropItem.called, true) + t.end() + }) +}) + +test('getting a featureservice calls the model find, getResource methods and controllers featureservice method', function (t) { + request(koop) + .get('/socrata/seattle/fake/FeatureServer') + .end(function () { + t.equals(model.find.called, true) + t.equals(model.getResource.called, true) + t.equals(controller.processFeatureServer.called, true) + t.end() + }) +}) + +test('teardown', function (t) { + model.register.restore() + model.find.restore() + model.dropItem.restore() + model.getResource.restore() + controller.processFeatureServer.restore() + t.end() +}) diff --git a/test/model.js b/test/model.js new file mode 100644 index 0000000..f9c4463 --- /dev/null +++ b/test/model.js @@ -0,0 +1,83 @@ +var koop = require('koop/lib'), + test = require('tape'), + sinon = require('sinon'), + fs = require('fs') + +// use Koop's local cache as a db for tests +koop.Cache = new koop.DataCache(koop) +koop.Cache.db = koop.LocalDB + +var socrata = require('../models/Socrata.js')(koop) +var data = JSON.parse(fs.readFileSync(__dirname + '/fixtures/earthquakes.json')) + +var id = 'seattle', + host = 'https://data.seattle.gov' + +test('adding a socrata instance', function (t) { + socrata.register(id, host, function (err, success) { + if (err) throw err + t.deepEqual(success, id) + t.end() + }) +}) + +test('parsing geojson', function (t) { + t.plan(2) + + socrata.toGeojson([], 'location', function (err, geojson) { + t.deepEqual(err, 'Error converting data to geojson') + }) + + socrata.toGeojson(data, 'location', function (err, geojson) { + if (err) throw err + t.deepEqual(geojson.features.length, 1000) + }) + +}) + +test('stub the request method', function (t) { + sinon.stub(socrata, 'request', function (url, callback) { + callback(null, { + 'body': '{ "features": [], "name": "Test" }', + 'headers': { + 'x-soda2-types': '[]', + 'x-soda2-fields': '[]' + } + }) + }) + + var feature = { + type: 'Feature', + properties: {}, + geometry: { + type: 'Point', + coordinates: [0, 0] + } + } + + sinon.stub(socrata, 'toGeojson', function (features, locationField, callback) { + callback(null, { features: [feature] }) + }) + sinon.stub(koop.Cache, 'insert', function (type, key, geojson, layer, callback) { + callback(null, [{ features: [feature] }]) + }) + t.end() +}) + +// This test wont close +test('requesting data', function (t) { + socrata.getResource(host, id, '2tje-83f6', {}, function (err, geojson) { + if (err) throw err + t.equal(socrata.request.called, true) + t.equal(socrata.toGeojson.called, true) + t.deepEqual(geojson[0].features.length, 1) + t.end() + }) +}) + +test('teardown', function (t) { + socrata.request.restore() + socrata.toGeojson.restore() + koop.Cache.insert.restore() + t.end() +}) From c111a32149ac863296dbb8be8d8b65d325229efe Mon Sep 17 00:00:00 2001 From: chelm Date: Tue, 21 Apr 2015 10:54:16 -0600 Subject: [PATCH 12/16] rebasing master and making it all standardized --- .travis.yml | 12 +++++ models/Socrata.js | 126 +++++++++++++++++++++++++--------------------- 2 files changed, 80 insertions(+), 58 deletions(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..cb10dad --- /dev/null +++ b/.travis.yml @@ -0,0 +1,12 @@ +language: node_js + +node_js: + - '0.12' + - '0.10' +sudo: false # Enable docker-based containers +cache: + directories: # Cache dependencies + - node_modules + +script: + - npm test diff --git a/models/Socrata.js b/models/Socrata.js index 2086b45..014e836 100644 --- a/models/Socrata.js +++ b/models/Socrata.js @@ -41,7 +41,7 @@ var Socrata = function (koop) { socrata.socrata_path = '/resource/' // got the service and get the item - socrata.getResource = function ( host, hostId, id, options, callback ) { + socrata.getResource = function (host, hostId, id, options, callback) { var type = 'Socrata', key = id, locFieldName, @@ -50,90 +50,103 @@ var Socrata = function (koop) { limit = 1000 // test id for '!' character indicating presence of a column name and handle - if (id.indexOf("!") != -1){ - locFieldName = id.substring(id.indexOf("!") + 1, id.length) - urlid = id.substring(0, id.indexOf("!")) - } - else { + if (id.indexOf('!') !== -1) { + locFieldName = id.substring(id.indexOf('!') + 1, id.length) + urlid = id.substring(0, id.indexOf('!')) + } else { urlid = id } // attempt to load from cache, if error perform new request and get first page - koop.Cache.get( type, key, options, function (err, entry ) { - if ( err ) { + koop.Cache.get(type, key, options, function (err, entry) { + if (err) { var url = host + socrata.socrata_path + urlid + '.json?$order=:id&$limit=' + limit - request.get(url, function(err, data, response ) { + request.get(url, function (err, data, response) { if (err) { callback(err, null) - } else { - + } else { // test to see if paging will be needed later if (Object.keys(JSON.parse(data.body)).length === limit) { paging = true } - // get name of location field try { var locationField if (locFieldName) { locationField = locFieldName - } - else { - var types = JSON.parse( data.headers['x-soda2-types'] ) - var fields = JSON.parse( data.headers['x-soda2-fields'] ) - types.forEach(function (t,i) { + } else { + var types = JSON.parse(data.headers['x-soda2-types']) + var fields = JSON.parse(data.headers['x-soda2-fields']) + types.forEach(function (t, i) { if (t === 'location') { locationField = fields[i] } + }) } // parse first page to geoJSON and insert - socrata.toGeojson( JSON.parse( data.body ), locationField, function (err, geojson) { + socrata.toGeojson(JSON.parse(data.body), locationField, function (err, geojson) { + if (err) { + return callback(err) + } geojson.updated_at = new Date(data.headers['last-modified']).getTime() geojson.name = id geojson.host = { id: hostId, url: host } - koop.Cache.insert( type, key, geojson, 0, function ( err, success) { - if ( success ) { + koop.Cache.insert(type, key, geojson, 0, function (err, success) { + if (err) { + return callback(err) + } + if (success) { // check to see if paging is needed if (paging === false) { - callback( null, [geojson] ) - } - else { + callback(null, [geojson]) + } else { // create GeoJSON return object - retGeoJSON = geojson + var retGeoJSON = geojson // detrmine count of table and needed pages - var count, pages - var pagesComplete = 0 - var countUrl = host + socrata.socrata_path + urlid + '.json?$select=count(*)' + var count, pages, + pagesComplete = 0, + countUrl = host + socrata.socrata_path + urlid + '.json?$select=count(*)' request.get(countUrl, function (err, data, response) { - count = parseInt(JSON.parse(data.body)[0].count,10) - if ((count/limit) % 1 === 0) { - pages = (count/limit - 1) + if (err) { + return callback(err) } - else { - pages = Math.floor(count/limit) + count = parseInt(JSON.parse(data.body)[0].count, 10) + if ((count / limit) % 1 === 0) { + pages = (count / limit - 1) + } else { + pages = Math.floor(count / limit) } // page through data for (var p = 1; p <= pages; p++) { - var pUrl = host + socrata.socrata_path + urlid + '.json?$order=:id&$limit=' + limit + '&$offset=' + (p*limit) - request.get(pUrl,function (err, data, response) { + var pUrl = host + socrata.socrata_path + urlid + '.json?$order=:id&$limit=' + limit + '&$offset=' + (p * limit) + request.get(pUrl, function (err, data, response) { + if (err) { + return callback(err) + } // parse pages to GeoJSON and insert partial - socrata.toGeojson( JSON.parse( data.body ), locationField, function (err, geojson) { + socrata.toGeojson(JSON.parse(data.body), locationField, function (err, geojson) { + if (err) { + return callback(err) + } geojson.updated_at = new Date(data.headers['last-modified']).getTime() geojson.name = id geojson.host = { id: hostId, url: host - }; - koop.Cache.insertPartial( type, key, geojson, 0, function ( err, success) { - if ( success ) { + } + koop.Cache.insertPartial(type, key, geojson, 0, function (err, success) { + if (err) { + return callback(err) + } + if (success) { // append geojson to return object - for (f = 0; f < geojson.features.length; f++) { - retGeoJSON.features.push(geojson.features[f]) - } + geojson.features.forEach(function (f) { + retGeoJSON.features.push(f) + }) // update pages completed and check for completion of pages pagesComplete++ checkDone() @@ -144,11 +157,11 @@ var Socrata = function (koop) { } // function to check completion of pages - var checkDone = function(){ - if (pagesComplete == pages){ - callback( null, [retGeoJSON]) - } - else { + var checkDone = function () { + if (pagesComplete === pages) { + callback(null, [retGeoJSON]) + } else { + } } }) @@ -161,7 +174,7 @@ var Socrata = function (koop) { koop.log.error('Unable to parse response %s', url) callback(e, null) } - }) + } }) } else { callback(null, entry) @@ -177,36 +190,33 @@ var Socrata = function (koop) { var geojsonFeature json.forEach(function (feature, i) { var lat, lon - geojsonFeature = { type: 'Feature', geometry: {}, id: i+1 } + geojsonFeature = { type: 'Feature', geometry: {}, id: i + 1 } if (feature && locationField) { lon = parseFloat(feature[locationField].longitude) lat = parseFloat(feature[locationField].latitude) if ((lon < -180 || lon > 180) || (lat < -90 || lat > 90)) { geojsonFeature.geometry = null geojsonFeature.properties = feature - geojson.features.push( geojsonFeature ) - } - else { + geojson.features.push(geojsonFeature) + } else { geojsonFeature.geometry.coordinates = [lon, lat] geojsonFeature.geometry.type = 'Point' delete feature.location geojsonFeature.properties = feature - geojson.features.push( geojsonFeature ) + geojson.features.push(geojsonFeature) } - } - else if ( feature && feature.latitude && feature.longitude ) { + } else if (feature && feature.latitude && feature.longitude) { lon = parseFloat(feature.longitude) lat = parseFloat(feature.latitude) if ((lon < -180 || lon > 180) || (lat < -90 || lat > 90)) { geojsonFeature.geometry = null geojsonFeature.properties = feature - geojson.features.push( geojsonFeature ) - } - else { + geojson.features.push(geojsonFeature) + } else { geojsonFeature.geometry.coordinates = [lon, lat] geojsonFeature.geometry.type = 'Point' geojsonFeature.properties = feature - geojson.features.push( geojsonFeature ) + geojson.features.push(geojsonFeature) } } else { geojsonFeature.geometry = null From 2e65d4f6617db732c2bb4a4aa074396474be7810 Mon Sep 17 00:00:00 2001 From: chelm Date: Tue, 21 Apr 2015 11:01:27 -0600 Subject: [PATCH 13/16] adding the change log --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ff50229 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,11 @@ +# Change Log +All notable changes to this project will be documented in this file. +This project adheres to [Semantic Versioning](http://semver.org/). + +## [0.1.0] - 2015-04-21 +### Changed +* This project now uses `standard` as its code formatting +* Keeping a legit changelog +* Added tape testing with sinon stubs in the controller tests + +[0.1.0]: https://github.com/Esri/koop/releases/tag/v0.1.0 From 3c8bc2e0ab428417c7ad11a69e95f18dcab1bd7b Mon Sep 17 00:00:00 2001 From: chelm Date: Tue, 21 Apr 2015 11:18:07 -0600 Subject: [PATCH 14/16] updating standard to pull the latest at all costs --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0428a21..7b9e754 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "devDependencies": { "koop": "~1.0.0", "sinon": "^1.14.1", - "standard": "^3.3.0", + "standard": "*", "supertest": "~0.11.0", "tap-spec": "^2.2.2", "tape": "^3.5.0" From a03b98beee77cf1c2020cd9646747714ce58162a Mon Sep 17 00:00:00 2001 From: chelm Date: Tue, 21 Apr 2015 11:44:54 -0600 Subject: [PATCH 15/16] updating the package to use the latest koop in dev as well --- package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package.json b/package.json index 7b9e754..a3b17dc 100644 --- a/package.json +++ b/package.json @@ -9,14 +9,13 @@ "dependencies": { "config": "~0.4.35", "ejs": "~1.0.0", - "node.extend": "~1.0.10", "request": "^2.51.0", "sphericalmercator": "~1.0.2" }, "author": "Chris Helm", "license": "BSD-2-Clause", "devDependencies": { - "koop": "~1.0.0", + "koop": "*", "sinon": "^1.14.1", "standard": "*", "supertest": "~0.11.0", From 0ef5e52f5310129490ca531fcb4043ad642eeb72 Mon Sep 17 00:00:00 2001 From: chelm Date: Mon, 4 May 2015 12:25:43 -0600 Subject: [PATCH 16/16] adding a limit to the controller queries --- controller/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/controller/index.js b/controller/index.js index 32581c2..1081877 100644 --- a/controller/index.js +++ b/controller/index.js @@ -136,6 +136,7 @@ var Controller = function (Socrata, BaseController) { res.send(err, 500) } else { // Get the item + req.query.limit = 10000000; Socrata.getResource(data.host, req.params.id, req.params.item, req.query, function (error, geojson) { if (error) { res.send(error, 500)