experiment importing statcan data

bin/cmd/import.js

@@ -9,7 +9,7 @@ module.exports = {
     yargs.positional('source', {
       type: 'string',
       describe: 'name of data source',
-      choices: ['whosonfirst', 'osmium', 'zcta'],
+      choices: ['whosonfirst', 'osmium', 'zcta', 'statcan'],
       demand: 'source is required'
     })
 

import/source/statcan.js

@@ -0,0 +1,8 @@
+const file = require('../../import/file')
+
+module.exports = {
+  ingress: file,
+  record_separator: /\r?\n/,
+  format: 'json',
+  mapper: require('./statcan/map/place')
+}

import/source/statcan/download.txt

@@ -0,0 +1 @@
+https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/bound-limit-eng.cfm

import/source/statcan/map/geometries.js

@@ -0,0 +1,17 @@
+const _ = require('lodash')
+const format = require('../../../format')
+const Geometry = require('../../../../model/Geometry')
+
+function mapper (place, doc) {
+  const geometry = _.get(doc, 'geometry')
+  const isPolygon = _.get(geometry, 'type', '').trim().toUpperCase().endsWith('POLYGON')
+
+  if (geometry) {
+    place.addGeometry(new Geometry(
+      format.from('geometry', 'geojson', geometry),
+      isPolygon ? 'boundary' : 'centroid'
+    ))
+  }
+}
+
+module.exports = mapper

import/source/statcan/map/geometries.test.js

@@ -0,0 +1,112 @@
+const Place = require('../../../../model/Place')
+const Geometry = require('../../../../model/Geometry')
+const map = require('./geometries')
+
+module.exports.tests = {}
+
+module.exports.tests.mapper = (test) => {
+  test('mapper: geometry empty', (t) => {
+    let p = new Place()
+    map(p, {})
+
+    t.equals(p.geometry.length, 0)
+    t.end()
+  })
+  test('mapper: maps polygon', (t) => {
+    let p = new Place()
+    map(p, {
+      geometry: {
+        'type': 'Polygon',
+        'coordinates': [
+          [
+            [
+              39.0234375,
+              48.922499263758255
+            ],
+            [
+              47.8125,
+              39.639537564366684
+            ],
+            [
+              61.17187499999999,
+              49.83798245308484
+            ],
+            [
+              39.0234375,
+              48.922499263758255
+            ]
+          ]
+        ]
+      }
+    })
+    t.equals(p.geometry.length, 1)
+    t.true(p.geometry[0] instanceof Geometry)
+    t.equal(p.geometry[0].geometry.constructor.name.toUpperCase(), 'POLYGON')
+    t.equal(p.geometry[0].role, 'boundary')
+    t.end()
+  })
+  test('mapper: maps point', (t) => {
+    let p = new Place()
+    map(p, {
+      geometry: {
+        'type': 'Point',
+        'coordinates': [
+          59.0625,
+          47.989921667414194
+        ]
+      }
+    })
+    t.equals(p.geometry.length, 1)
+    t.true(p.geometry[0] instanceof Geometry)
+    t.equal(p.geometry[0].geometry.constructor.name.toUpperCase(), 'POINT')
+    t.equal(p.geometry[0].role, 'centroid')
+    t.end()
+  })
+  test('mapper: internal point', (t) => {
+    let p = new Place()
+    map(p, {
+      geometry: {
+        'type': 'Polygon',
+        'coordinates': [
+          [
+            [
+              39.0234375,
+              48.922499263758255
+            ],
+            [
+              47.8125,
+              39.639537564366684
+            ],
+            [
+              61.17187499999999,
+              49.83798245308484
+            ],
+            [
+              39.0234375,
+              48.922499263758255
+            ]
+          ]
+        ]
+      },
+      properties: {
+        'INTPTLON10': 1.1,
+        'INTPTLAT10': 2.2
+      }
+    })
+    t.equals(p.geometry.length, 2)
+    t.true(p.geometry[1] instanceof Geometry)
+    t.equal(p.geometry[1].geometry.constructor.name.toUpperCase(), 'POINT')
+    t.equal(p.geometry[1].role, 'centroid')
+    t.end()
+  })
+}
+
+module.exports.all = (tape) => {
+  function test (name, testFunction) {
+    return tape(`geometries: ${name}`, testFunction)
+  }
+
+  for (var testCase in module.exports.tests) {
+    module.exports.tests[testCase](test)
+  }
+}

import/source/statcan/map/names.js

@@ -0,0 +1,9 @@
+const _ = require('lodash')
+const Name = require('../../../../model/Name')
+
+function mapper (place, properties) {
+  // generic name properties
+  place.addName(new Name('eng', 'default', false, _.get(properties, 'CDNAME', '').trim()))
+}
+
+module.exports = mapper

import/source/statcan/map/names.test.js

@@ -0,0 +1,35 @@
+const Place = require('../../../../model/Place')
+const map = require('./names')
+
+module.exports.tests = {}
+
+module.exports.tests.mapper = (test) => {
+  test('mapper: properties empty', (t) => {
+    let p = new Place()
+    map(p, {})
+
+    t.equals(p.name.length, 0)
+    t.end()
+  })
+  test('mapper: name', (t) => {
+    let p = new Place()
+    map(p, { 'ZCTA5CE10': ' example1 ' })
+
+    t.equals(p.name.length, 1)
+    t.equals(p.name[0].lang, 'und')
+    t.equals(p.name[0].tag, 'default')
+    t.equals(p.name[0].abbr, false)
+    t.equals(p.name[0].name, 'example1')
+    t.end()
+  })
+}
+
+module.exports.all = (tape) => {
+  function test (name, testFunction) {
+    return tape(`names: ${name}`, testFunction)
+  }
+
+  for (var testCase in module.exports.tests) {
+    module.exports.tests[testCase](test)
+  }
+}

import/source/statcan/map/place.js

@@ -0,0 +1,42 @@
+const _ = require('lodash')
+const Identity = require('../../../../model/Identity')
+const Ontology = require('../../../../model/Ontology')
+const Place = require('../../../../model/Place')
+
+const map = {
+  properties: require('./properties'),
+  names: require('./names'),
+  geometries: require('./geometries')
+}
+
+// {
+//   "CSDUID": "5933045",
+//   "CSDNAME": "Sun Peaks Mountain",
+//   "CSDTYPE": "VL",
+//   "PRUID": "59",
+//   "PRNAME": "British Columbia / Colombie-Britannique",
+//   "CDUID": "5933",
+//   "CDNAME": "Thompson-Nicola",
+//   "CDTYPE": "RD"
+// }
+
+function mapper (doc) {
+  // get document properties
+  const properties = _.get(doc, 'properties')
+  if (!_.isPlainObject(properties)) { return null }
+
+  // instantiate a new place
+  const place = new Place(
+    new Identity('statcan', _.get(properties, 'CDUID', '').trim()),
+    new Ontology('admin', _.get(properties, 'CDTYPE', '').trim())
+  )
+
+  // run mappers
+  map.properties(place, properties)
+  map.names(place, properties)
+  map.geometries(place, doc)
+
+  return place
+}
+
+module.exports = mapper

import/source/statcan/map/place.test.js

@@ -0,0 +1,48 @@
+const Place = require('../../../../model/Place')
+const map = require('./place')
+
+module.exports.tests = {}
+
+module.exports.tests.mapper = (test) => {
+  test('mapper: properties empty', (t) => {
+    let place = map({})
+    t.equal(place, null)
+    t.end()
+  })
+  test('mapper: maps identity & ontology', (t) => {
+    let place = map({
+      properties: {
+        'ZCTA5CE10': '90210'
+      }
+    })
+    t.true(place instanceof Place)
+    t.equal(place.identity.source, 'uscensus')
+    t.equal(place.identity.id, 'zcta:90210')
+    t.equal(place.ontology.class, 'admin')
+    t.equal(place.ontology.type, 'postalcode')
+    t.end()
+  })
+  test('mapper: maps geometry', (t) => {
+    let place = map({
+      properties: {
+        'ZCTA5CE10': '90210'
+      },
+      geometry: require('../../../../test/fixture/geojson.triangle')
+    })
+    t.true(place instanceof Place)
+    t.equal(place.geometry.length, 1)
+    t.equal(place.geometry[0].geometry.constructor.name.toUpperCase(), 'POLYGON')
+    t.equal(place.geometry[0].role, 'boundary')
+    t.end()
+  })
+}
+
+module.exports.all = (tape) => {
+  function test (name, testFunction) {
+    return tape(`place: ${name}`, testFunction)
+  }
+
+  for (var testCase in module.exports.tests) {
+    module.exports.tests[testCase](test)
+  }
+}

import/source/statcan/map/properties.js

@@ -0,0 +1,9 @@
+const Property = require('../../../../model/Property')
+
+function mapper (place, properties) {
+  for (let key in properties) {
+    place.addProperty(new Property(`statcan:${key}`, properties[key]))
+  }
+}
+
+module.exports = mapper

import/source/statcan/map/properties.test.js

@@ -0,0 +1,40 @@
+const Place = require('../../../../model/Place')
+const map = require('./properties')
+
+module.exports.tests = {}
+
+module.exports.tests.mapper = (test) => {
+  test('mapper: properties empty', (t) => {
+    let p = new Place()
+    map(p, {})
+
+    t.equals(p.property.length, 0)
+    t.end()
+  })
+  test('mapper: uscensus-specific properties', (t) => {
+    let p = new Place()
+    map(p, {
+      'CLASSFP10': 'B5',
+      'MTFCC10': 'G6350',
+      'INTPTLON10': 1.1,
+      'INTPTLAT10': 2.2
+    })
+
+    t.equals(p.property.length, 2)
+    t.equals(p.property[0].key, 'uscensus:CLASSFP10', 'CLASSFP10')
+    t.equals(p.property[0].value, 'B5', 'CLASSFP10')
+    t.equals(p.property[1].key, 'uscensus:MTFCC10', 'MTFCC10')
+    t.equals(p.property[1].value, 'G6350', 'MTFCC10')
+    t.end()
+  })
+}
+
+module.exports.all = (tape) => {
+  function test (name, testFunction) {
+    return tape(`properties: ${name}`, testFunction)
+  }
+
+  for (var testCase in module.exports.tests) {
+    module.exports.tests[testCase](test)
+  }
+}