fix: update country names (#76)

Co-authored-by: Marcin Rataj <lidel@lidel.org>

.travis.yml

@@ -7,7 +7,7 @@ stages:
 
 node_js:
   - '12'
-  - '10'
+  - 'lts/*'
 
 os:
   - linux

README.md

@@ -115,7 +115,7 @@ Pretty result: Mountain View, CA, United States, Earth
 
 ## Root hash
 
-The current root hash for lookups is `QmRn43NNNBEibc6m7zVNcS6UusB1u3qTTfyoLmkugbeeGJ`.
+The current root hash for lookups is defined under `GEOIP_ROOT` in `src/lookup.js`
 
 ## Contribute
 

bin/generate

@@ -28,7 +28,10 @@ ipfs.id()
 
       if (event.type === 'put') {
         counter++
-        gauge.show('Uploading', (counter / (length / 32)))
+        const objects = length / 32
+        const completed = counter / objects
+        gauge.pulse(`${counter}/${objects.toFixed()} (${(completed * 100).toFixed()}%)`)
+        gauge.show('importing objects to IPFS', completed)
       }
 
       if (event.status === 'start' && event.type !== 'put') {

package.json

@@ -32,21 +32,23 @@
   },
   "dependencies": {
     "inet_ipv4": "^1.0.1",
+    "it-concat": "^1.0.0",
     "memoizee": "~0.4.14",
-    "multihashes": "~0.4.15"
+    "cids": "^1.0.0"
   },
   "devDependencies": {
     "aegir": "^22.0.0",
-    "bl": "^4.0.0",
     "bluebird": "^3.7.1",
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
-    "csv": "^5.1.3",
-    "gauge": "^2.7.4",
-    "iconv-lite": "~0.5.0",
-    "ipfs": "~0.39.0",
-    "ipfsd-ctl": "~0.47.4",
+    "csv": "^5.3.2",
+    "gauge": "^3.0.0",
+    "iconv-lite": "~0.6.2",
+    "ipfs": "~0.49.1",
+    "ipfs-http-client": "^46.0.1",
+    "ipfsd-ctl": "^7.0.0",
     "lodash": "^4.17.15",
+    "multihashes": "^3.0.1",
     "pre-commit": "^1.2.2"
   },
   "directories": {

src/generate/index.js

@@ -5,7 +5,9 @@ const csv = Promise.promisifyAll(require('csv'))
 const iconv = require('iconv-lite')
 const _ = require('lodash')
 const EventEmitter = require('events').EventEmitter
-const bl = require('bl')
+const concat = require('it-concat')
+
+const normalizeName = require('./overrides')
 
 // Btree size
 const CHILDREN = 32
@@ -30,13 +32,14 @@ function emit (type, status, attrs) {
 
 function parseCountries (countries) {
   emit('countries', 'start')
-  return csv.parseAsync(countries.toString(), {
+  return csv.parseAsync(countries, {
     columns: true,
+    cast: false,
     skip_empty_lines: true
   })
     .then((parsed) => {
       return _.reduce(parsed, (acc, row) => {
-        acc[row.alpha2] = row.name
+        acc[row.alpha2] = normalizeName(row.name)
         return acc
       }, {})
     })
@@ -48,9 +51,9 @@ function parseCountries (countries) {
 
 function parseLocations (locations, countries) {
   emit('locations', 'start')
-  return csv.parseAsync(iconv.decode(locations, 'latin1'), {
+  return csv.parseAsync(locations, {
     columns: true,
-    auto_parse: true,
+    cast: false,
     skip_empty_lines: true,
     comment: '#'
   })
@@ -60,7 +63,7 @@ function parseLocations (locations, countries) {
           countries[row.country],
           row.country,
           row.region,
-          row.city,
+          normalizeName(row.city),
           row.postalCode,
           Number(row.latitude),
           Number(row.longitude),
@@ -78,9 +81,9 @@ function parseLocations (locations, countries) {
 
 function parseBlocks (blocks, locations) {
   emit('blocks', 'start')
-  return csv.parseAsync(blocks.toString(), {
+  return csv.parseAsync(blocks, {
     columns: true,
-    auto_parse: true,
+    cast: false,
     skip_empty_lines: true,
     comment: '#'
   })
@@ -117,18 +120,18 @@ function parseBlocks (blocks, locations) {
 }
 
 function putObject (data, min, api) {
-  return api.object.put(data, 'json')
-    .then((put) => {
-      return api.object.stat(put.Hash)
+  return api.object.put(data, { enc: 'json' })
+    .then((cid) => {
+      return api.object.stat(cid)
         .then((stat) => {
           if (!stat) {
-            throw new Error(`Could not stat object ${put.Hash}`)
+            throw new Error(`Could not stat object ${cid.toString()}`)
           }
           emit('put', 'end')
           return {
             min: min,
             size: stat.CumulativeSize,
-            hash: put.Hash
+            hash: cid.toString()
           }
         })
     })
@@ -173,21 +176,16 @@ function toNode (things, api) {
 
   // divide
   return Promise.map(_.chunk(things, CHILDREN), (res) => toNode(res, api), {
-    concurrency: 5
+    concurrency: require('os').cpus().length * 2
   })
     .then((res) => toNode(res, api))
 }
 
-function file (ipfs, dir) {
-  return ipfs.cat(`${DATA_HASH}/${dir}`)
-    .then((buffer) => {
-      return new Promise((resolve, reject) => {
-        buffer.pipe(bl((err, data) => {
-          if (err) return reject(err)
-          resolve(data)
-        }))
-      })
-    })
+async function file (ipfs, dir) {
+  const buffer = await concat(ipfs.cat(`${DATA_HASH}/${dir}`), { type: 'buffer' })
+  // source files are in latin1, which requires handling with care
+  iconv.skipDecodeWarning = true
+  return iconv.decode(buffer, 'latin1')
 }
 
 function main (ipfs) {
@@ -217,7 +215,7 @@ function main (ipfs) {
     })
     .then((result) => {
       emit('pinning', 'end')
-      return result.Pinned[0]
+      return result[0].cid.toString()
     })
 }
 

src/generate/overrides.js

@@ -0,0 +1,49 @@
+'use strict'
+
+// Source data is not perfect, below are manual overrides applied to geo names
+const overrides = new Map([
+  // ['original', 'override']
+  ['Antigua And Barbuda', 'Antigua and Barbuda'],
+  ['Bosnia & Herzegovina', 'Bosnia and Herzegovina'],
+  ['Bolivia, Plurinational State Of', 'Bolivia'],
+  ['Bonaire, Saint Eustatius And Saba', 'Bonaire, Saint Eustatius and Saba'],
+  ['Democratic Republic Of Congo', 'Democratic Republic of Congo'],
+  ['Republic Of Congo', 'Republic of Congo'],
+  ['Czech Republic', 'Czechia'],
+  ['Ceuta, Mulilla', 'Ceuta and Mulilla'],
+  ['Micronesia, Federated States Of', 'Micronesia'],
+  ['France, Metropolitan', 'Metropolitan France'],
+  ['South Georgia And The South Sandwich Islands', 'South Georgia and the South Sandwich Islands'],
+  ['Guinea-bissau', 'Guinea-Bissau'],
+  ['Heard Island And McDonald Islands', 'Heard Island and McDonald Islands'],
+  ['Isle Of Man', 'Isle of Man'],
+  ['Iran, Islamic Republic Of', 'Iran'],
+  ['Saint Kitts And Nevis', 'Saint Kitts and Nevis'],
+  ["Korea, Democratic People's Republic Of", 'DPR Korea'],
+  ['Korea, Republic Of', 'Republic of Korea'],
+  ["Lao People's Democratic Republic", 'Laos'],
+  ['Macedonia, The Former Yugoslav Republic Of', 'North Macedonia'],
+  ['Saint Pierre And Miquelon', 'Saint Pierre and Miquelon'],
+  ['Palestinian Territory, Occupied', 'Palestine'],
+  ['Saint Helena, Ascension And Tristan Da Cunha', 'Saint Helena, Ascension and Tristan da Cunha'],
+  ['Svalbard And Jan Mayen', 'Svalbard and Jan Mayen'],
+  ['Syrian Arab Republic', 'Syria'],
+  ['Turks And Caicos Islands', 'Turks and Caicos Islands'],
+  ['Tristan de Cunha', 'Tristan da Cunha'],
+  ['Trinidad And Tobago', 'Trinidad and Tobago'],
+  ['Taiwan, Province Of China', 'Taiwan'],
+  ['Tanzania, United Republic Of', 'Tanzania'],
+  ['United States', 'USA'],
+  ['Vatican City State', 'Vatican City'],
+  ['Saint Vincent And The Grenadines', 'Saint Vincent and the Grenadines'],
+  ['Venezuela, Bolivarian Republic Of', 'Venezuela'],
+  ['Virgin Islands (British)', 'British Virgin Islands'],
+  ['Virgin Islands (US)', 'US Virgin Islands'],
+  ['Viet Nam', 'Vietnam'],
+  ['Wallis And Futuna', 'Wallis and Futuna']
+])
+
+module.exports = function normalizeName (name) {
+  if (overrides.has(name)) return overrides.get(name)
+  return name
+}

src/lookup.js

@@ -2,21 +2,25 @@
 
 const memoize = require('memoizee')
 const inet = require('inet_ipv4')
-const mh = require('multihashes')
+const CID = require('cids')
 
 const formatData = require('./format')
 
-const GEOIP_ROOT = mh.fromB58String('QmRn43NNNBEibc6m7zVNcS6UusB1u3qTTfyoLmkugbeeGJ')
+const GEOIP_ROOT = new CID('Qmdzx1Xm3JTbtVaF4AYwfHHs372X6q2UJXS3XHoQu2Z3qM')
 
 /**
  * @param {Object} ipfs
- * @param {string} hash
+ * @param {CID} cid
  * @param {string} lookfor - ip
  * @returns {Promise}
  */
-async function _lookup (ipfs, hash, lookfor) {
-  const res = await ipfs.object.get(hash)
+async function _lookup (ipfs, cid, lookfor) {
+  // ensure input is a valid cid, but switch to string representation
+  // to avoid serialization issues when mix of cids >1.0 and <1.0 are used
+  cid = new CID(cid).toString()
+  const res = await ipfs.object.get(cid)
   const obj = JSON.parse(res.Data)
+
   let child = 0
 
   if (obj.type === 'Node') {
@@ -69,6 +73,6 @@ module.exports = function lookup (ipfs, ip) {
 
 function getCid (node) {
   if (!node) return null
-  if (node.Hash) return node.Hash.toString()
+  if (node.Hash) return node.Hash
   return null
 }

test/format.spec.js

@@ -9,34 +9,34 @@ describe('format', () => {
   it('formats with all details present', () => {
     expect(
       format([
-        'United States',
+        'USA',
         'US',
         'CA',
         'Mountain View',
-        94040,
+        '94040',
         37.386,
         -122.0838,
-        807,
-        650
+        '807',
+        '650'
       ])
     ).to.be.eql({
-      country_name: 'United States',
+      country_name: 'USA',
       country_code: 'US',
       region_code: 'CA',
       city: 'Mountain View',
-      postal_code: 94040,
+      postal_code: '94040',
       latitude: 37.386,
       longitude: -122.0838,
-      metro_code: 807,
-      area_code: 650,
+      metro_code: '807',
+      area_code: '650',
       planet: 'Earth'
     })
   })
 
   it('formats with missing details', () => {
     expect(
       format([
-        'United States',
+        'USA',
         'US',
         'CA',
         '',
@@ -47,7 +47,7 @@ describe('format', () => {
         ''
       ])
     ).to.be.eql({
-      country_name: 'United States',
+      country_name: 'USA',
       country_code: 'US',
       region_code: 'CA',
       city: '',