Skip to content

Commit

Permalink
fix: update country names (#76)
Browse files Browse the repository at this point in the history
Co-authored-by: Marcin Rataj <lidel@lidel.org>
  • Loading branch information
jessicaschilling and lidel authored Sep 1, 2020
1 parent d8cd0f8 commit 670b55b
Show file tree
Hide file tree
Showing 11 changed files with 10,520 additions and 10,582 deletions.
2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ stages:

node_js:
- '12'
- '10'
- 'lts/*'

os:
- linux
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ Pretty result: Mountain View, CA, United States, Earth

## Root hash

The current root hash for lookups is `QmRn43NNNBEibc6m7zVNcS6UusB1u3qTTfyoLmkugbeeGJ`.
The current root hash for lookups is defined under `GEOIP_ROOT` in `src/lookup.js`

## Contribute

Expand Down
5 changes: 4 additions & 1 deletion bin/generate
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,10 @@ ipfs.id()

if (event.type === 'put') {
counter++
gauge.show('Uploading', (counter / (length / 32)))
const objects = length / 32
const completed = counter / objects
gauge.pulse(`${counter}/${objects.toFixed()} (${(completed * 100).toFixed()}%)`)
gauge.show('importing objects to IPFS', completed)
}

if (event.status === 'start' && event.type !== 'put') {
Expand Down
20,885 changes: 10,373 additions & 10,512 deletions package-lock.json

Large diffs are not rendered by default.

16 changes: 9 additions & 7 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -32,21 +32,23 @@
},
"dependencies": {
"inet_ipv4": "^1.0.1",
"it-concat": "^1.0.0",
"memoizee": "~0.4.14",
"multihashes": "~0.4.15"
"cids": "^1.0.0"
},
"devDependencies": {
"aegir": "^22.0.0",
"bl": "^4.0.0",
"bluebird": "^3.7.1",
"chai": "^4.2.0",
"chai-as-promised": "^7.1.1",
"csv": "^5.1.3",
"gauge": "^2.7.4",
"iconv-lite": "~0.5.0",
"ipfs": "~0.39.0",
"ipfsd-ctl": "~0.47.4",
"csv": "^5.3.2",
"gauge": "^3.0.0",
"iconv-lite": "~0.6.2",
"ipfs": "~0.49.1",
"ipfs-http-client": "^46.0.1",
"ipfsd-ctl": "^7.0.0",
"lodash": "^4.17.15",
"multihashes": "^3.0.1",
"pre-commit": "^1.2.2"
},
"directories": {
Expand Down
48 changes: 23 additions & 25 deletions src/generate/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ const csv = Promise.promisifyAll(require('csv'))
const iconv = require('iconv-lite')
const _ = require('lodash')
const EventEmitter = require('events').EventEmitter
const bl = require('bl')
const concat = require('it-concat')

const normalizeName = require('./overrides')

// Btree size
const CHILDREN = 32
Expand All @@ -30,13 +32,14 @@ function emit (type, status, attrs) {

function parseCountries (countries) {
emit('countries', 'start')
return csv.parseAsync(countries.toString(), {
return csv.parseAsync(countries, {
columns: true,
cast: false,
skip_empty_lines: true
})
.then((parsed) => {
return _.reduce(parsed, (acc, row) => {
acc[row.alpha2] = row.name
acc[row.alpha2] = normalizeName(row.name)
return acc
}, {})
})
Expand All @@ -48,9 +51,9 @@ function parseCountries (countries) {

function parseLocations (locations, countries) {
emit('locations', 'start')
return csv.parseAsync(iconv.decode(locations, 'latin1'), {
return csv.parseAsync(locations, {
columns: true,
auto_parse: true,
cast: false,
skip_empty_lines: true,
comment: '#'
})
Expand All @@ -60,7 +63,7 @@ function parseLocations (locations, countries) {
countries[row.country],
row.country,
row.region,
row.city,
normalizeName(row.city),
row.postalCode,
Number(row.latitude),
Number(row.longitude),
Expand All @@ -78,9 +81,9 @@ function parseLocations (locations, countries) {

function parseBlocks (blocks, locations) {
emit('blocks', 'start')
return csv.parseAsync(blocks.toString(), {
return csv.parseAsync(blocks, {
columns: true,
auto_parse: true,
cast: false,
skip_empty_lines: true,
comment: '#'
})
Expand Down Expand Up @@ -117,18 +120,18 @@ function parseBlocks (blocks, locations) {
}

function putObject (data, min, api) {
return api.object.put(data, 'json')
.then((put) => {
return api.object.stat(put.Hash)
return api.object.put(data, { enc: 'json' })
.then((cid) => {
return api.object.stat(cid)
.then((stat) => {
if (!stat) {
throw new Error(`Could not stat object ${put.Hash}`)
throw new Error(`Could not stat object ${cid.toString()}`)
}
emit('put', 'end')
return {
min: min,
size: stat.CumulativeSize,
hash: put.Hash
hash: cid.toString()
}
})
})
Expand Down Expand Up @@ -173,21 +176,16 @@ function toNode (things, api) {

// divide
return Promise.map(_.chunk(things, CHILDREN), (res) => toNode(res, api), {
concurrency: 5
concurrency: require('os').cpus().length * 2
})
.then((res) => toNode(res, api))
}

function file (ipfs, dir) {
return ipfs.cat(`${DATA_HASH}/${dir}`)
.then((buffer) => {
return new Promise((resolve, reject) => {
buffer.pipe(bl((err, data) => {
if (err) return reject(err)
resolve(data)
}))
})
})
async function file (ipfs, dir) {
const buffer = await concat(ipfs.cat(`${DATA_HASH}/${dir}`), { type: 'buffer' })
// source files are in latin1, which requires handling with care
iconv.skipDecodeWarning = true
return iconv.decode(buffer, 'latin1')
}

function main (ipfs) {
Expand Down Expand Up @@ -217,7 +215,7 @@ function main (ipfs) {
})
.then((result) => {
emit('pinning', 'end')
return result.Pinned[0]
return result[0].cid.toString()
})
}

Expand Down
49 changes: 49 additions & 0 deletions src/generate/overrides.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
'use strict'

// Source data is not perfect, below are manual overrides applied to geo names
const overrides = new Map([
// ['original', 'override']
['Antigua And Barbuda', 'Antigua and Barbuda'],
['Bosnia & Herzegovina', 'Bosnia and Herzegovina'],
['Bolivia, Plurinational State Of', 'Bolivia'],
['Bonaire, Saint Eustatius And Saba', 'Bonaire, Saint Eustatius and Saba'],
['Democratic Republic Of Congo', 'Democratic Republic of Congo'],
['Republic Of Congo', 'Republic of Congo'],
['Czech Republic', 'Czechia'],
['Ceuta, Mulilla', 'Ceuta and Mulilla'],
['Micronesia, Federated States Of', 'Micronesia'],
['France, Metropolitan', 'Metropolitan France'],
['South Georgia And The South Sandwich Islands', 'South Georgia and the South Sandwich Islands'],
['Guinea-bissau', 'Guinea-Bissau'],
['Heard Island And McDonald Islands', 'Heard Island and McDonald Islands'],
['Isle Of Man', 'Isle of Man'],
['Iran, Islamic Republic Of', 'Iran'],
['Saint Kitts And Nevis', 'Saint Kitts and Nevis'],
["Korea, Democratic People's Republic Of", 'DPR Korea'],
['Korea, Republic Of', 'Republic of Korea'],
["Lao People's Democratic Republic", 'Laos'],
['Macedonia, The Former Yugoslav Republic Of', 'North Macedonia'],
['Saint Pierre And Miquelon', 'Saint Pierre and Miquelon'],
['Palestinian Territory, Occupied', 'Palestine'],
['Saint Helena, Ascension And Tristan Da Cunha', 'Saint Helena, Ascension and Tristan da Cunha'],
['Svalbard And Jan Mayen', 'Svalbard and Jan Mayen'],
['Syrian Arab Republic', 'Syria'],
['Turks And Caicos Islands', 'Turks and Caicos Islands'],
['Tristan de Cunha', 'Tristan da Cunha'],
['Trinidad And Tobago', 'Trinidad and Tobago'],
['Taiwan, Province Of China', 'Taiwan'],
['Tanzania, United Republic Of', 'Tanzania'],
['United States', 'USA'],
['Vatican City State', 'Vatican City'],
['Saint Vincent And The Grenadines', 'Saint Vincent and the Grenadines'],
['Venezuela, Bolivarian Republic Of', 'Venezuela'],
['Virgin Islands (British)', 'British Virgin Islands'],
['Virgin Islands (US)', 'US Virgin Islands'],
['Viet Nam', 'Vietnam'],
['Wallis And Futuna', 'Wallis and Futuna']
])

module.exports = function normalizeName (name) {
if (overrides.has(name)) return overrides.get(name)
return name
}
16 changes: 10 additions & 6 deletions src/lookup.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,25 @@

const memoize = require('memoizee')
const inet = require('inet_ipv4')
const mh = require('multihashes')
const CID = require('cids')

const formatData = require('./format')

const GEOIP_ROOT = mh.fromB58String('QmRn43NNNBEibc6m7zVNcS6UusB1u3qTTfyoLmkugbeeGJ')
const GEOIP_ROOT = new CID('Qmdzx1Xm3JTbtVaF4AYwfHHs372X6q2UJXS3XHoQu2Z3qM')

/**
* @param {Object} ipfs
* @param {string} hash
* @param {CID} cid
* @param {string} lookfor - ip
* @returns {Promise}
*/
async function _lookup (ipfs, hash, lookfor) {
const res = await ipfs.object.get(hash)
async function _lookup (ipfs, cid, lookfor) {
// ensure input is a valid cid, but switch to string representation
// to avoid serialization issues when mix of cids >1.0 and <1.0 are used
cid = new CID(cid).toString()
const res = await ipfs.object.get(cid)
const obj = JSON.parse(res.Data)

let child = 0

if (obj.type === 'Node') {
Expand Down Expand Up @@ -69,6 +73,6 @@ module.exports = function lookup (ipfs, ip) {

function getCid (node) {
if (!node) return null
if (node.Hash) return node.Hash.toString()
if (node.Hash) return node.Hash
return null
}
20 changes: 10 additions & 10 deletions test/format.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,34 +9,34 @@ describe('format', () => {
it('formats with all details present', () => {
expect(
format([
'United States',
'USA',
'US',
'CA',
'Mountain View',
94040,
'94040',
37.386,
-122.0838,
807,
650
'807',
'650'
])
).to.be.eql({
country_name: 'United States',
country_name: 'USA',
country_code: 'US',
region_code: 'CA',
city: 'Mountain View',
postal_code: 94040,
postal_code: '94040',
latitude: 37.386,
longitude: -122.0838,
metro_code: 807,
area_code: 650,
metro_code: '807',
area_code: '650',
planet: 'Earth'
})
})

it('formats with missing details', () => {
expect(
format([
'United States',
'USA',
'US',
'CA',
'',
Expand All @@ -47,7 +47,7 @@ describe('format', () => {
''
])
).to.be.eql({
country_name: 'United States',
country_name: 'USA',
country_code: 'US',
region_code: 'CA',
city: '',
Expand Down
Loading

0 comments on commit 670b55b

Please sign in to comment.