forked from eric-kargo/kemenperin-crawler
-
Notifications
You must be signed in to change notification settings - Fork 0
/
geocoder.js
76 lines (67 loc) · 1.74 KB
/
geocoder.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
const axios = require('axios');
const csv = require('csv-parser');
const fs = require('fs');
const csvWriter = require('csv-write-stream');
const { pRateLimit } = require('p-ratelimit');
const ACCESS_TOKEN = '';
const writer = csvWriter({
headers: [
'Company Name',
'Address',
'Lattitude',
'Longitude',
]
});
const rateLimit = pRateLimit({
interval: 1000, // 1000 ms == 1 second
rate: 10, // 30 API calls per interval
concurrency: 1, // no more than 10 running at once
});
writer.pipe(fs.createWriteStream('address.csv'));
function readData() {
const results = [];
return new Promise((resolve, reject) => {
fs.createReadStream('data.csv')
.pipe(csv())
.on('data', (data) => results.push(data))
.on('end', () => {
resolve(results);
})
.on('error', (error) => reject(error));
})
};
async function getLatLngFromAddress({address, companyName} = {}) {
try {
console.log('fetching', address);
const encodedUri = encodeURIComponent(address);
const data = await axios.get(`https://api.mapbox.com/geocoding/v5/mapbox.places/${encodedUri}.json`, {
params: {
access_token: ACCESS_TOKEN,
},
});
const center = data.data.features[0].center;
writer.write([
companyName,
address,
center[0],
center[1],
]);
return center;
} catch (error) {
console.error(error)
return [0, 0]
}
}
async function generate() {
const data = await readData();
const address = data.map(item => ({
address: item.Address,
companyName: item["Company Name"],
}));
const latLngFetch = address.map(item => {
return rateLimit(() => getLatLngFromAddress(item));
});
await Promise.all(latLngFetch);
return writer.end();
}
generate();