forked from eric-kargo/kemenperin-crawler
-
Notifications
You must be signed in to change notification settings - Fork 0
/
transformer.js
88 lines (80 loc) · 2.15 KB
/
transformer.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
const fs = require('fs');
const csv = require('csv-parser');
const csvWriter = require('csv-write-stream');
const writer = csvWriter({
headers: [
'Company Name',
'Address',
]
});
function getAddressWithLatLng() {
const res = [];
return new Promise((resolve, reject) => {
fs.createReadStream('address.csv')
.pipe(csv())
.on('data', (data) => res.push(data))
.on('end', () => {
resolve(res.map(item => item.Address));
});
});
}
function getFullAddress() {
const res = [];
return new Promise((resolve, reject) => {
fs.createReadStream('data.csv')
.pipe(csv())
.on('data', (data) => res.push(data))
.on('end', () => {
resolve(res.map(item => ({address: item.Address, company: item["Company Name"]})));
});
})
}
async function getDistinct() {
const addressWithLatLng = await getAddressWithLatLng();
const addressWithoutLatLng = await getFullAddress();
writer.pipe(fs.createWriteStream('dirty.csv'));
addressWithoutLatLng.forEach(item => {
if (!addressWithLatLng.includes(item.address)) {
writer.write([
item.company,
item.address,
])
}
});
writer.end();
console.log(addressWithLatLng.length, addressWithoutLatLng.length);
}
function writeGeoJson() {
const results = [];
fs.createReadStream('address.csv')
.pipe(csv())
.on('data', (data) => results.push(data))
.on('end', () => {
const geoJsonObj = {
type: 'FeatureCollection',
crs: {
type: 'name',
properties: {
name: 'urn:ogc:def:crs:OGC:1.3:CRS84'
}
},
};
const features = results.map(item => (
{
geometry: {
coordinates: [Number(item.Lattitude), Number(item.Longitude)],
type: 'Point',
},
properties: {
company: item["Company Name"],
address: item.Address,
mag: Math.floor(Math.random() * 3) + 1,
},
type: 'Feature'
}
));
geoJsonObj.features = features;
fs.writeFileSync('points.geojson', JSON.stringify(geoJsonObj, null, 2));
})
}
writeGeoJson();