-
Notifications
You must be signed in to change notification settings - Fork 0
/
jobfile-units.js
104 lines (102 loc) · 2.79 KB
/
jobfile-units.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
import _ from 'lodash'
import moment from 'moment'
import path from 'path'
import { fileURLToPath } from 'url'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const dbUrl = process.env.DB_URL || 'mongodb://127.0.0.1:27017/rte'
// FIXME: now we only have information about nuclear plant/reactors
export default {
id: 'rte-units',
store: 'memory',
options: {
workersLimit: 1,
faultTolerant: true,
},
tasks: [{
id: 'reactors.csv',
type: 'store',
options: {
store: 'fs'
}
}],
hooks: {
tasks: {
after: {
readCSV: {
header: true
},
apply: {
function: (item) => {
const plants = _.get(item, 'plants', [])
delete item.plants
const units = _.get(item, 'data', [])
_.forEach(units, (unit) => {
// Find owing plant to get location and other useful information
const plant = _.find(plants, { id: unit.plantId })
if (plant) {
_.merge(unit, _.omit(plant, ['id', 'name']))
}
// Convert some properties
_.forOwn(unit, (value, key) => {
if (key.endsWith('_MW')) unit[key] = _.toNumber(unit[key])
if (key.endsWith('Date')) unit[key] = moment.utc(unit[key], 'DD/MM/YYYY').toDate()
})
})
console.log('Found ' + units.length + ' production units')
}
},
convertToGeoJson: {
latitude: 'lat',
longitude: 'long'
},
updateMongoCollection: {
collection: 'rte-units',
filter: { 'properties.eicCode': '<%= properties.eicCode %>' },
upsert: true,
chunkSize: 256
},
clearData: {}
}
},
jobs: {
before: {
createStores: [{
id: 'memory'
}, {
id: 'fs', options: { path: __dirname }
}],
connectMongo: {
url: dbUrl,
// Required so that client is forwarded from job to tasks
clientPath: 'taskTemplate.client'
},
createMongoCollection: {
clientPath: 'taskTemplate.client',
collection: 'rte-units',
indices: [
[{ 'properties.eicCode': 1 }, { unique: true }],
{ geometry: '2dsphere' }
]
},
readCSV: {
key: 'plants.csv',
store: 'fs',
header: true,
dataPath: 'data.taskTemplate.plants'
}
},
after: {
disconnectMongo: {
clientPath: 'taskTemplate.client'
},
removeStores: [ 'memory', 'fs' ]
},
error: {
disconnectMongo: {
clientPath: 'taskTemplate.client'
},
removeStores: [ 'memory', 'fs' ]
}
}
}
}