/
io.js
234 lines (209 loc) · 6.14 KB
/
io.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
const fs = require("fs-extra");
const path = require("path");
const log = require("./log");
const GenerateSchema = require("generate-schema");
const config = require("./config");
const git = require("./git");
const json = require("./json");
const jsonschema = require("./jsonschema");
capitalizeTittel = string =>
string.charAt(0).toUpperCase() + string.slice(1).toLowerCase();
function getLength(o) {
if (o.features) return o.features.length;
if (o.items) return o.items.length || Object.keys(o.items).length;
if (o.length) return o.length;
return Object.keys(o).length;
}
const sourceFileMetas = []
/**
* Read JSON from temp directory.
*/
function lesTempJson(filename, extension, defaultJson) {
const jsonPath = config.getTempPath(filename, extension);
const content = readJson(jsonPath, defaultJson)
sourceFileMetas.push({ name: filename, ...(content.meta || {}) })
return content.items ? content.items : content;
}
/**
* Read JSON from build directory.
*/
function lesBuildfil(filename, extension, defaultJson) {
const jsonPath = config.getBuildPath(filename, extension);
return readJson(jsonPath, defaultJson);
}
/**
* Write JSON to temp directory.
*/
function skrivDatafil(filename, o) {
const jsonPath = config.getTempPath(filename);
return writeJson(jsonPath, o);
}
function skrivBuildfil(filename, o, delimiter = " ") {
const jsonPath = config.getBuildPath(filename);
o = convertObjectToArray(o);
const dokument = writeJson(jsonPath, o, delimiter);
// Lag schema json for build outputs
const schema = GenerateSchema.json(dokument);
writeJson(jsonPath.replace(".json", "") + ".schema.json", schema, delimiter);
return dokument;
}
function convertObjectToArray(o) {
if (o.features) return o; // Don't mess with a GeoJSON
if (!o.items) o = { items: o };
if (!Array.isArray(o.items))
// Skriv alltid arrays (da fungerer schema osv)
// med mindre output er GeoJSON
o.items = json.objectToArray(o.items, "kode");
o.items = o.items.sort((a, b) => (a.kode > b.kode ? 1 : -1));
return o;
}
function skrivLoggLinje(aksjon, filePath, json) {
let produsertUtc = null;
if (json.meta && json.meta.produsertUtc)
produsertUtc = new Date(json.meta.produsertUtc);
else produsertUtc = new Date(fs.statSync(filePath).ctime);
const now = new Date();
const timerGammel =
Math.round((10 * (now - produsertUtc)) / 1000 / 60 / 60) / 10;
if (json.data) json = json.data;
log.info(
"Lest " +
getLength(json) +
" elementer fra " +
timerGammel +
" timer gammel fil."
);
}
function readRaw(filePath) {
let data = fs.readFileSync(filePath, "utf8");
// data = data.replace(/^\uFEFF/, '') // node #fail https://github.com/nodejs/node/issues/6924
if (data.charCodeAt(0) === 0xfeff) data = data.slice(1);
return data;
}
// Read file, parse to JSON. If file doesn't exists defaultJson will be returned
function readJson(filePath, defaultJson) {
log.info("Åpner " + filePath);
if (defaultJson && !fs.existsSync(filePath)) return defaultJson;
let data = readRaw(filePath);
let json = JSON.parse(data);
skrivLoggLinje("Lest", filePath, json);
return json;
}
/**
* Read JSONLines format from file.
* http://jsonlines.org/
*/
function lesJsonLines(filnavn) {
const filePath = config.getTempPath(filnavn, "");
log.info("Åpner " + filePath);
let data = readRaw(filePath)
.trim()
.split("\n");
let json = data.map(line => JSON.parse(line));
skrivLoggLinje("Lest", filePath, json);
return json;
}
/**
* Read raw data from file.
*/
function lesDataRå(filnavn) {
const filePath = config.getTempPath(filnavn, "");
return readRaw(filePath);
}
/**
* Write JSON to file.
*/
function writeJson(filePath, o, delimiter) {
const parsedFn = path.parse(filePath);
const basename = parsedFn.name;
let dokument = o.items || o.features ? o : { items: o }
const meta = Object.assign({}, dokument.meta, { source: sourceFileMetas })
const pjson = config.getPackage();
dokument.meta = {
tittel: capitalizeTittel(basename.replace(/_/g, " ")),
produsertUtc: new Date().toJSON(),
utgiver: "Artsdatabanken",
url: git.getUpstreamUrlForFile(filePath),
tool: {
name: pjson.name,
url: pjson.homepage,
commit: git.getGithubCommitUrl()
},
elementer: getLength(dokument)
};
if (meta) Object.assign(dokument.meta, meta);
dokument.items = json.sortKeys(dokument.items);
const schemaFilename = filePath.replace(parsedFn.ext, ".schema.json");
if (fs.existsSync(schemaFilename))
jsonschema.addSchemaLink(dokument, schemaFilename);
const bytes = writeBinary(
filePath,
stringify(dokument, parsedFn.ext, delimiter)
);
log.info("Skrevet " + getLength(o) + " elementer, " + bytes + " bytes");
return dokument;
}
function stringify(o, extension, delimiter) {
if (extension === ".geojson") return json.stringifyGeojsonCompact(o);
return JSON.stringify(o, null, delimiter);
}
/**
* Write binary data to file.
*/
function writeBinary(filePath, o) {
if (!filePath) throw new Error("Filename is required");
if (!o) throw new Error("No data provided");
log.info("Writing " + filePath);
const dir = path.dirname(filePath);
mkdir(dir);
fs.writeFileSync(filePath, o, "utf8");
return o.length;
}
/**
* Create a directory if it does not already exist.
*/
function mkdir(path) {
fs.ensureDirSync(path);
}
/**
* Check if a file exists.
*/
function fileExists(path) {
return fs.existsSync(path);
}
/**
* Check if a directory exists.
*/
function directoryExists(path) {
return fs.pathExistsSync(path);
}
/**
* Recursive find files in startPath satisfying filter
*/
function findFiles(startPath, filter) {
let r = [];
var files = fs.readdirSync(startPath);
for (var file of files) {
var filename = path.join(startPath, file);
var stat = fs.lstatSync(filename);
if (stat.isDirectory()) {
r = r.concat(findFiles(filename, filter));
} else if (filter && filter !== path.parse(filename).ext) {
} else r.push(filename);
}
return r;
}
module.exports = {
lesDataRå,
lesTempJson,
readJson,
lesJsonLines,
writeBinary,
writeJson,
skrivDatafil,
skrivBuildfil,
findFiles,
fileExists,
directoryExists,
mkdir
};