diff --git a/bin/gtfs2lc.js b/bin/gtfs2lc.js index 7092381..bb75490 100755 --- a/bin/gtfs2lc.js +++ b/bin/gtfs2lc.js @@ -11,6 +11,7 @@ program .option('-f, --format ', 'Format of the output. Possibilities: csv, n-triples, turtle, json, jsonld, mongo (extended JSON format to be used with mongoimport) or mongold (default: json)') .option('-b, --baseUris ', 'Path to a file that describes the baseUris in json') .option('-o, --output ', 'Path to the folder where the result file will be stored') + .option('-c, --compressed', 'Compress resulting connections file using gzip') .option('-s, --stream', 'Get the connections as a stream on the standard output') .option('-S, --store ', 'Store type: LevelStore (uses your disk to avoid that you run out of RAM) or MemStore (default)') .option('--fresh', 'Make sure to convert all Connection and ignore existing Historic records (which will be deleted)') @@ -42,6 +43,7 @@ if (program.baseUris) { var mapper = new gtfs2lc.Connections({ store: !program.store || program.store === 'undefined' ? 'MemStore' : program.store, format: !program.format || program.format === 'undefined' ? 'json' : program.format, + compressed: program.compressed, fresh: program.fresh, baseUris: baseUris }); diff --git a/lib/gtfs2connections.js b/lib/gtfs2connections.js index 8868a99..d225872 100644 --- a/lib/gtfs2connections.js +++ b/lib/gtfs2connections.js @@ -53,7 +53,7 @@ Mapper.prototype.resultStream = async function (path, output, done) { console.error('Creating Connection rules...'); await StopTimes2Cxs(path, output, stores, this._options.fresh); - // Step 4: Create connections in parallel using worker threads + // Step 4: Materialize connections in parallel using worker threads let w = 0; const raws = []; // Create as many worker threads as there are available CPUs @@ -78,6 +78,8 @@ Mapper.prototype.resultStream = async function (path, output, done) { // Step 5: Merge all the created files into one const format = this._options.format; let ext = null; + let mergeCommand = 'zcat'; + if (!format || ['json', 'mongo', 'jsonld', 'mongold'].indexOf(format) >= 0) { await appendLineBreaks(output); ext = 'json'; @@ -92,9 +94,13 @@ Mapper.prototype.resultStream = async function (path, output, done) { try { console.error('Merging final Linked Connections file...'); - + if(this._options.compressed) { + mergeCommand = 'cat'; + ext += '.gz'; + } + // Join all resulting files into one - await exec(`for i in ${raws.map(r => { return `${r}.${ext}.gz` }).join(" ")} ; do zcat "$i" >> linkedConnections.${ext} && rm "$i" || break ; done`, { cwd: output }); + await exec(`for i in ${raws.map(r => { return `${r}.${ext}.gz` }).join(" ")} ; do ${mergeCommand} "$i" >> linkedConnections.${ext} && rm "$i" || break ; done`, { cwd: output }); let t1 = new Date(); console.error('linkedConnections.' + ext + ' File created in ' + (t1.getTime() - t0.getTime()) + ' ms'); await del( diff --git a/package-lock.json b/package-lock.json index 354be46..668c760 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "gtfs2lc", - "version": "2.1.7", + "version": "2.1.8", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "gtfs2lc", - "version": "2.1.7", + "version": "2.1.8", "license": "MIT", "dependencies": { "commander": "^4.1.1", diff --git a/package.json b/package.json index 7cfa9a0..b1bfd56 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gtfs2lc", - "version": "2.1.7", + "version": "2.1.8", "description": "Mapping script from gtfs to (linked) connections", "main": "lib/gtfs2lc.js", "bin": {