Skip to content

Commit

Permalink
Merge pull request #141 from linkedconnections/development
Browse files Browse the repository at this point in the history
v2.1.8
  • Loading branch information
julianrojas87 committed Nov 22, 2022
2 parents 04252be + 6005b34 commit 0680725
Show file tree
Hide file tree
Showing 4 changed files with 14 additions and 6 deletions.
2 changes: 2 additions & 0 deletions bin/gtfs2lc.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ program
.option('-f, --format <format>', 'Format of the output. Possibilities: csv, n-triples, turtle, json, jsonld, mongo (extended JSON format to be used with mongoimport) or mongold (default: json)')
.option('-b, --baseUris <baseUris>', 'Path to a file that describes the baseUris in json')
.option('-o, --output <output>', 'Path to the folder where the result file will be stored')
.option('-c, --compressed', 'Compress resulting connections file using gzip')
.option('-s, --stream', 'Get the connections as a stream on the standard output')
.option('-S, --store <store>', 'Store type: LevelStore (uses your disk to avoid that you run out of RAM) or MemStore (default)')
.option('--fresh', 'Make sure to convert all Connection and ignore existing Historic records (which will be deleted)')
Expand Down Expand Up @@ -42,6 +43,7 @@ if (program.baseUris) {
var mapper = new gtfs2lc.Connections({
store: !program.store || program.store === 'undefined' ? 'MemStore' : program.store,
format: !program.format || program.format === 'undefined' ? 'json' : program.format,
compressed: program.compressed,
fresh: program.fresh,
baseUris: baseUris
});
Expand Down
12 changes: 9 additions & 3 deletions lib/gtfs2connections.js
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ Mapper.prototype.resultStream = async function (path, output, done) {
console.error('Creating Connection rules...');
await StopTimes2Cxs(path, output, stores, this._options.fresh);

// Step 4: Create connections in parallel using worker threads
// Step 4: Materialize connections in parallel using worker threads
let w = 0;
const raws = [];
// Create as many worker threads as there are available CPUs
Expand All @@ -78,6 +78,8 @@ Mapper.prototype.resultStream = async function (path, output, done) {
// Step 5: Merge all the created files into one
const format = this._options.format;
let ext = null;
let mergeCommand = 'zcat';

if (!format || ['json', 'mongo', 'jsonld', 'mongold'].indexOf(format) >= 0) {
await appendLineBreaks(output);
ext = 'json';
Expand All @@ -92,9 +94,13 @@ Mapper.prototype.resultStream = async function (path, output, done) {

try {
console.error('Merging final Linked Connections file...');

if(this._options.compressed) {
mergeCommand = 'cat';
ext += '.gz';
}

// Join all resulting files into one
await exec(`for i in ${raws.map(r => { return `${r}.${ext}.gz` }).join(" ")} ; do zcat "$i" >> linkedConnections.${ext} && rm "$i" || break ; done`, { cwd: output });
await exec(`for i in ${raws.map(r => { return `${r}.${ext}.gz` }).join(" ")} ; do ${mergeCommand} "$i" >> linkedConnections.${ext} && rm "$i" || break ; done`, { cwd: output });
let t1 = new Date();
console.error('linkedConnections.' + ext + ' File created in ' + (t1.getTime() - t0.getTime()) + ' ms');
await del(
Expand Down
4 changes: 2 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "gtfs2lc",
"version": "2.1.7",
"version": "2.1.8",
"description": "Mapping script from gtfs to (linked) connections",
"main": "lib/gtfs2lc.js",
"bin": {
Expand Down

0 comments on commit 0680725

Please sign in to comment.