Merge pull request #145 from linkedconnections/development

v2.2.0

bin/gtfs2lc-clean.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 

lib/gtfs2connections.js

@@ -63,9 +63,6 @@ Mapper.prototype.resultStream = async function (path, output, done) {
             instance: i,
             output,
             options: this._options
-         },
-         resourceLimits: {
-            maxOldGenerationSizeMb: 12000
          }
       });
 
@@ -95,7 +92,7 @@ Mapper.prototype.resultStream = async function (path, output, done) {
 
             try {
                console.error('Merging final Linked Connections file...');
-               if(this._options.compressed) {
+               if (this._options.compressed) {
                   mergeCommand = 'cat';
                   gz = '.gz';
                }
@@ -167,8 +164,11 @@ async function removePrefixes(output) {
 if (!isMainThread) {
    let fmt = 'json';
    // Read the connection rules file created in the master thread and build the Connection objects!
-   let connectionStream = fs.createReadStream(workerData['output'] + '/connections_' + workerData['instance'] + '.txt', { encoding: 'utf8', objectMode: true })
-      .pipe(JSONLParser())
+   // Use a low highWaterMark to prevent backpressure memory leaks given that this is an inflating pipeline.
+   let connectionStream = fs.createReadStream(
+      `${workerData['output']}/connections_${workerData['instance']}.txt`,
+      { encoding: 'utf8', objectMode: true, highWaterMark: 4 * 1024 }
+   ).pipe(JSONLParser())
       .pipe(new ConnectionsBuilder())
       .on('error', function (e) {
          console.error(e);