Attempt to reduce memory usage.

jarib · Aug 1, 2015 · cc1f2e4 · cc1f2e4
1 parent 8b37918
commit cc1f2e4
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 2 deletions.
diff --git a/.babelrc b/.babelrc
@@ -1,3 +1,17 @@
 {
-    "stage": 0
+    "stage": 0,
+    "optional": [
+      "runtime",
+      "optimisation",
+      "minification.memberExpressionLiterals",
+      "minification.propertyLiterals",
+      "optimisation.flow.forOf",
+      "optimisation.modules.system",
+      ],
+    "loose": ["all"],
+    "compact": true,
+    "comments": false,
+    "ast": false,
+    "nonStandard": false,
+    "only": ["lib/Scraper.js"]
 }
diff --git a/lib/Scraper.js b/lib/Scraper.js
@@ -8,6 +8,16 @@ import slug from 'slug';
 import util from 'util';
 import { Iconv } from 'iconv';
 
+if (global.gc) {
+    for (var i = 0; i < 50; i++) {
+        console.log(process.memoryUsage().rss / (1024 * 1024));
+        gc();
+        console.log(process.memoryUsage().rss / (1024 * 1024));
+    };
+} else {
+    global.gc = (() => {});
+}
+
 const request = Promise.promisify(requestRaw);
 const USER_AGENT = 'pollofpolls-scraper | https://github.com/jarib/pollofpolls-scraper';
 const DATE = moment().format('YYYY-MM-DD');
@@ -121,7 +131,7 @@ export default class Scraper {
                         name,
                         date,
                         ...results
-                    });
+                    }).then(gc);
                 });
             });
     }

diff --git a/package.json b/package.json
@@ -14,6 +14,7 @@
   },
   "dependencies": {
     "babel": "^5.8.20",
+    "babel-runtime": "^5.8.20",
     "bluebird": "^2.9.34",
     "cheerio": "^0.19.0",
     "iconv": "^2.1.8",