0.9.53

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Luminati Proxy manager - Change Log
 
+## 0.9.53:
+- :bug: Allow windows and other OS to handle the large CSV file properly
+
 ## 0.9.52:
 - :bug: Fix bug in loading of countries to UI
 

lib/cities.js

@@ -3,7 +3,9 @@
 const path = require('path');
 const _ = require('lodash');
 const hutil = require('hutil');
-const file = hutil.file;
+const fs = require('fs');
+const readline = require('readline');
+const etask = hutil.etask;
 const csv = hutil.csv;
 const qw = hutil.string.qw;
 const E = module.exports;
@@ -48,32 +50,46 @@ const prepare_data = csv_data=>{
     return {cities, regions, countries};
 };
 
-var data;
+let data;
+let pending;
 
-E.ensure_data = ()=>{
+E.ensure_data = ()=>etask(function*(){
     if (data)
         return;
-    data = prepare_data(csv.to_arr(file.read_e(cities_csv)));
-};
+    if (pending)
+    {
+        pending.push(this);
+        return yield this.wait();
+    }
+    pending = [];
+    const csv_data = [];
+    readline.createInterface({input: fs.createReadStream(cities_csv)})
+    .on('line', l=>csv_data.push(csv.to_arr(l)[0]))
+    .on('close', this.continue_fn());
+    yield this.wait();
+    data = prepare_data(csv_data);
+    pending.forEach(p=>p.continue());
+    pending = null;
+});
 
-E.countries = ()=>{
-    E.ensure_data();
+E.countries = ()=>etask(function*(){
+    yield E.ensure_data();
     return data.countries||[];
-};
+});
 
-E.regions = country_id=>{
+E.regions = country_id=>etask(function*(){
     if (!country_id)
         return [];
-    E.ensure_data();
+    yield E.ensure_data();
     return data.regions[country_id.toLowerCase()]||[];
-};
+});
 
-E.cities = (country_id, region_id)=>{
+E.cities = (country_id, region_id)=>etask(function*(){
     if (!country_id)
         return [];
-    E.ensure_data();
+    yield E.ensure_data();
     country_id = country_id.toLowerCase();
     region_id = region_id && region_id.toLowerCase();
     return data.cities.filter(c=>c.country_id == country_id &&
         (!region_id || c.region_id == region_id));
-};
+});

lib/manager.js

@@ -523,7 +523,7 @@ E.prototype.get_consts = etask._fn(function*get_consts(_this, req, res){
     });
     log_levels.unshift({key: `Default (${_this.opts.log})`, value: ''});
     proxy.log.values = log_levels;
-    proxy.country = {values: _this.get_countries()};
+    proxy.country = {values: yield _this.get_countries()};
     proxy.debug.values = [
         {key: `Default (${proxy.debug.def})`, value: ''},
         {key: 'none', value: 'none'},
@@ -1150,44 +1150,46 @@ E.prototype.history_get = etask._fn(function*history_get(_this, req, res){
     res.json(result);
 });
 
-E.prototype.get_countries = function(){
-    let countries = cities.countries().map(c=>({
+E.prototype.get_countries = etask._fn(function*get_countries(_this){
+    let countries = yield cities.countries();
+    countries = countries.map(c=>({
         key: c.country_name,
         value: c.country_id.toLowerCase(),
     }));
     countries.forEach(c=>c.key=`${c.key} (${c.value.toUpperCase()})`);
     countries.unshift({key: 'Any', value: '*'});
-    countries.unshift({key: 'Default ('+(this.opts.country||'Any')+')',
+    countries.unshift({key: 'Default ('+(_this.opts.country||'Any')+')',
         value: ''});
     return countries;
-};
+});
 
-E.prototype.get_regions = function(req, res){
-    let regions = cities.regions(req.params.country_id).map(r=>({
+E.prototype.get_regions = etask._fn(function*(_this, req, res){
+    let regions = yield cities.regions(req.params.country_id);
+    regions = regions.map(r=>({
         key: `${r.region_name} (${r.region_id.toUpperCase()})`,
         value: r.region_id,
     }));
     regions.unshift({key: 'Any', value: '*'});
-    regions.unshift({key: 'Default ('+(this.opts.state||'Any')+')',
+    regions.unshift({key: 'Default ('+(_this.opts.state||'Any')+')',
         value: ''});
     res.json(regions);
-};
+});
 
-E.prototype.get_cities = function(req, res){
+E.prototype.get_cities = etask._fn(function*(_this, req, res){
     const c_id = req.params.country_id;
     const r_id = req.params.region_id;
     if (!c_id)
         return res.json([]);
-    let _cities = cities.cities(c_id, r_id)
-        .map(c=>({
+    let _cities = yield cities.cities(c_id, r_id);
+    _cities = _cities.map(c=>({
             key: c.city_name,
             value: c.city_name,
             region: c.region_id,
         }));
     _cities.unshift({key: 'Any', value: '*'});
-    _cities.unshift({key: 'Default ('+(this.opts.city||'Any')+')', value: ''});
+    _cities.unshift({key: 'Default ('+(_this.opts.city||'Any')+')', value: ''});
     res.json(_cities);
-};
+});
 
 E.prototype.get_free_port = function(req, res){
     const server = http.createServer();
@@ -1603,11 +1605,20 @@ function*create_web_interface(_this){
         res.locals.path = req.path;
         next();
     });
-    app.get(/^\/((|proxies|tools|faq)|zones(\/[^\/]+)?)$/, (req, res, next)=>{
-        cities.ensure_data();
-        req.url = '/index.html';
-        next('route');
-    });
+    app.get(/^\/((|proxies|tools|faq)|zones(\/[^\/]+)?)$/, _this._api(
+        (req, res, next)=>{
+            setInterval(()=>etask(function*(){
+                try {
+                    yield cities.ensure_data();
+                } catch(e){ ef(e);
+                    _this._log.error('Failed to load cities data', e);
+                    _this._log.silly(e.stack);
+                }
+            }), 0);
+            req.url = '/index.html';
+            next('route');
+        }
+    ));
     app.use('/uib', express.static(path.resolve(__dirname,
         '../node_modules/angular-ui-bootstrap')));
     app.use(express.static(path.resolve(__dirname, '../bin/pub')));

lib/swagger.json

@@ -1,7 +1,7 @@
 {
     "swagger": "2.0",
     "info": {
-        "version": "0.9.52",
+        "version": "0.9.53",
         "title": "Luminati Proxy Manager",
         "license": {
             "name": "MIT",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "luminati-proxy",
-  "version": "0.9.52",
+  "version": "0.9.53",
   "description": "A configurable local proxy for luminati.io",
   "homepage": "https://luminati.io/",
   "main": "index.js",