1.108.399

luminati-io · Sep 27, 2018 · 9119c55 · 9119c55
1 parent 11b8dcf
commit 9119c55
Show file tree

Hide file tree

Showing 15 changed files with 214 additions and 63 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,9 @@
 Luminati Proxy manager - Change Log
 
+## 1.108.399 Development Latest
+- :star: Migration to node 10.X
+- :star: Performance improvements
+
 ## 1.108.119 Stable
 - :star: Updated dependencies
 - :star: New rule: switch proxy port based on URL

diff --git a/README-zh-CN.md b/README-zh-CN.md
@@ -29,10 +29,10 @@
 - <a href="https://nodejs.org/en/download/">Node.js</a> 6+版
 
 ### Windows
-下载 <a href="https://luminati-china.io/static/lpm/luminati-proxy-manager-v1.108.119-setup.exe">代理管理安装器</a>.
+下载 <a href="https://luminati-china.io/static/lpm/luminati-proxy-manager-v1.108.399-setup.exe">代理管理安装器</a>.
 
 ### Linux/MacOS
-- 安装 Node.js 6+版 (最好用x
+- 安装 Node.js 10版 (最好用x
   [nave](https://github.com/isaacs/nave))
 - 从终端安装 Luminati 代理:
 ```sh

diff --git a/README.md b/README.md
@@ -25,7 +25,7 @@ This tool requires a [Luminati](https://luminati.io/?cam=github-proxy) account.
 ## Installation
 
 ### Windows
-Download the [Luminati Proxy Manager installer](https://github.com/luminati-io/luminati-proxy/releases/download/v1.108.119/luminati-proxy-manager-v1.108.119-setup.exe)
+Download the [Luminati Proxy Manager installer](https://github.com/luminati-io/luminati-proxy/releases/download/v1.108.399/luminati-proxy-manager-v1.108.399-setup.exe)
 
 ### Linux/MacOs - Install script
 - Run the setup script to install
@@ -37,11 +37,10 @@ Or
 curl -L https://luminati.io/static/lpm/luminati-proxy-latest-setup.sh | bash
 ```
 ### Linux/MacOS - Manual install
-- Install Node.js 8 ([nodejs.org](https://nodejs.org/en/download/))
-  Node.js version for the proxy manager should be any v 8.X.X.
-  The Proxy Manager will not work properly with later node versions. 
-- Make sure npm version is 4.6.1 or higher
-  - if not, run: `sudo npm install -g npm@4.6.1`
+- Install Node.js 10 ([nodejs.org](https://nodejs.org/en/download/))
+  Node.js version for the proxy manager should be any v 10.X.X.
+- Make sure npm version is 6.4.1 or higher
+  - if not, run: `sudo npm install -g npm@6.4.1`
 - Install Luminati Proxy from the terminal prompt:
 ```sh
 sudo npm install -g @luminati-io/luminati-proxy
@@ -125,6 +124,15 @@ see <a href="https://luminati.io/cp/api_example?manager=all&group=active">the
 API Example page in your Luminati.io account</a>.
 
 ### Complete list of command line options
+
+All command line options below are also available also as ENV variables.
+Example:
+```sh
+cli: --port 22900 , env: LPM_PORT=22900
+cli: --ssl true , env: LPM_SSL=true
+cli: --log "error" , env: LPM_LOG=error
+```
+
 ```sh
 luminati --help
 Usage:

diff --git a/bin/lpm_install.sh b/bin/lpm_install.sh
@@ -7,7 +7,7 @@ if [ $(id -u) = 0 ]; then
     IS_ROOT=1
 fi
 LUM=0
-VERSION="1.108.119"
+VERSION="1.108.399"
 if [ -f  "/usr/local/hola/zon_config.sh" ]; then
     LUM=1
 fi
@@ -21,8 +21,8 @@ INSTALL_NPM=0
 INSTALL_CURL=0
 INSTALL_BREW=0
 USE_NVM=0
-NODE_VER='8.11.3'
-NPM_VER='4.6.1'
+NODE_VER='10.11.0'
+NPM_VER='6.4.1'
 NETWORK_RETRY=3
 NETWORK_ERROR=0
 UPDATE_NODE=0
@@ -380,13 +380,8 @@ check_node()
         local node_ver=$(node -v)
         zerr "check_node: $node_ver"
         echo "node ${node_ver} is installed"
-        if ! [[ "$node_ver" =~ ^(v[7-9]\.|v[1-9][0-9]+\.) ]]; then
-            echo "minimum required node version is 7"
-            perr "check_node_bad_version" "$node_ver"
-            UPDATE_NODE=1
-        fi
-        if [[ "$node_ver" =~ ^(v[1-9][0-9]+\.) ]]; then
-            echo "maximum supported node version is 9"
+        if ! [[ "$node_ver" =~ ^(v[1-9]\.|v[0-9][0-9]+\.) ]]; then
+            echo "minimum required node version is 10"
             perr "check_node_bad_version" "$node_ver"
             UPDATE_NODE=1
         fi

diff --git a/bin/lum_node.js b/bin/lum_node.js
@@ -15,6 +15,7 @@ const version = require('../package.json').version;
 const analytics = require('universal-analytics');
 const _ = require('lodash');
 const crypto = require('crypto');
+const ps_list = require('ps-list');
 const ua = analytics('UA-60520689-2');
 const E = module.exports = {};
 const is_win = process.platform=='win32';
@@ -174,6 +175,30 @@ E.handle_signal = (sig, err)=>{
         E.shutdown(errstr, true, err);
 };
 
+let _show_port_conflict = (addr, port)=>etask(function*(){
+    let tasks;
+    try { tasks = yield ps_list(); }
+    catch(e){ process.exit(); }
+    tasks = tasks.filter(t=>t.name.includes('node') &&
+        /.*lum_node\.js.*/.test(t.cmd) && t.ppid!=process.pid &&
+        t.pid!=process.pid);
+    if (!tasks.length)
+        return E.manager.stop();
+    zerr.notice(`There is already an application running on ${addr}:${port}\n`+
+        'Trying to kill it and restart.');
+    for (const t of tasks)
+        process.kill(t.ppid, 'SIGTERM');
+    E.manager.restart();
+});
+
+let conflict_shown;
+let show_port_conflict = (addr, port)=>{
+    if (conflict_shown)
+        return;
+    conflict_shown = true;
+    _show_port_conflict(addr, port);
+};
+
 E.run = (argv, run_config)=>{
     E.read_status_file();
     E.write_status_file('initializing', null,
@@ -190,6 +215,9 @@ E.run = (argv, run_config)=>{
     .on('error', (e, fatal)=>{
         zerr(e.raw ? e.message : 'Unhandled error: '+e);
         let handle_fatal = ()=>{
+            let err;
+            if (err = (e.message||'').match(/((?:\d{1,3}\.?){4}):(\d+)$/))
+                return show_port_conflict(err[1], err[2]);
             if (fatal)
                 E.manager.stop();
         };

diff --git a/lib/loki.js b/lib/loki.js
@@ -1,19 +1,20 @@
 // LICENSE_CODE ZON ISC
 'use strict'; /*jslint node:true, esnext:true*/
-const _ = require('lodash');
 const lokijs = require('lokijs');
 const lfsa = require('lokijs/src/loki-fs-structured-adapter.js');
 const etask = require('../util/etask.js');
 const string = require('../util/string.js');
 const file = require('../util/file.js');
 const qw = string.qw;
+const log = require('./log.js');
 
 const E = module.exports = Loki;
 
-function Loki(path){
+function Loki(path, log_lvl){
     this.path = path;
     this.colls = {};
     this.db_names = qw`port status_code hostname protocol`;
+    this.log = log('loki', log_lvl);
 }
 
 E.prototype.prepare = etask._fn(function*loki_prepare(_this){
@@ -25,7 +26,7 @@ E.prototype.prepare = etask._fn(function*loki_prepare(_this){
         autoload: true,
         autoloadCallback: ()=>this.continue(),
         autosave: true,
-        autosaveInterval: 4000,
+        autosaveInterval: 60000,
     });
     yield this.wait();
     _this.db_names.forEach(db=>{
@@ -35,6 +36,18 @@ E.prototype.prepare = etask._fn(function*loki_prepare(_this){
     });
 });
 
+E.prototype.save = etask._fn(function*loki_save(_this){
+    const et = this;
+    _this.log.notice('Saving stats to the file');
+    _this.loki.saveDatabase(function(err){
+        if (!err)
+            return et.continue(true);
+        _this.log.error(err);
+        et.continue(err);
+    });
+    yield this.wait();
+});
+
 E.prototype.stats_clear = function(){
     this.db_names.forEach(db=>this.colls[db].findAndRemove());
 };

diff --git a/lib/luminati.js b/lib/luminati.js
@@ -27,12 +27,12 @@ const zerr = require('../util/zerr.js');
 const zfile = require('../util/file.js');
 const lpm_config = require('../util/lpm_config.js');
 const qw = require('../util/string.js').qw;
-const child_process = require('child_process');
 const uuid_v4 = require('uuid/v4');
 const sessions = require('./session.js');
 const Context = require('./context.js');
 const Router = require('./router.js');
 const Rules = require('./rules.js');
+const ReqProcess = require('./req_process.js');
 const assign = Object.assign, {SEC} = date.ms;
 const E = module.exports = Luminati;
 E.user_agent = 'luminati-proxy-manager/'+version;
@@ -162,6 +162,8 @@ function Luminati(opt, mgr){
     this.failure = {};
     this.requests_queue = [];
     this.throttle_queue = [];
+    this.req_process = new ReqProcess({listen_port: opt.listen_port,
+        log: opt.log});
     this.http_server = http.createServer((req, res, head)=>{
         if (this.is_overload())
             return this.send_overload(res, 'http_req');
@@ -177,6 +179,7 @@ function Luminati(opt, mgr){
     http_shutdown(this.http_server);
     this.socks_server = new Socks({port: opt.port||E.default.port,
         log: opt.log});
+    this.socks_server.on('error', err=>this.emit('error', err));
     if (opt.ssl)
     {
         this.authorization = {};
@@ -708,7 +711,7 @@ function*init_proxy_req(_this, req, res){
     };
     ctx.connect_headers = {
         'proxy-authorization': 'Basic '+
-            new Buffer(ctx.cred.username+':'+ctx.cred.password)
+            Buffer.from(ctx.cred.username+':'+ctx.cred.password)
             .toString('base64'),
     };
     if (!ctx.ext_proxy)
@@ -755,17 +758,7 @@ E.prototype._request = etask._fn(function*_request(_this, req, res, head){
                 'HTTPS connection to IP will be done from super agent');
         }
         const remote = req.socket.remoteAddress+':'+req.socket.remotePort;
-        const netstat_cmd = `netstat -antp 2>/dev/null | grep "${remote}"`;
-        let pid;
-        if (!lpm_config.is_win && req.socket.remoteAddress=='127.0.0.1')
-        {
-            try {
-                const stdout = child_process.execSync(netstat_cmd);
-                pid = stdout.toString().split('\n').filter(Boolean)
-                .map(s=>s.replace(/\s+/g, ' ').split(' '))
-                .find(r=>r[3]==remote);
-            } catch(e){ pid = ''; }
-        }
+        _this.req_process.add(req);
         _this._handle_usage_start(req);
         let resp;
         if (ctx.rules)
@@ -787,7 +780,8 @@ E.prototype._request = etask._fn(function*_request(_this, req, res, head){
             throw new Error('invalid_response');
         if (ctx.wait_bw)
             yield this.wait_ext(ctx.wait_bw);
-        resp.remote_address = pid ? `${remote}, ${pid[6]}` : remote;
+        resp.remote_address = req.process ?
+            `${remote}, ${req.process}` : remote;
         resp.uuid = req.ctx.uuid;
         const auth = username.parse(ctx.h_proxy_authorization);
         if (auth&&auth.tool=='proxy_tester')

diff --git a/lib/manager.js b/lib/manager.js
@@ -251,7 +251,7 @@ E.prototype.init = function(argv, run_config){
     this._defaults = conf._defaults;
     this.proxies = conf.proxies;
     this.mode = argv.mode;
-    this.loki = new Loki(argv.loki);
+    this.loki = new Loki(argv.loki, argv.log);
     if (!['root', 'normal', 'guest'].includes(this.mode))
     {
         console.log(`Unrecognized UI mode (${this.mode}); treating as guest`);
@@ -511,6 +511,7 @@ function*mgr_stop(_this, reason, force, restart){
     _this.is_running = false;
     yield _this.perr(restart ? 'restart' : 'exit', {reason});
     yield _this.stop_dbs();
+    yield _this.loki.save();
     if (reason!='config change')
         yield _this.save_config();
     if (reason instanceof Error)
@@ -1783,7 +1784,7 @@ E.prototype.test_api = function(req, res){
     });
     const password = this.proxies_running[port].config.password;
     const user = 'tool-proxy_tester';
-    const basic = new Buffer(user+':'+password).toString('base64');
+    const basic = Buffer.from(user+':'+password).toString('base64');
     opt.headers = opt.headers||{};
     opt.headers['proxy-authorization'] = 'Basic '+basic;
     if (+port)
@@ -1792,8 +1793,6 @@ E.prototype.test_api = function(req, res){
         if (this.proxies_running[port].opt.ssl)
             opt.ca = ssl.ca.cert;
         opt.headers = opt.headers||{};
-        opt.headers['x-hola-context'] = opt.headers['x-hola-context']
-            ||'PROXY TESTER TOOL';
     }
     request(opt, err=>{
         if (!err)
@@ -1896,8 +1895,6 @@ E.prototype.logs_resend = etask._fn(function*mgr_logs_resend(_this, req, res){
         };
         if (proxy.opt.ssl)
             opt.ca = ssl.ca.cert;
-        opt.headers['x-hola-context'] = opt.headers['x-hola-context']
-            ||'HAR VIEWER';
         request(opt);
     }
     res.send('ok');
@@ -2048,6 +2045,7 @@ E.prototype.get_settings = function(){
         api: this.argv.api,
         logs: this._defaults.logs,
         request_stats: this._defaults.request_stats,
+        dropin: this._defaults.dropin,
     };
 };
 
@@ -2271,7 +2269,7 @@ function*mgr_refresh_vips(_this, req, res){
         res.json({error: response.body});
 });
 
-E.prototype.shutdown = function(req, res){
+E.prototype.shutdown_api = function(req, res){
     res.json({result: 'ok'});
     this.stop(true);
 };
@@ -2355,7 +2353,37 @@ E.prototype.stats_clear_api = function(req, res){
     res.end();
 };
 
-E.prototype.sync_recent_stats = etask._fn(function*mgr_sync(_this){
+E.prototype.api_request = etask._fn(function*mgr_api_request(_this, opt){
+    const jar = _this.luminati_jar.jar;
+    yield etask.nfn_apply(request, [{url: _this.argv.api, jar}]);
+    const xsrf = (jar.getCookies(_this.argv.api).find(e=>
+        e.key=='XSRF-TOKEN')||{}).value;
+    return yield etask.nfn_apply(request, [{
+        method: opt.post||'GET',
+        url: opt.url,
+        qs: assign(_.pick(_this._defaults, qw`customer token`)),
+        jar,
+        json: true,
+        headers: {'X-XSRF-Token': xsrf},
+        form: opt.form,
+    }]);
+});
+
+E.prototype.sync_config_file = etask._fn(function*mgr_sync_config(_this){
+    if (!_this.logged_in)
+        return;
+    const res = yield _this.api_request({
+        url: `${_this.argv.api}/update_lpm_config`,
+        method: 'POST', form: {def: 1},
+    });
+    if (res.statusCode!=200)
+    {
+        _this._log.warn('Config file was not synced with mongodb, %s',
+            res.statusCode);
+    }
+});
+
+E.prototype.sync_recent_stats = etask._fn(function*mgr_sync_stats(_this){
     if (!_this.logged_in)
         return;
     const stats = _this.loki.stats_get();
@@ -2480,7 +2508,7 @@ E.prototype.create_api_interface = function(){
     app.get('/allocated_vips', this._api(this.allocated_vips_get, ['root']));
     app.post('/refresh_ips', this._api(this.refresh_ips_api, ['root']));
     app.post('/refresh_vips', this._api(this.refresh_vips_api, ['root']));
-    app.post('/shutdown', this._api(this.shutdown, ['root']));
+    app.post('/shutdown', this._api(this.shutdown_api, ['root']));
     app.post('/logout', this._api(this.logout_api));
     app.post('/upgrade', this._api(this.upgrade, ['root']));
     app.post('/restart', this._api(this.restart, ['root']));