1.108.6

luminati-io · Sep 21, 2018 · 55859be · 55859be
1 parent 0e6f129
commit 55859be
Show file tree

Hide file tree

Showing 17 changed files with 118 additions and 54 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,8 @@
 Luminati Proxy manager - Change Log
 
+## 1.108.6 Development Latest
+- :sparkles: Logs speed optimization
+
 ## 1.107.868 Stable
 - :bug: Bug fixes
 - :sparkles: Rule: 'request time more than' improved

diff --git a/README-zh-CN.md b/README-zh-CN.md
@@ -29,7 +29,7 @@
 - <a href="https://nodejs.org/en/download/">Node.js</a> 6+版
 
 ### Windows
-下载 <a href="https://luminati-china.io/static/lpm/luminati-proxy-manager-v1.107.868-setup.exe">代理管理安装器</a>.
+下载 <a href="https://luminati-china.io/static/lpm/luminati-proxy-manager-v1.108.6-setup.exe">代理管理安装器</a>.
 
 ### Linux/MacOS
 - 安装 Node.js 6+版 (最好用x

diff --git a/README.md b/README.md
@@ -25,7 +25,7 @@ This tool requires a [Luminati](https://luminati.io/?cam=github-proxy) account.
 ## Installation
 
 ### Windows
-Download the [Luminati Proxy Manager installer](https://github.com/luminati-io/luminati-proxy/releases/download/v1.107.868/luminati-proxy-manager-v1.107.868-setup.exe)
+Download the [Luminati Proxy Manager installer](https://github.com/luminati-io/luminati-proxy/releases/download/v1.108.6/luminati-proxy-manager-v1.108.6-setup.exe)
 
 ### Linux/MacOs - Install script
 - Run the setup script to install

diff --git a/bin/lpm_install.sh b/bin/lpm_install.sh
@@ -7,7 +7,7 @@ if [ $(id -u) = 0 ]; then
     IS_ROOT=1
 fi
 LUM=0
-VERSION="1.107.868"
+VERSION="1.108.6"
 if [ -f  "/usr/local/hola/zon_config.sh" ]; then
     LUM=1
 fi

diff --git a/bin/pub/516e25dab62f9b5f161f.runtime.js → bin/pub/52f613a2de6df5210997.runtime.js b/bin/pub/516e25dab62f9b5f161f.runtime.js → bin/pub/52f613a2de6df5210997.runtime.js
@@ -91,7 +91,7 @@
 /******/ 		if (__webpack_require__.nc) {
 /******/ 			script.setAttribute("nonce", __webpack_require__.nc);
 /******/ 		}
-/******/ 		script.src = __webpack_require__.p + "" + {"0":"97ef8b010229e03187d4","1":"dff8cc8460349bcab527"}[chunkId] + "." + chunkId + ".js";
+/******/ 		script.src = __webpack_require__.p + "" + {"0":"97ef8b010229e03187d4","1":"91af6797d2fc1d4ccfbe"}[chunkId] + "." + chunkId + ".js";
 /******/ 		var timeout = setTimeout(onScriptComplete, 120000);
 /******/ 		script.onerror = script.onload = onScriptComplete;
 /******/ 		function onScriptComplete() {

diff --git a/bin/pub/dff8cc8460349bcab527.app.js → bin/pub/91af6797d2fc1d4ccfbe.app.js b/bin/pub/dff8cc8460349bcab527.app.js → bin/pub/91af6797d2fc1d4ccfbe.app.js
@@ -16539,11 +16539,12 @@ var Body_section = function (_Pure_component5) {
         key: 'render',
         value: function render() {
             if (!this.props.body) return null;
-            var body = void 0;
+            var json = void 0;
+            var raw_body = void 0;
             try {
-                body = JSON.parse(this.props.body);
+                json = JSON.parse(this.props.body);
             } catch (e) {
-                return null;
+                raw_body = this.props.body;
             }
             return [_react2.default.createElement(
                 'li',
@@ -16554,7 +16555,8 @@ var Body_section = function (_Pure_component5) {
                 'ol',
                 { key: 'ol',
                     className: (0, _classnames2.default)('children', { open: this.state.open }) },
-                _react2.default.createElement(_json_viewer2.default, { json: body })
+                !!json && _react2.default.createElement(_json_viewer2.default, { json: json }),
+                !!raw_body && _react2.default.createElement(Header_pair, { name: 'p', value: raw_body })
             )];
         }
     }]);
@@ -38925,13 +38927,14 @@ var provider = function provider(provide) {
 
 var trigger_types = [{ key: 'i.e. Status code', value: '', tooltip: 'Choose a trigger type.\n        For each request the system will check if the trigger is matching\n        the response' }, { key: 'URL', value: 'url', tooltip: 'Trigger will be pulled for all\n        requests to the selected URL' }, { key: 'Status code', value: 'status', tooltip: 'Trigger will be pulled\n        for all the requests that returns the matching status code' }, { key: 'HTML body element', value: 'body', tooltip: 'Trigger will be\n        pulled when the response <body> contain the selected string' }, { key: 'Request time more than', value: 'min_req_time',
     tooltip: 'Triggers when the request time is above the selected value',
-    type: 'pre' }, { key: 'Request time less than', value: 'max_req_time',
+    type: '--pre' }, { key: 'Request time less than', value: 'max_req_time',
     tooltip: 'Triggers when the request time is below the selected value' }];
 
 var default_action = { key: 'Choose an action type', value: '',
     tooltip: 'Select an action.  Once the trigger rule is met the selected\n    action is executed automatically.' };
-var action_types = [{ key: 'Retry with new IP', value: 'retry', tooltip: 'System will send the\n        exact same request again with newly refreshed IP', min_req_time: true }, { key: 'Retry with new proxy port (Waterfall)', value: 'retry_port',
-    tooltip: 'System will send another request using different port\n        from your port list. This can allow cost optimization by escalating the\n        request between different types of networks according to the port\n        configuration.', min_req_time: true }, { key: 'Ban IP', value: 'ban_ip', tooltip: 'Will ban the IP for custom\n        amount of time. usually used for failed request.' }, { key: 'Refresh IP', value: 'refresh_ip', tooltip: 'Refresh the current\n        Data Center IP with new allocated IP. This action contain\n        additional charges. View the cost of IP refreshing in your zones\n        page http://luminati.io/cp/zones' }, { key: 'Save IP to reserved pool', value: 'save_to_pool', tooltip: 'Save\n        the current IP to a pool of reserved IPs.  you can then download all\n        the IPs at a later time.' }, { key: 'Save IP to fast pool', value: 'save_to_fast_pool', tooltip: 'Save\n        the current IP to fast IP pool to increase the speed of your requests.\n        You will need to specify the size of this pool.' }, { key: 'Null response', value: 'null_response', tooltip: 'LPM will return a\n        "null response" without proxying. It is useful when users do not want\n        to make a request, but a browser expects 200 response.', type: 'pre',
+var action_types = [{ key: 'Retry with new IP', value: 'retry', tooltip: 'System will send the\n        exact same request again with newly refreshed IP',
+    min_req_time: false }, { key: 'Retry with new proxy port (Waterfall)', value: 'retry_port',
+    tooltip: 'System will send another request using different port\n        from your port list. This can allow cost optimization by escalating the\n        request between different types of networks according to the port\n        configuration.', min_req_time: false }, { key: 'Ban IP', value: 'ban_ip', tooltip: 'Will ban the IP for custom\n        amount of time. usually used for failed request.' }, { key: 'Refresh IP', value: 'refresh_ip', tooltip: 'Refresh the current\n        Data Center IP with new allocated IP. This action contain\n        additional charges. View the cost of IP refreshing in your zones\n        page http://luminati.io/cp/zones' }, { key: 'Save IP to reserved pool', value: 'save_to_pool', tooltip: 'Save\n        the current IP to a pool of reserved IPs.  you can then download all\n        the IPs at a later time.' }, { key: 'Save IP to fast pool', value: 'save_to_fast_pool', tooltip: 'Save\n        the current IP to fast IP pool to increase the speed of your requests.\n        You will need to specify the size of this pool.' }, { key: 'Null response', value: 'null_response', tooltip: 'LPM will return a\n        "null response" without proxying. It is useful when users do not want\n        to make a request, but a browser expects 200 response.', type: 'pre',
     only_url: true }, { key: 'Bypass proxy', value: 'bypass_proxy', tooltip: 'Requests will be\n        passed directly to target site without any proxy (super proxy or\n        peer).', type: 'pre', only_url: true }, { key: 'Direct super proxy', value: 'direct', tooltip: 'Requests will be\n        passed through super proxy (not through peers)', type: 'pre',
     only_url: true }, { key: 'Process data', value: 'process', only_url: true }];
 
@@ -39342,9 +39345,9 @@ var Rule = (0, _common.with_proxy_ports)((0, _reactRouterDom.withRouter)(functio
                 return at.value != 'save_to_fast_pool' || rule.trigger_type == 'max_req_time';
             }).filter(function (at) {
                 return rule.trigger_type == 'url' && at.only_url || rule.trigger_type != 'url' && !at.only_url;
-            })).filter(function (at) {
-                return rule.trigger_type != 'min_req_time' || at.min_req_time;
-            });
+            }));
+            // .filter(at=>rule.trigger_type!='min_req_time'||
+            //     at.min_req_time);
             var ports = this.props.ports_opt.filter(function (p) {
                 return p.value != _this10.props.match.params.port;
             });

diff --git a/bin/pub/index.html b/bin/pub/index.html
@@ -21,6 +21,6 @@
   <body>
     <div id="react_root"></div>
     <div id=notif_react_modal></div>
-  <script type="text/javascript" src="/516e25dab62f9b5f161f.runtime.js"></script><script type="text/javascript" src="/97ef8b010229e03187d4.vendor.js"></script><script type="text/javascript" src="/dff8cc8460349bcab527.app.js"></script></body>
+  <script type="text/javascript" src="/52f613a2de6df5210997.runtime.js"></script><script type="text/javascript" src="/97ef8b010229e03187d4.vendor.js"></script><script type="text/javascript" src="/91af6797d2fc1d4ccfbe.app.js"></script></body>
 </html>
 
diff --git a/lib/luminati.js b/lib/luminati.js
@@ -289,12 +289,12 @@ E.prototype._send_rule_mail = function(to, trigger, action, _url){
 };
 
 E.prototype._handle_usage_start = function(req){
-    const uuid = uuid_v4();
-    req.ctx.uuid = uuid;
+    if (!req.ctx.uuid)
+        req.ctx.uuid = uuid_v4();
     if (!this.opt.handle_usage_start)
         return;
     const data = {
-        uuid,
+        uuid: req.ctx.uuid,
         port: this.port,
         url: req.url,
         method: req.method,
@@ -775,6 +775,12 @@ E.prototype._request = etask._fn(function*_request(_this, req, res, head){
             yield _this.init_proxy_req(req, res);
             resp = yield _this.route_req(req, res, head);
         }
+        if (resp=='switched')
+        {
+            if (_this.opt.handle_abort)
+                _this.opt.handle_abort(req.ctx.uuid);
+            yield this.wait();
+        }
         if (resp instanceof Error)
             throw resp;
         if (!resp)
@@ -1248,7 +1254,7 @@ E.prototype._abort_proxy_req = function(req, proxy, task){
     proxy.destroy();
     if (task)
         task.return('abort');
-    // XXX krzysztof: fix waterfall
+    // XXX krzysztof: fix retry
     if (this.opt.handle_abort)
         this.opt.handle_abort(req.ctx.uuid);
 };

diff --git a/lib/manager.js b/lib/manager.js
@@ -180,7 +180,7 @@ E.prototype.handle_usage = function(data){
     {
         this.loki.stats_process(data);
     }
-    setImmediate(()=>this.history_aggregator(data));
+    this.logs_process(data);
 };
 
 E.prototype.handle_abort = function(uuid){
@@ -210,30 +210,43 @@ E.prototype.handle_usage_start = function(data){
     this.wss.broadcast(req, 'har_viewer_start');
 };
 
-E.prototype.history_aggregator = etask._fn(function*hist_agg_mgr(_this, data){
-    this.on('uncaught', e=>{
-        _this._log.error('history aggregator %s', zerr.e2s(e));
-    });
-    const har_req = _this.har([data]).log.entries[0];
-    if (!_this._defaults.logs.value)
-        return _this.emit('request_log', har_req);
-    yield _this.adjust_logs_size({delete_more: true});
+E.prototype.logs_process = function(data){
+    const har_req = this.har([data]).log.entries[0];
+    if (!this._defaults.logs.value)
+        return this.emit('request_log', har_req);
+    if (this.wss)
+        this.wss.broadcast(har_req, 'har_viewer');
+    this.emit('request_log', har_req);
     const row = {};
     for (let f in data)
         row['$'+f] = data[f];
-    _this.db.main.stmt.history.run(row, function(err){
-        if (_this.wss)
-            _this.wss.broadcast(har_req, 'har_viewer');
-        _this.emit('request_log', har_req);
+    this.logs_queue.push(row);
+    if (this.logs_queue.length<100)
+        return;
+    this.history_aggregator();
+};
+
+E.prototype.history_aggregator = function(){
+    const _this = this;
+    this.db.main.serialize(function(){
+        _this.db.main.run('begin transaction');
+        for (let row of _this.logs_queue)
+            _this.db.main.stmt.history.run(row);
+        _this.logs_queue = [];
+        _this.db.main.run('commit');
+        _this.db.main.run('vacuum');
     });
-});
+    this.logs_queue.ts = Date.now();
+};
 
 E.prototype.init = function(argv, run_config){
     this.rmt_cfg = new Rmt_lpm_cnf(this);
     this.run_config = run_config||{};
     if (!this.run_config.id)
         this.run_config.id = +new Date();
     this.db = {};
+    this.logs_queue = [];
+    this.logs_queue.ts = Date.now();
     this.proxies_running = {};
     this.warnings = [];
     this.argv = sanitize_argv(argv);
@@ -782,6 +795,27 @@ function*mgr_archiv_tables(_this, tables){
     }
 });
 
+E.prototype.start_syncing_logs = function(){
+    if (this._defaults.logs.value)
+        this.syncing_logs_task = this.sync_logs();
+};
+
+E.prototype.sync_logs = etask._fn(function*mgr_sync_logs(_this){
+    this.on('uncaught', e=>_this._log.error('sync_logs %s', zerr.e2s(e)));
+    yield _this.adjust_logs_size();
+    const now = Date.now();
+    if (_this.logs_queue && now-_this.logs_queue.ts>=1000)
+    {
+        _this.history_aggregator();
+        yield etask.sleep(1000);
+    }
+    else
+        yield etask.sleep(now-_this.logs_queue.ts);
+    if (_this.syncing_logs_task)
+        _this.syncing_logs_task.return();
+    _this.start_syncing_logs();
+});
+
 E.prototype.prepare_database = etask._fn(function*mgr_prepare_database(_this){
     this.on('uncaught', e=>_this._log.error('prepare db %s', zerr.e2s(e)));
     if (_this.argv.log=='debug')
@@ -2082,7 +2116,6 @@ function*adjust_logs_mgr(_this, opt={}){
             yield _this.delete_logs_sql(to_del);
         }
     }
-    yield _this.vacuum_sql({force: true});
 });
 
 // XXX krzysztof: improve mechanism for defaults values
@@ -2091,6 +2124,10 @@ function*update_settings_mgr(_this, req, res){
     if (req.body.zone!==undefined)
         _this._defaults.zone = req.body.zone;
     _this._defaults.logs = req.body.logs;
+    if (_this.syncing_logs_task)
+        _this.syncing_logs_task.return();
+    _this.syncing_logs_task = null;
+    _this.start_syncing_logs();
     yield _this.adjust_logs_size();
     _this._defaults.request_stats = req.body.request_stats;
     if (_this._defaults.request_stats===undefined||
@@ -2620,7 +2657,7 @@ function*mgr_get_lum_local_conf(_this, customer, token, force_login){
     token = token||_this._defaults.token;
     if (!_this.lum_conf)
         _this.lum_conf = {ts: 0};
-    let now = Date.now();
+    const now = Date.now();
     if (_this.lum_conf._defaults && now-_this.lum_conf.ts < 60*1000 ||
         _this.use_local_lum_conf&&!force_login)
     {
@@ -2877,6 +2914,7 @@ E.prototype.start = etask._fn(function*mgr_start(_this){
         yield _this.loki.prepare();
         yield _this.prepare_database();
         _this.prepare_db_stmt();
+        _this.start_syncing_logs();
         yield _this.logged_update();
         yield _this.sync_recent_stats();
         yield _this.init_proxies();

diff --git a/lib/rules.js b/lib/rules.js
@@ -148,7 +148,8 @@ module.exports = class Rules {
                     return _this.send_null_response(req, res);
                 ctx.rules = serv.rules;
                 ctx.port = serv.port;
-                return yield serv._request(req, res, head);
+                serv._request(req, res, head);
+                return 'switched';
             }
         }
     }); }
@@ -379,7 +380,7 @@ module.exports = class Rules {
     }
     // XXX krzysztof: WIP _can_retry is a newer implementation of can_retry
     _can_retry(req, res, rule){
-        if (rule.action!='retry'&&rule.action!='retry_ip')
+        if (rule.action!='retry'&&rule.action!='retry_port')
             return false;
         const retry = req.retry||0;
         const port = rule.retry_port;

diff --git a/lib/swagger.json b/lib/swagger.json
@@ -1,7 +1,7 @@
 {
     "swagger": "2.0",
     "info": {
-        "version": "1.107.868",
+        "version": "1.108.6",
         "title": "Luminati Proxy Manager",
         "license": {
             "name": "MIT",

diff --git a/package.json b/package.json
@@ -347,7 +347,7 @@
       "util/lpm_util.js",
       "webpack.config.js"
     ],
-    "md5": "40612fa1c6ff7d8f8aa71f17c1f75189"
+    "md5": "9838ba850137540bb11e04dd4ba03e80"
   },
   "main": "bin/index.js",
   "name": "@luminati-io/luminati-proxy",
@@ -382,5 +382,5 @@
     "test-win": "node node_modules/mocha/bin/mocha",
     "build": "node_modules/webpack/bin/webpack.js"
   },
-  "version": "1.107.868"
+  "version": "1.108.6"
 }
diff --git a/src/pub/har_preview.js b/src/pub/har_preview.js
@@ -155,17 +155,19 @@ class Body_section extends Pure_component {
     render(){
         if (!this.props.body)
             return null;
-        let body;
-        try { body = JSON.parse(this.props.body); }
-        catch(e){ return null; }
+        let json;
+        let raw_body;
+        try { json = JSON.parse(this.props.body); }
+        catch(e){ raw_body = this.props.body; }
         return [
             <li key="li" onClick={this.toggle}
               className={classnames('parent_title', {open: this.state.open})}>
               {this.props.title}
             </li>,
             <ol key="ol"
               className={classnames('children', {open: this.state.open})}>
-              <JSON_viewer json={body}/>
+              {!!json && <JSON_viewer json={json}/>}
+              {!!raw_body && <Header_pair name="p" value={raw_body}/>}
             </ol>
         ];
     }

diff --git a/src/pub/proxy_edit/rules.js b/src/pub/proxy_edit/rules.js
@@ -26,7 +26,7 @@ const trigger_types = [
         pulled when the response <body> contain the selected string`},
     {key: 'Request time more than', value: 'min_req_time',
         tooltip: `Triggers when the request time is above the selected value`,
-        type: 'pre'},
+        type: '--pre'},
     {key: 'Request time less than', value: 'max_req_time',
         tooltip: `Triggers when the request time is below the selected value`},
 ];
@@ -36,12 +36,13 @@ const default_action = {key: 'Choose an action type', value: '',
     action is executed automatically.`};
 const action_types = [
     {key: 'Retry with new IP', value: 'retry', tooltip: `System will send the
-        exact same request again with newly refreshed IP`, min_req_time: true},
+        exact same request again with newly refreshed IP`,
+        min_req_time: false},
     {key: 'Retry with new proxy port (Waterfall)', value: 'retry_port',
         tooltip: `System will send another request using different port
         from your port list. This can allow cost optimization by escalating the
         request between different types of networks according to the port
-        configuration.`, min_req_time: true},
+        configuration.`, min_req_time: false},
     {key: 'Ban IP', value: 'ban_ip', tooltip: `Will ban the IP for custom
         amount of time. usually used for failed request.`},
     {key: 'Refresh IP', value: 'refresh_ip', tooltip: `Refresh the current
@@ -393,9 +394,9 @@ const Rule = with_proxy_ports(withRouter(class Rule extends Pure_component {
         .filter(at=>at.value!='save_to_fast_pool'||
             rule.trigger_type=='max_req_time')
         .filter(at=>rule.trigger_type=='url'&&at.only_url||
-            rule.trigger_type!='url'&&!at.only_url))
-        .filter(at=>rule.trigger_type!='min_req_time'||
-            at.min_req_time);
+            rule.trigger_type!='url'&&!at.only_url));
+        // .filter(at=>rule.trigger_type!='min_req_time'||
+        //     at.min_req_time);
         const ports = this.props.ports_opt.filter(p=>
             p.value!=this.props.match.params.port);
         return <div className="rule_wrapper">