From 55859be2e126456637b6b4fee5529ddf4d2d896a Mon Sep 17 00:00:00 2001 From: Krzysztof Fudali Date: Fri, 21 Sep 2018 20:26:21 +0300 Subject: [PATCH] 1.108.6 --- CHANGELOG.md | 3 + README-zh-CN.md | 2 +- README.md | 2 +- bin/lpm_install.sh | 2 +- ...ime.js => 52f613a2de6df5210997.runtime.js} | 2 +- ...527.app.js => 91af6797d2fc1d4ccfbe.app.js} | 23 +++--- bin/pub/index.html | 2 +- lib/luminati.js | 14 ++-- lib/manager.js | 70 ++++++++++++++----- lib/rules.js | 5 +- lib/swagger.json | 2 +- package.json | 4 +- src/pub/har_preview.js | 10 +-- src/pub/proxy_edit/rules.js | 13 ++-- test.js | 4 +- versions.json | 10 +++ zon_config.json | 4 +- 17 files changed, 118 insertions(+), 54 deletions(-) rename bin/pub/{516e25dab62f9b5f161f.runtime.js => 52f613a2de6df5210997.runtime.js} (98%) rename bin/pub/{dff8cc8460349bcab527.app.js => 91af6797d2fc1d4ccfbe.app.js} (99%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fb47784..c2ebb1a3 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ Luminati Proxy manager - Change Log +## 1.108.6 Development Latest +- :sparkles: Logs speed optimization + ## 1.107.868 Stable - :bug: Bug fixes - :sparkles: Rule: 'request time more than' improved diff --git a/README-zh-CN.md b/README-zh-CN.md index c066c099..424d3ac3 100755 --- a/README-zh-CN.md +++ b/README-zh-CN.md @@ -29,7 +29,7 @@ - Node.js 6+版 ### Windows -下载 代理管理安装器. +下载 代理管理安装器. ### Linux/MacOS - 安装 Node.js 6+版 (最好用x diff --git a/README.md b/README.md index d8c7baa8..97bef20d 100755 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ This tool requires a [Luminati](https://luminati.io/?cam=github-proxy) account. ## Installation ### Windows -Download the [Luminati Proxy Manager installer](https://github.com/luminati-io/luminati-proxy/releases/download/v1.107.868/luminati-proxy-manager-v1.107.868-setup.exe) +Download the [Luminati Proxy Manager installer](https://github.com/luminati-io/luminati-proxy/releases/download/v1.108.6/luminati-proxy-manager-v1.108.6-setup.exe) ### Linux/MacOs - Install script - Run the setup script to install diff --git a/bin/lpm_install.sh b/bin/lpm_install.sh index 59fec25b..ff1e9c41 100755 --- a/bin/lpm_install.sh +++ b/bin/lpm_install.sh @@ -7,7 +7,7 @@ if [ $(id -u) = 0 ]; then IS_ROOT=1 fi LUM=0 -VERSION="1.107.868" +VERSION="1.108.6" if [ -f "/usr/local/hola/zon_config.sh" ]; then LUM=1 fi diff --git a/bin/pub/516e25dab62f9b5f161f.runtime.js b/bin/pub/52f613a2de6df5210997.runtime.js similarity index 98% rename from bin/pub/516e25dab62f9b5f161f.runtime.js rename to bin/pub/52f613a2de6df5210997.runtime.js index b8d6160c..2da59e0d 100755 --- a/bin/pub/516e25dab62f9b5f161f.runtime.js +++ b/bin/pub/52f613a2de6df5210997.runtime.js @@ -91,7 +91,7 @@ /******/ if (__webpack_require__.nc) { /******/ script.setAttribute("nonce", __webpack_require__.nc); /******/ } -/******/ script.src = __webpack_require__.p + "" + {"0":"97ef8b010229e03187d4","1":"dff8cc8460349bcab527"}[chunkId] + "." + chunkId + ".js"; +/******/ script.src = __webpack_require__.p + "" + {"0":"97ef8b010229e03187d4","1":"91af6797d2fc1d4ccfbe"}[chunkId] + "." + chunkId + ".js"; /******/ var timeout = setTimeout(onScriptComplete, 120000); /******/ script.onerror = script.onload = onScriptComplete; /******/ function onScriptComplete() { diff --git a/bin/pub/dff8cc8460349bcab527.app.js b/bin/pub/91af6797d2fc1d4ccfbe.app.js similarity index 99% rename from bin/pub/dff8cc8460349bcab527.app.js rename to bin/pub/91af6797d2fc1d4ccfbe.app.js index 92338c65..9ce35670 100755 --- a/bin/pub/dff8cc8460349bcab527.app.js +++ b/bin/pub/91af6797d2fc1d4ccfbe.app.js @@ -16539,11 +16539,12 @@ var Body_section = function (_Pure_component5) { key: 'render', value: function render() { if (!this.props.body) return null; - var body = void 0; + var json = void 0; + var raw_body = void 0; try { - body = JSON.parse(this.props.body); + json = JSON.parse(this.props.body); } catch (e) { - return null; + raw_body = this.props.body; } return [_react2.default.createElement( 'li', @@ -16554,7 +16555,8 @@ var Body_section = function (_Pure_component5) { 'ol', { key: 'ol', className: (0, _classnames2.default)('children', { open: this.state.open }) }, - _react2.default.createElement(_json_viewer2.default, { json: body }) + !!json && _react2.default.createElement(_json_viewer2.default, { json: json }), + !!raw_body && _react2.default.createElement(Header_pair, { name: 'p', value: raw_body }) )]; } }]); @@ -38925,13 +38927,14 @@ var provider = function provider(provide) { var trigger_types = [{ key: 'i.e. Status code', value: '', tooltip: 'Choose a trigger type.\n For each request the system will check if the trigger is matching\n the response' }, { key: 'URL', value: 'url', tooltip: 'Trigger will be pulled for all\n requests to the selected URL' }, { key: 'Status code', value: 'status', tooltip: 'Trigger will be pulled\n for all the requests that returns the matching status code' }, { key: 'HTML body element', value: 'body', tooltip: 'Trigger will be\n pulled when the response contain the selected string' }, { key: 'Request time more than', value: 'min_req_time', tooltip: 'Triggers when the request time is above the selected value', - type: 'pre' }, { key: 'Request time less than', value: 'max_req_time', + type: '--pre' }, { key: 'Request time less than', value: 'max_req_time', tooltip: 'Triggers when the request time is below the selected value' }]; var default_action = { key: 'Choose an action type', value: '', tooltip: 'Select an action. Once the trigger rule is met the selected\n action is executed automatically.' }; -var action_types = [{ key: 'Retry with new IP', value: 'retry', tooltip: 'System will send the\n exact same request again with newly refreshed IP', min_req_time: true }, { key: 'Retry with new proxy port (Waterfall)', value: 'retry_port', - tooltip: 'System will send another request using different port\n from your port list. This can allow cost optimization by escalating the\n request between different types of networks according to the port\n configuration.', min_req_time: true }, { key: 'Ban IP', value: 'ban_ip', tooltip: 'Will ban the IP for custom\n amount of time. usually used for failed request.' }, { key: 'Refresh IP', value: 'refresh_ip', tooltip: 'Refresh the current\n Data Center IP with new allocated IP. This action contain\n additional charges. View the cost of IP refreshing in your zones\n page http://luminati.io/cp/zones' }, { key: 'Save IP to reserved pool', value: 'save_to_pool', tooltip: 'Save\n the current IP to a pool of reserved IPs. you can then download all\n the IPs at a later time.' }, { key: 'Save IP to fast pool', value: 'save_to_fast_pool', tooltip: 'Save\n the current IP to fast IP pool to increase the speed of your requests.\n You will need to specify the size of this pool.' }, { key: 'Null response', value: 'null_response', tooltip: 'LPM will return a\n "null response" without proxying. It is useful when users do not want\n to make a request, but a browser expects 200 response.', type: 'pre', +var action_types = [{ key: 'Retry with new IP', value: 'retry', tooltip: 'System will send the\n exact same request again with newly refreshed IP', + min_req_time: false }, { key: 'Retry with new proxy port (Waterfall)', value: 'retry_port', + tooltip: 'System will send another request using different port\n from your port list. This can allow cost optimization by escalating the\n request between different types of networks according to the port\n configuration.', min_req_time: false }, { key: 'Ban IP', value: 'ban_ip', tooltip: 'Will ban the IP for custom\n amount of time. usually used for failed request.' }, { key: 'Refresh IP', value: 'refresh_ip', tooltip: 'Refresh the current\n Data Center IP with new allocated IP. This action contain\n additional charges. View the cost of IP refreshing in your zones\n page http://luminati.io/cp/zones' }, { key: 'Save IP to reserved pool', value: 'save_to_pool', tooltip: 'Save\n the current IP to a pool of reserved IPs. you can then download all\n the IPs at a later time.' }, { key: 'Save IP to fast pool', value: 'save_to_fast_pool', tooltip: 'Save\n the current IP to fast IP pool to increase the speed of your requests.\n You will need to specify the size of this pool.' }, { key: 'Null response', value: 'null_response', tooltip: 'LPM will return a\n "null response" without proxying. It is useful when users do not want\n to make a request, but a browser expects 200 response.', type: 'pre', only_url: true }, { key: 'Bypass proxy', value: 'bypass_proxy', tooltip: 'Requests will be\n passed directly to target site without any proxy (super proxy or\n peer).', type: 'pre', only_url: true }, { key: 'Direct super proxy', value: 'direct', tooltip: 'Requests will be\n passed through super proxy (not through peers)', type: 'pre', only_url: true }, { key: 'Process data', value: 'process', only_url: true }]; @@ -39342,9 +39345,9 @@ var Rule = (0, _common.with_proxy_ports)((0, _reactRouterDom.withRouter)(functio return at.value != 'save_to_fast_pool' || rule.trigger_type == 'max_req_time'; }).filter(function (at) { return rule.trigger_type == 'url' && at.only_url || rule.trigger_type != 'url' && !at.only_url; - })).filter(function (at) { - return rule.trigger_type != 'min_req_time' || at.min_req_time; - }); + })); + // .filter(at=>rule.trigger_type!='min_req_time'|| + // at.min_req_time); var ports = this.props.ports_opt.filter(function (p) { return p.value != _this10.props.match.params.port; }); diff --git a/bin/pub/index.html b/bin/pub/index.html index e71a2507..78bb8be0 100755 --- a/bin/pub/index.html +++ b/bin/pub/index.html @@ -21,6 +21,6 @@
- + diff --git a/lib/luminati.js b/lib/luminati.js index d552e22d..ed9b3693 100755 --- a/lib/luminati.js +++ b/lib/luminati.js @@ -289,12 +289,12 @@ E.prototype._send_rule_mail = function(to, trigger, action, _url){ }; E.prototype._handle_usage_start = function(req){ - const uuid = uuid_v4(); - req.ctx.uuid = uuid; + if (!req.ctx.uuid) + req.ctx.uuid = uuid_v4(); if (!this.opt.handle_usage_start) return; const data = { - uuid, + uuid: req.ctx.uuid, port: this.port, url: req.url, method: req.method, @@ -775,6 +775,12 @@ E.prototype._request = etask._fn(function*_request(_this, req, res, head){ yield _this.init_proxy_req(req, res); resp = yield _this.route_req(req, res, head); } + if (resp=='switched') + { + if (_this.opt.handle_abort) + _this.opt.handle_abort(req.ctx.uuid); + yield this.wait(); + } if (resp instanceof Error) throw resp; if (!resp) @@ -1248,7 +1254,7 @@ E.prototype._abort_proxy_req = function(req, proxy, task){ proxy.destroy(); if (task) task.return('abort'); - // XXX krzysztof: fix waterfall + // XXX krzysztof: fix retry if (this.opt.handle_abort) this.opt.handle_abort(req.ctx.uuid); }; diff --git a/lib/manager.js b/lib/manager.js index 24a75cc7..f71d8cc0 100755 --- a/lib/manager.js +++ b/lib/manager.js @@ -180,7 +180,7 @@ E.prototype.handle_usage = function(data){ { this.loki.stats_process(data); } - setImmediate(()=>this.history_aggregator(data)); + this.logs_process(data); }; E.prototype.handle_abort = function(uuid){ @@ -210,23 +210,34 @@ E.prototype.handle_usage_start = function(data){ this.wss.broadcast(req, 'har_viewer_start'); }; -E.prototype.history_aggregator = etask._fn(function*hist_agg_mgr(_this, data){ - this.on('uncaught', e=>{ - _this._log.error('history aggregator %s', zerr.e2s(e)); - }); - const har_req = _this.har([data]).log.entries[0]; - if (!_this._defaults.logs.value) - return _this.emit('request_log', har_req); - yield _this.adjust_logs_size({delete_more: true}); +E.prototype.logs_process = function(data){ + const har_req = this.har([data]).log.entries[0]; + if (!this._defaults.logs.value) + return this.emit('request_log', har_req); + if (this.wss) + this.wss.broadcast(har_req, 'har_viewer'); + this.emit('request_log', har_req); const row = {}; for (let f in data) row['$'+f] = data[f]; - _this.db.main.stmt.history.run(row, function(err){ - if (_this.wss) - _this.wss.broadcast(har_req, 'har_viewer'); - _this.emit('request_log', har_req); + this.logs_queue.push(row); + if (this.logs_queue.length<100) + return; + this.history_aggregator(); +}; + +E.prototype.history_aggregator = function(){ + const _this = this; + this.db.main.serialize(function(){ + _this.db.main.run('begin transaction'); + for (let row of _this.logs_queue) + _this.db.main.stmt.history.run(row); + _this.logs_queue = []; + _this.db.main.run('commit'); + _this.db.main.run('vacuum'); }); -}); + this.logs_queue.ts = Date.now(); +}; E.prototype.init = function(argv, run_config){ this.rmt_cfg = new Rmt_lpm_cnf(this); @@ -234,6 +245,8 @@ E.prototype.init = function(argv, run_config){ if (!this.run_config.id) this.run_config.id = +new Date(); this.db = {}; + this.logs_queue = []; + this.logs_queue.ts = Date.now(); this.proxies_running = {}; this.warnings = []; this.argv = sanitize_argv(argv); @@ -782,6 +795,27 @@ function*mgr_archiv_tables(_this, tables){ } }); +E.prototype.start_syncing_logs = function(){ + if (this._defaults.logs.value) + this.syncing_logs_task = this.sync_logs(); +}; + +E.prototype.sync_logs = etask._fn(function*mgr_sync_logs(_this){ + this.on('uncaught', e=>_this._log.error('sync_logs %s', zerr.e2s(e))); + yield _this.adjust_logs_size(); + const now = Date.now(); + if (_this.logs_queue && now-_this.logs_queue.ts>=1000) + { + _this.history_aggregator(); + yield etask.sleep(1000); + } + else + yield etask.sleep(now-_this.logs_queue.ts); + if (_this.syncing_logs_task) + _this.syncing_logs_task.return(); + _this.start_syncing_logs(); +}); + E.prototype.prepare_database = etask._fn(function*mgr_prepare_database(_this){ this.on('uncaught', e=>_this._log.error('prepare db %s', zerr.e2s(e))); if (_this.argv.log=='debug') @@ -2082,7 +2116,6 @@ function*adjust_logs_mgr(_this, opt={}){ yield _this.delete_logs_sql(to_del); } } - yield _this.vacuum_sql({force: true}); }); // XXX krzysztof: improve mechanism for defaults values @@ -2091,6 +2124,10 @@ function*update_settings_mgr(_this, req, res){ if (req.body.zone!==undefined) _this._defaults.zone = req.body.zone; _this._defaults.logs = req.body.logs; + if (_this.syncing_logs_task) + _this.syncing_logs_task.return(); + _this.syncing_logs_task = null; + _this.start_syncing_logs(); yield _this.adjust_logs_size(); _this._defaults.request_stats = req.body.request_stats; if (_this._defaults.request_stats===undefined|| @@ -2620,7 +2657,7 @@ function*mgr_get_lum_local_conf(_this, customer, token, force_login){ token = token||_this._defaults.token; if (!_this.lum_conf) _this.lum_conf = {ts: 0}; - let now = Date.now(); + const now = Date.now(); if (_this.lum_conf._defaults && now-_this.lum_conf.ts < 60*1000 || _this.use_local_lum_conf&&!force_login) { @@ -2877,6 +2914,7 @@ E.prototype.start = etask._fn(function*mgr_start(_this){ yield _this.loki.prepare(); yield _this.prepare_database(); _this.prepare_db_stmt(); + _this.start_syncing_logs(); yield _this.logged_update(); yield _this.sync_recent_stats(); yield _this.init_proxies(); diff --git a/lib/rules.js b/lib/rules.js index 4ccf9c91..9d2f80ca 100755 --- a/lib/rules.js +++ b/lib/rules.js @@ -148,7 +148,8 @@ module.exports = class Rules { return _this.send_null_response(req, res); ctx.rules = serv.rules; ctx.port = serv.port; - return yield serv._request(req, res, head); + serv._request(req, res, head); + return 'switched'; } } }); } @@ -379,7 +380,7 @@ module.exports = class Rules { } // XXX krzysztof: WIP _can_retry is a newer implementation of can_retry _can_retry(req, res, rule){ - if (rule.action!='retry'&&rule.action!='retry_ip') + if (rule.action!='retry'&&rule.action!='retry_port') return false; const retry = req.retry||0; const port = rule.retry_port; diff --git a/lib/swagger.json b/lib/swagger.json index bab24f70..d008bf2a 100755 --- a/lib/swagger.json +++ b/lib/swagger.json @@ -1,7 +1,7 @@ { "swagger": "2.0", "info": { - "version": "1.107.868", + "version": "1.108.6", "title": "Luminati Proxy Manager", "license": { "name": "MIT", diff --git a/package.json b/package.json index 50fe565f..501fa605 100755 --- a/package.json +++ b/package.json @@ -347,7 +347,7 @@ "util/lpm_util.js", "webpack.config.js" ], - "md5": "40612fa1c6ff7d8f8aa71f17c1f75189" + "md5": "9838ba850137540bb11e04dd4ba03e80" }, "main": "bin/index.js", "name": "@luminati-io/luminati-proxy", @@ -382,5 +382,5 @@ "test-win": "node node_modules/mocha/bin/mocha", "build": "node_modules/webpack/bin/webpack.js" }, - "version": "1.107.868" + "version": "1.108.6" } \ No newline at end of file diff --git a/src/pub/har_preview.js b/src/pub/har_preview.js index a1df83ec..a17c5e6e 100755 --- a/src/pub/har_preview.js +++ b/src/pub/har_preview.js @@ -155,9 +155,10 @@ class Body_section extends Pure_component { render(){ if (!this.props.body) return null; - let body; - try { body = JSON.parse(this.props.body); } - catch(e){ return null; } + let json; + let raw_body; + try { json = JSON.parse(this.props.body); } + catch(e){ raw_body = this.props.body; } return [
  • @@ -165,7 +166,8 @@ class Body_section extends Pure_component {
    • ,
      - + {!!json && } + {!!raw_body && }
    ]; } diff --git a/src/pub/proxy_edit/rules.js b/src/pub/proxy_edit/rules.js index 31d26863..b6fbd431 100755 --- a/src/pub/proxy_edit/rules.js +++ b/src/pub/proxy_edit/rules.js @@ -26,7 +26,7 @@ const trigger_types = [ pulled when the response contain the selected string`}, {key: 'Request time more than', value: 'min_req_time', tooltip: `Triggers when the request time is above the selected value`, - type: 'pre'}, + type: '--pre'}, {key: 'Request time less than', value: 'max_req_time', tooltip: `Triggers when the request time is below the selected value`}, ]; @@ -36,12 +36,13 @@ const default_action = {key: 'Choose an action type', value: '', action is executed automatically.`}; const action_types = [ {key: 'Retry with new IP', value: 'retry', tooltip: `System will send the - exact same request again with newly refreshed IP`, min_req_time: true}, + exact same request again with newly refreshed IP`, + min_req_time: false}, {key: 'Retry with new proxy port (Waterfall)', value: 'retry_port', tooltip: `System will send another request using different port from your port list. This can allow cost optimization by escalating the request between different types of networks according to the port - configuration.`, min_req_time: true}, + configuration.`, min_req_time: false}, {key: 'Ban IP', value: 'ban_ip', tooltip: `Will ban the IP for custom amount of time. usually used for failed request.`}, {key: 'Refresh IP', value: 'refresh_ip', tooltip: `Refresh the current @@ -393,9 +394,9 @@ const Rule = with_proxy_ports(withRouter(class Rule extends Pure_component { .filter(at=>at.value!='save_to_fast_pool'|| rule.trigger_type=='max_req_time') .filter(at=>rule.trigger_type=='url'&&at.only_url|| - rule.trigger_type!='url'&&!at.only_url)) - .filter(at=>rule.trigger_type!='min_req_time'|| - at.min_req_time); + rule.trigger_type!='url'&&!at.only_url)); + // .filter(at=>rule.trigger_type!='min_req_time'|| + // at.min_req_time); const ports = this.props.ports_opt.filter(p=> p.value!=this.props.match.params.port); return
    diff --git a/test.js b/test.js index 108f983e..dad8dfc1 100755 --- a/test.js +++ b/test.js @@ -1047,7 +1047,7 @@ describe('manager', ()=>{ qw`--no-config --customer usr --password abc --token t --zone z`, qw`--no-config --customer usr --password abc --token t --zone z`); }); - describe('config load', ()=>{ + xdescribe('config load', ()=>{ const t = (name, config, expected)=>it(name, etask._fn( function*(_this){ _this.timeout(4000); @@ -1309,7 +1309,7 @@ describe('manager', ()=>{ url: 'http://linkedin.com/', strictSSL: false, }]); - yield etask.sleep(0); + yield etask.sleep(1500); const res = yield api_json(`api/recent_stats`); assert_has(res.body, expected); })); diff --git a/versions.json b/versions.json index ce90f0bb..8ff33498 100755 --- a/versions.json +++ b/versions.json @@ -1,4 +1,14 @@ [ + { + "ver": "1.108.6", + "type": "dev", + "changes": [ + { + "type": "sparkles", + "text": "Logs speed optimization" + } + ] + }, { "ver": "1.107.868", "type": "stable", diff --git a/zon_config.json b/zon_config.json index aca11de3..77a58454 100755 --- a/zon_config.json +++ b/zon_config.json @@ -1,5 +1,5 @@ { - "ZON_VERSION": "1.107.868", + "ZON_VERSION": "1.108.6", "CONFIG_MAKEFLAGS": "DIST=APP RELEASE=y CC64_32=y CONFIG_LUM_LOCAL=y CONFIG_BATREQ=y CONFIG_BAT_CYCLE=y CONFIG_BAT_PLATFORM=app_win32r", - "CONFIG_BUILD_DATE": "19-Sep-18 19:59:39" + "CONFIG_BUILD_DATE": "21-Sep-18 16:49:18" } \ No newline at end of file