From f2de77d8f492578c38303752430e0a7bb2d08b24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 10 Aug 2018 11:22:37 +0200 Subject: [PATCH 001/217] Fixed `nosql.update()` and `nosql.modify()`. --- changes.txt | 4 ++++ nosql.js | 10 ++++++++-- nosqlworker.js | 8 ++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/changes.txt b/changes.txt index 346be8ae0..a02a80215 100755 --- a/changes.txt +++ b/changes.txt @@ -1,3 +1,7 @@ +======= 3.0.1 + +- fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function + ======= 3.0.0 - added: (IMPORTANT) bundles diff --git a/nosql.js b/nosql.js index 947258c97..f99c895bb 100755 --- a/nosql.js +++ b/nosql.js @@ -370,11 +370,17 @@ exports.worker = function() { }; TP.update = DP.update = function(doc, insert) { - return send(this, 'update', framework_builders.isSchema(doc) ? doc.$clean() : doc, insert).builder = new DatabaseBuilder(this); + var val = framework_builders.isSchema(doc) ? doc.$clean() : doc; + if (typeof(val) === 'function') + val = val.toString(); + return send(this, 'update', val, insert).builder = new DatabaseBuilder(this); }; TP.modify = DP.modify = function(doc, insert) { - return send(this, 'modify', framework_builders.isSchema(doc) ? doc.$clean() : doc, insert).builder = new DatabaseBuilder(this); + var val = framework_builders.isSchema(doc) ? doc.$clean() : doc; + if (typeof(val) === 'function') + val = val.toString(); + return send(this, 'modify', val, insert).builder = new DatabaseBuilder(this); }; DP.restore = function(filename, callback) { diff --git a/nosqlworker.js b/nosqlworker.js index 8ca8d67c2..1789cd98d 100755 --- a/nosqlworker.js +++ b/nosqlworker.js @@ -122,6 +122,10 @@ process.on('message', function(msg) { }); break; case 'update': + + if (typeof(msg.arg[0]) === 'string') + msg.arg[0] = eval('(' + msg.arg[0] + ')'); + db.update(msg.arg[0], msg.arg[1]).parse(msg.data).callback(function(err, response, repository) { RESUPDATE.err = err; RESUPDATE.response = response; @@ -131,6 +135,10 @@ process.on('message', function(msg) { }); break; case 'modify': + + if (typeof(msg.arg[0]) === 'string') + msg.arg[0] = eval('(' + msg.arg[0] + ')'); + db.modify(msg.arg[0], msg.arg[1]).parse(msg.data).callback(function(err, response, repository) { RESUPDATE.err = err; RESUPDATE.response = response; From f10d3358923ca48c4b9d67ac3ed937b9c48f5c5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 10 Aug 2018 11:38:41 +0200 Subject: [PATCH 002/217] Added default values for CSS variables. --- changes.txt | 1 + internal.js | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/changes.txt b/changes.txt index a02a80215..e074db6e8 100755 --- a/changes.txt +++ b/changes.txt @@ -1,5 +1,6 @@ ======= 3.0.1 +- added: CSS variables supports default values `border-radius: $radius || 10px` - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function ======= 3.0.0 diff --git a/internal.js b/internal.js index 7c30acde2..4762f30f8 100755 --- a/internal.js +++ b/internal.js @@ -76,7 +76,7 @@ const REG_CSS_7 = /\s\}/g; const REG_CSS_8 = /\s\{/g; const REG_CSS_9 = /;\}/g; const REG_CSS_10 = /\$[a-z0-9-_]+:.*?;/gi; -const REG_CSS_11 = /\$[a-z0-9-_]+/gi; +const REG_CSS_11 = /\$.*?(;|\})/gi; const REG_CSS_12 = /(margin|padding):.*?(;|})/g; const AUTOVENDOR = ['filter', 'appearance', 'column-count', 'column-gap', 'column-rule', 'display', 'transform', 'transform-style', 'transform-origin', 'transition', 'user-select', 'animation', 'perspective', 'animation-name', 'animation-duration', 'animation-timing-function', 'animation-delay', 'animation-iteration-count', 'animation-direction', 'animation-play-state', 'opacity', 'background', 'background-image', 'font-smoothing', 'text-size-adjust', 'backface-visibility', 'box-sizing', 'overflow-scrolling']; const WRITESTREAM = { flags: 'w' }; @@ -2613,8 +2613,23 @@ function variablesCSS(content) { }); content = content.replace(REG_CSS_11, function(text) { - var variable = variables[text]; - return variable ? variable : text; + + var index = text.indexOf('||'); + var variable = ''; + var last = text[text.length - 1]; + var len = text.length; + + if (last === ';' || last === '}') + len = len - 1; + else + last = ''; + + if (index !== -1) + variable = variables[text.substring(0, index).trim()] || text.substring(index + 2, len).trim(); + else + variable = variables[text.substring(0, len).trim()]; + + return variable ? (variable + last) : text; }).trim(); return content; From 6003acf92de276ab544b0e0105364d8883286295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 10 Aug 2018 17:57:27 +0200 Subject: [PATCH 003/217] Fixed typo. --- changes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index e074db6e8..da750095b 100755 --- a/changes.txt +++ b/changes.txt @@ -1,6 +1,6 @@ ======= 3.0.1 -- added: CSS variables supports default values `border-radius: $radius || 10px` +- added: CSS variables support default values `border-radius: $radius || 10px` - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function ======= 3.0.0 From 52b146dd54f83493539a916cde5200115e8478a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 10 Aug 2018 19:13:39 +0200 Subject: [PATCH 004/217] Fixed `F.wait()` in test mode. --- changes.txt | 2 ++ test.js | 18 +++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/changes.txt b/changes.txt index da750095b..a6f79c2a1 100755 --- a/changes.txt +++ b/changes.txt @@ -1,7 +1,9 @@ ======= 3.0.1 - added: CSS variables support default values `border-radius: $radius || 10px` + - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function +- fixed: `F.wait()` in the test mode ======= 3.0.0 diff --git a/test.js b/test.js index 55d4a2b3b..58e61b814 100644 --- a/test.js +++ b/test.js @@ -181,13 +181,17 @@ exports.load = function() { T.current = null; next(); }, function() { - T.tests.quicksort('priority'); - F.emit('test-begin', T); - console.log('===================== TESTING ======================'); - console.log(''); - T.running = true; - T.start = Date.now(); - NEXT(); + U.wait(function() { + return F._length_wait === 0; + }, function() { + T.tests.quicksort('priority'); + F.emit('test-begin', T); + console.log('===================== TESTING ======================'); + console.log(''); + T.running = true; + T.start = Date.now(); + NEXT(); + }); }); }); }; From 6c85f59a9b337f06be09c429ef40b18e1d66c1a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 11 Aug 2018 10:41:16 +0200 Subject: [PATCH 005/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 98e5cb010..06d9619b8 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.0", + "version": "3.0.1-1", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From a2d7286bb96cb626d79cfdd112eed3c650075e8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 15 Aug 2018 10:54:31 +0200 Subject: [PATCH 006/217] Fixed `LOCALIZE()`. --- index.js | 26 ++++++++++++++++---------- package.json | 2 +- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/index.js b/index.js index 469ff5036..2c666b859 100755 --- a/index.js +++ b/index.js @@ -21,7 +21,7 @@ /** * @module Framework - * @version 3.0.0 + * @version 3.0.1 */ 'use strict'; @@ -626,8 +626,8 @@ var PERF = {}; function Framework() { this.$id = null; // F.id ==> property - this.version = 3000; - this.version_header = '3.0.0'; + this.version = 3001; + this.version_header = '3.0.1'; this.version_node = process.version.toString(); this.syshash = (Os.hostname() + '-' + Os.platform() + '-' + Os.arch() + '-' + Os.release() + '-' + Os.tmpdir()).md5(); @@ -3223,7 +3223,7 @@ global.LOCALIZE = F.localize = function(url, flags, minify) { if (url[0] === '#') url = sitemapurl(url.substring(1)); - url = url.replace('*', ''); + url = url.replace('*.*', ''); if (minify == null) minify = true; @@ -3235,24 +3235,30 @@ global.LOCALIZE = F.localize = function(url, flags, minify) { flags = []; var index; + var ext = false; flags = flags.remove(function(item) { item = item.toLowerCase(); if (item === 'nocompress') minify = false; + if (item[0] === '.') + ext = true; return item === 'compress' || item === 'nocompress' || item === 'minify'; }); var index = url.lastIndexOf('.'); - if (index === -1) - flags.push('.html', '.htm', '.md', '.txt'); - else { - flags.push(url.substring(index).toLowerCase()); - url = url.substring(0, index); + if (!ext) { + if (index === -1) + flags.push('.html', '.htm', '.md', '.txt'); + else { + flags.push(url.substring(index).toLowerCase()); + url = url.substring(0, index).replace('*', ''); + } } - url = framework_internal.preparePath(url); + url = framework_internal.preparePath(url.replace('.*', '')); + F.file(url, function(req, res) { F.onLocale && (req.$language = F.onLocale(req, res, req.isStaticFile)); diff --git a/package.json b/package.json index 06d9619b8..e819243bd 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-1", + "version": "3.0.1-2", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From cc2d5a28a5bf2ca4636fa7f5df728b334f9fadfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 15 Aug 2018 10:55:45 +0200 Subject: [PATCH 007/217] Added new fix. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index a6f79c2a1..c74d4d61e 100755 --- a/changes.txt +++ b/changes.txt @@ -4,6 +4,7 @@ - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function - fixed: `F.wait()` in the test mode +- fixed: `LOCALIZE()` for nested directories ======= 3.0.0 From 129b5f6a8f8757bbf18b5fd02c4b13e5b41cc7ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 16 Aug 2018 08:23:29 +0200 Subject: [PATCH 008/217] Added a new test script. --- test/sql.js | 192 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 test/sql.js diff --git a/test/sql.js b/test/sql.js new file mode 100644 index 000000000..05fd36741 --- /dev/null +++ b/test/sql.js @@ -0,0 +1,192 @@ +require('../index'); + +function SQL(query) { + + var self = this; + self.options = {}; + self.builder = new framework_nosql.DatabaseBuilder(null); + self.query = query; + + var type = self.parseType(); + switch (type) { + case 'select': + self.parseLimit(); + self.parseOrder(); + self.parseWhere(); + self.parseJoins(); + self.parseTable(); + self.parseNames(); + break; + case 'update': + self.parseWhere(); + self.parseTable(); + self.parseUpdate(); + break; + case 'insert': + self.parseWhere(); + self.parseInsert(); + break; + case 'delete': + self.parseWhere(); + self.parseTable(); + break; + } + + console.log(self.options); +} + +var SQLP = SQL.prototype; + +SQLP.parseLimit = function() { + var self = this; + var tmp = self.query.match(/take \d+ skip \d+$/i); + !tmp && (tmp = self.query.match(/skip \d+ take \d+$/i)); + !tmp && (tmp = self.query.match(/take \d+$/i)); + !tmp && (tmp = self.query.match(/skip \d+$/i)); + if (!tmp) + return self; + self.query = self.query.replace(tmp, '').trim(); + var arr = tmp[0].toString().toLowerCase().split(' '); + if (arr[0] === 'take') + self.options.take = +arr[1]; + else if (arr[2] === 'take') + self.options.take = +arr[3]; + if (arr[0] === 'skip') + self.options.skip = +arr[1]; + else if (arr[2] === 'skip') + self.options.skip = +arr[3]; + return self; +}; + +SQLP.parseOrder = function() { + var self = this; + var tmp = self.query.match(/order by .*?$/i); + + if (!tmp) + return self; + + self.query = self.query.replace(tmp, '').trim(); + var arr = tmp[0].toString().substring(9).split(','); + + self.options.sort = []; + + for (var i = 0; i < arr.length; i++) { + tmp = arr[i].trim().split(' '); + self.options.sort.push({ name: tmp[0], desc: (tmp[1] || '').toLowerCase() === 'desc' }); + } + + return self; +}; + +SQLP.parseWhere = function() { + var self = this; + var tmp = self.query.match(/where .*?$/i); + if (!tmp) + return self; + self.query = self.query.replace(tmp, ''); + + tmp = tmp[0].toString().substring(6).replace(/\sAND\s/gi, ' && ').replace(/\sOR\s/gi, ' || ').replace(/[a-z0-9]=/gi, function(text) { + return text + '='; + }); + + self.options.where = tmp; + return self; +}; + +SQLP.parseJoins = function() { + var self = this; + var tmp = self.query.match(/left join.*?$/i); + if (!tmp) { + tmp = self.query.match(/join.*?$/i); + if (!tmp) + return self; + } + self.query = self.query.replace(tmp, ''); + tmp = tmp[0].toString().trim(); + // console.log(tmp); + return self; +}; + +SQLP.parseTable = function() { + var self = this; + var tmp = self.query.match(/from\s.*?$/i); + if (!tmp) + return self; + self.query = self.query.replace(tmp, ''); + tmp = tmp[0].toString().substring(5).trim(); + + var arr = tmp.split(' '); + // console.log(arr); + + return self; +}; + +SQLP.parseNames = function() { + var self = this; + var tmp = self.query.match(/select\s.*?$/i); + if (!tmp) + return self; + + self.query = self.query.replace(tmp, ''); + tmp = tmp[0].toString().substring(6).trim().split(','); + + self.options.fields = []; + + for (var i = 0; i < tmp.length; i++) { + var field = tmp[i].trim(); + var alias = field.match(/as\s.*?$/); + var name = ''; + var type = 0; + + if (alias) { + field = field.replace(alias, ''); + alias = alias.toString().substring(3); + } + + var index = field.indexOf('('); + if (index !== -1) { + switch (field.substring(0, index).toLowerCase()) { + case 'count': + type = 1; + break; + case 'min': + type = 2; + break; + case 'max': + type = 3; + break; + case 'avg': + type = 4; + break; + case 'sum': + type = 5; + break; + case 'distinct': + type = 6; + break; + } + name = field.substring(index + 1, field.lastIndexOf(')')); + } else + name = field; + self.options.fields.push({ alias: alias || name, name: name, type: type }); + } + + return self; +}; + +SQLP.parseUpdate = function() { + var self = this; + return self; +}; + +SQLP.parseInsert = function() { + var self = this; + return self; +}; + +SQLP.parseType = function() { + var self = this; + return self.query.substring(0, self.query.indexOf(' ')).toLowerCase(); +}; + +var sql = new SQL('SELECT COUNT(*) as count, id FROM table a JOIN users ON id=id WHERE a.name="Peter" AND a.age=30 ORDER BY a.name, a.age ASC TAKE 20 SKIP 10'); \ No newline at end of file From d90b8c8620f20c9afc85c552c25619fb542a03b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 20 Aug 2018 08:45:57 +0200 Subject: [PATCH 009/217] Added a default `agent` for the requests. --- utils.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils.js b/utils.js index 623a2f388..cd566f4c6 100755 --- a/utils.js +++ b/utils.js @@ -879,7 +879,7 @@ function request_response(res, uri, options) { var tmp = Url.parse(loc); tmp.headers = uri.headers; - tmp.agent = false; + // tmp.agent = false; tmp.method = uri.method; res.req.removeAllListeners(); @@ -1147,7 +1147,7 @@ exports.download = function(url, flags, data, callback, cookies, headers, encodi var uri = Url.parse(url); uri.method = method; - uri.agent = false; + // uri.agent = false; uri.headers = headers; options.uri = uri; @@ -1257,7 +1257,7 @@ function download_response(res, uri, options) { var tmp = Url.parse(res.headers['location']); tmp.headers = uri.headers; - tmp.agent = false; + // tmp.agent = false; tmp.method = uri.method; res.req.removeAllListeners(); res.req = null; From 0687ceca7cf006e6d749500796ec2eba2dc95f91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 21 Aug 2018 12:34:44 +0200 Subject: [PATCH 010/217] Added SQL parser. --- test/test-tmp.js | 199 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) diff --git a/test/test-tmp.js b/test/test-tmp.js index 25f47af6a..b03fe94bc 100644 --- a/test/test-tmp.js +++ b/test/test-tmp.js @@ -1,2 +1,201 @@ require('../index'); +function SQL(query) { + + var self = this; + self.options = {}; + self.builder = new framework_nosql.DatabaseBuilder(null); + self.query = query; + + var type = self.parseType(); + + switch (type) { + case 'select': + self.parseLimit(); + self.parseOrder(); + self.parseWhere(); + self.parseJoins(); + self.parseTable(); + self.parseNames(); + break; + case 'update': + self.parseWhere(); + self.parseTable(); + self.parseUpdate(); + break; + case 'insert': + self.parseWhere(); + self.parseInsert(); + break; + case 'delete': + self.parseWhere(); + self.parseTable(); + break; + } + + console.log(self.options); +} + +var SQLP = SQL.prototype; + +SQLP.parseLimit = function() { + var self = this; + var tmp = self.query.match(/take \d+ skip \d+$/i); + !tmp && (tmp = self.query.match(/skip \d+ take \d+$/i)); + !tmp && (tmp = self.query.match(/take \d+$/i)); + !tmp && (tmp = self.query.match(/skip \d+$/i)); + if (!tmp) + return self; + self.query = self.query.replace(tmp, '').trim(); + var arr = tmp[0].toString().toLowerCase().split(' '); + if (arr[0] === 'take') + self.options.take = +arr[1]; + else if (arr[2] === 'take') + self.options.take = +arr[3]; + if (arr[0] === 'skip') + self.options.skip = +arr[1]; + else if (arr[2] === 'skip') + self.options.skip = +arr[3]; + return self; +}; + +SQLP.parseOrder = function() { + var self = this; + var tmp = self.query.match(/order by .*?$/i); + + if (!tmp) + return self; + + self.query = self.query.replace(tmp, '').trim(); + var arr = tmp[0].toString().substring(9).split(','); + + self.options.sort = []; + + for (var i = 0; i < arr.length; i++) { + tmp = arr[i].trim().split(' '); + self.options.sort.push({ name: tmp[0], desc: (tmp[1] || '').toLowerCase() === 'desc' }); + } + + return self; +}; + +SQLP.parseWhere = function() { + var self = this; + var tmp = self.query.match(/where .*?$/i); + if (!tmp) + return self; + self.query = self.query.replace(tmp, ''); + + tmp = tmp[0].toString().substring(6).replace(/\sAND\s/gi, ' && ').replace(/\sOR\s/gi, ' || ').replace(/[a-z0-9]=/gi, function(text) { + return text + '='; + }); + + self.options.where = tmp; + return self; +}; + +SQLP.parseJoins = function() { + var self = this; + var tmp = self.query.match(/left join.*?$/i); + if (!tmp) { + tmp = self.query.match(/join.*?$/i); + if (!tmp) + return self; + } + self.query = self.query.replace(tmp, ''); + tmp = tmp[0].toString().trim(); + + self.options.joins = []; + + tmp = tmp.substring(5).split(/\s?JOIN\s/i); + for (var i = 0; i < tmp.length; i++) { + var join = tmp[i].split(/\son\s/i); + self.options.joins.push({ table: join[0], condition: join[1] }); + } + + return self; +}; + +SQLP.parseTable = function() { + var self = this; + var tmp = self.query.match(/from\s.*?$/i); + if (!tmp) + return self; + self.query = self.query.replace(tmp, ''); + tmp = tmp[0].toString().substring(5).trim(); + + var arr = tmp.split(' '); + // console.log(arr); + + return self; +}; + +SQLP.parseNames = function() { + var self = this; + var tmp = self.query.match(/select\s.*?$/i); + if (!tmp) + return self; + + self.query = self.query.replace(tmp, ''); + tmp = tmp[0].toString().substring(6).trim().split(','); + + self.options.fields = []; + + for (var i = 0; i < tmp.length; i++) { + var field = tmp[i].trim(); + var alias = field.match(/as\s.*?$/); + var name = ''; + var type = 0; + + if (alias) { + field = field.replace(alias, ''); + alias = alias.toString().substring(3); + } + + var index = field.indexOf('('); + if (index !== -1) { + switch (field.substring(0, index).toLowerCase()) { + case 'count': + type = 1; + break; + case 'min': + type = 2; + break; + case 'max': + type = 3; + break; + case 'avg': + type = 4; + break; + case 'sum': + type = 5; + break; + case 'distinct': + type = 6; + break; + } + name = field.substring(index + 1, field.lastIndexOf(')')); + } else + name = field; + self.options.fields.push({ alias: alias || name, name: name, type: type }); + } + + return self; +}; + +SQLP.parseUpdate = function() { + var self = this; + return self; +}; + +SQLP.parseInsert = function() { + var self = this; + return self; +}; + +SQLP.parseType = function() { + var self = this; + return self.query.substring(0, self.query.indexOf(' ')).toLowerCase(); +}; + +var sql = new SQL('SELECT COUNT(*) as count, id FROM table a JOIN users ON users.id=a.id JOIN orders ON orders.id=a.id WHERE a.name="Peter" AND a.age=30 ORDER BY a.name, a.age ASC TAKE 20 SKIP 10'); \ No newline at end of file From 95178316c224e74e1acb6aa7f4c4271e741a80cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 21 Aug 2018 15:07:20 +0200 Subject: [PATCH 011/217] Fixed storing files. --- changes.txt | 1 + nosql.js | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/changes.txt b/changes.txt index c74d4d61e..813b5e95f 100755 --- a/changes.txt +++ b/changes.txt @@ -2,6 +2,7 @@ - added: CSS variables support default values `border-radius: $radius || 10px` +- fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function - fixed: `F.wait()` in the test mode - fixed: `LOCALIZE()` for nested directories diff --git a/nosql.js b/nosql.js index f99c895bb..dbd521169 100755 --- a/nosql.js +++ b/nosql.js @@ -4738,7 +4738,7 @@ Binary.prototype.insert = function(name, buffer, custom, callback) { if (typeof(buffer) === 'string') buffer = framework_utils.createBuffer(buffer, 'base64'); else if (buffer.resume) - return self.insertstream(null, name, type, buffer, callback); + return self.insertstream(null, name, type, buffer, callback, custom); var size = buffer.length; var dimension; @@ -4898,7 +4898,7 @@ Binary.prototype.insertstream = function(id, name, type, stream, callback, custo h.size = writer.bytesWritten; - Fs.open(filepath, 'a', function(err, fd) { + Fs.open(filepath, 'r+', function(err, fd) { if (!err) { var header = framework_utils.createBufferSize(BINARY_HEADER_LENGTH); header.fill(' '); @@ -4953,7 +4953,7 @@ Binary.prototype.update = function(id, name, buffer, custom, callback) { buffer = framework_utils.createBuffer(buffer, 'base64'); if (buffer.resume) - return self.insertstream(id, name, type, buffer, callback); + return self.insertstream(id, name, type, buffer, callback, custom); var isnew = false; var time = NOW.format('yyyyMMdd'); From 9ee0150f045efd2ae95c85b2cfeba8ae6ea81db1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 23 Aug 2018 16:35:30 +0200 Subject: [PATCH 012/217] Improved NoSQL filtering. --- nosql.js | 1951 +++++++++++++++--------------------------------------- 1 file changed, 539 insertions(+), 1412 deletions(-) diff --git a/nosql.js b/nosql.js index dbd521169..9a8c670ad 100755 --- a/nosql.js +++ b/nosql.js @@ -872,7 +872,8 @@ DP.update = function(doc, insert) { self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); var data = framework_builders.isSchema(doc) ? doc.$clean() : doc; - self.pending_update.push({ builder: builder, doc: data, count: 0, insert: insert === true ? data : insert }); + builder.$options.readertype = 1; + self.pending_update.push({ builder: builder, doc: data, insert: insert === true ? data : insert }); setImmediate(next_operation, self, 2); return builder; }; @@ -885,6 +886,8 @@ DP.modify = function(doc, insert) { var keys = Object.keys(data); var inc = null; + builder.$options.readertype = 1; + if (keys.length) { for (var i = 0; i < keys.length; i++) { var key = keys[i]; @@ -902,7 +905,7 @@ DP.modify = function(doc, insert) { break; } } - self.pending_update.push({ builder: builder, doc: data, count: 0, keys: keys, inc: inc, insert: insert === true ? data : insert }); + self.pending_update.push({ builder: builder, doc: data, keys: keys, inc: inc, insert: insert === true ? data : insert }); setImmediate(next_operation, self, 2); } @@ -1095,16 +1098,11 @@ TP.lock = DP.lock = function(callback) { return self; }; -DP.remove = function(filename) { +DP.remove = function() { var self = this; self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); - var backup = filename === undefined ? undefined : filename || self.filenameBackup2; - - if (backup) - backup = new Backuper(backup); - - self.pending_remove.push({ builder: builder, count: 0, backup: backup }); + self.pending_remove.push(builder); setImmediate(next_operation, self, 3); return builder; }; @@ -1117,7 +1115,7 @@ DP.listing = function(builder) { builder = new DatabaseBuilder(self); builder.$options.listing = true; builder.$take = builder.$options.take = 100; - self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -1128,7 +1126,7 @@ DP.find = function(builder) { builder.db = self; else builder = new DatabaseBuilder(self); - self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -1141,7 +1139,7 @@ DP.find2 = function(builder) { builder = new DatabaseBuilder(self); if (self.readonly) return self.find(builder); - self.pending_reader2.push({ builder: builder, count: 0, counter: 0 }); + self.pending_reader2.push(builder); setImmediate(next_operation, self, 11); return builder; }; @@ -1170,7 +1168,8 @@ DP.scalar = function(type, field) { DP.count = function() { var self = this; var builder = new DatabaseBuilder(self); - self.pending_reader.push({ builder: builder, count: 0, type: 1 }); + builder.$options.readertype = 1; + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -1179,7 +1178,7 @@ DP.one = function() { var self = this; var builder = new DatabaseBuilder(self); builder.first(); - self.pending_reader.push({ builder: builder, count: 0 }); + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -1188,7 +1187,7 @@ DP.one2 = function() { var self = this; var builder = new DatabaseBuilder(self); builder.first(); - self.pending_reader2.push({ builder: builder, count: 0 }); + self.pending_reader2.push(builder); setImmediate(next_operation, self, 11); return builder; }; @@ -1197,7 +1196,7 @@ DP.top = function(max) { var self = this; var builder = new DatabaseBuilder(self); builder.take(max); - self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -1468,21 +1467,12 @@ DP.$update = function() { self.$writting = true; var filter = self.pending_update.splice(0); - var length = filter.length; - var backup = false; - var filters = 0; + var filters = new NoSQLReader(); + var fs = new NoSQLStream(self.filename); var change = false; - for (var i = 0; i < length; i++) { - var fil = filter[i]; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - if (fil.backup || fil.builder.$options.backup) - backup = true; - } - - var indexer = 0; - var fs = new NoSQLStream(self.filename); + for (var i = 0; i < filter.length; i++) + filters.add(filter[i].builder, true); if (self.buffersize) fs.buffersize = self.buffersize; @@ -1490,126 +1480,84 @@ DP.$update = function() { if (self.buffercount) fs.buffercount = self.buffercount; - fs.ondocuments = function() { - - var docs = JSON.parse('[' + fs.docs + ']', jsonparser); - - for (var a = 0; a < docs.length; a++) { - - indexer++; - - var doc = docs[a]; - var is = false; - var rec = fs.docsbuffer[a]; - - for (var i = 0; i < length; i++) { - - var item = filter[i]; - if (item.skip) - continue; - - item.filter.index = indexer; - - var output = item.compare(doc, item.filter, indexer); - if (output) { - - var e = item.keys ? 'modify' : 'update'; - var old = self.$events[e] ? CLONE(output) : 0; - - if (item.filter.options.first) { - item.skip = true; - filters++; - } - - if (item.keys) { - for (var j = 0; j < item.keys.length; j++) { - var key = item.keys[j]; - var val = item.doc[key]; - if (val !== undefined) { - if (typeof(val) === 'function') - output[key] = val(output[key], output); - else if (item.inc && item.inc[key]) { - switch (item.inc[key]) { - case '+': - output[key] = (output[key] || 0) + val; - break; - case '-': - output[key] = (output[key] || 0) - val; - break; - case '*': - output[key] = (output[key] || 0) + val; - break; - case '/': - output[key] = (output[key] || 0) / val; - break; - } - } else - output[key] = val; - } + var update = function(docs, doc, dindex, f, findex) { + + var rec = fs.docsbuffer[dindex]; + var fil = filter[findex]; + var e = fil.keys ? 'modify' : 'update'; + var old = self.$events[e] ? CLONE(doc) : 0; + + if (f.first) + f.canceled = true; + + if (fil.keys) { + for (var j = 0; j < fil.keys.length; j++) { + var key = fil.keys[j]; + var val = fil.doc[key]; + if (val !== undefined) { + if (typeof(val) === 'function') + doc[key] = val(doc[key], doc); + else if (fil.inc && fil.inc[key]) { + switch (fil.inc[key]) { + case '+': + doc[key] = (doc[key] || 0) + val; + break; + case '-': + doc[key] = (doc[key] || 0) - val; + break; + case '*': + doc[key] = (doc[key] || 0) + val; + break; + case '/': + doc[key] = (doc[key] || 0) / val; + break; } } else - output = typeof(item.doc) === 'function' ? item.doc(output) : item.doc; - - self.$events[e] && self.emit(e, output, old); - item.count++; - doc = output; - is = true; + doc[key] = val; } } + } else + docs[dindex] = typeof(f.doc) === 'function' ? fil.doc(doc) : fil.doc; - if (is) { + self.$events[e] && self.emit(e, doc, old); + f.builder.$options.backup && f.builder.$backupdoc(rec.doc); + }; - if (backup) { - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.backup && item.backup.write(rec.doc + NEWLINE); - item.builder.$options.backup && item.builder.$backupdoc(rec.doc); - } - } + var updateflush = function(docs, doc, dindex) { - var upd = JSON.stringify(doc).replace(REGBOOL, JSONBOOL); - if (upd === rec.doc) - continue; + doc = docs[dindex]; - var was = true; + var rec = fs.docsbuffer[dindex]; + var upd = JSON.stringify(doc).replace(REGBOOL, JSONBOOL); + if (upd === rec.doc) + return; - if (!change) - change = true; + !change && (change = true); - if (rec.doc.length === upd.length) { - var b = Buffer.byteLength(upd); - if (rec.length === b) { - fs.write(upd + NEWLINE, rec.position); - was = false; - } - } + var was = true; + !change && (change = true); - if (was) { - var tmp = '-' + rec.doc.substring(1) + NEWLINE; - fs.write(tmp, rec.position); - fs.write2(upd + NEWLINE); - } + if (rec.doc.length === upd.length) { + var b = Buffer.byteLength(upd); + if (rec.length === b) { + fs.write(upd + NEWLINE, rec.position); + was = false; } + } - if (filters === length) - return false; + if (was) { + var tmp = fs.remchar + rec.doc.substring(1) + NEWLINE; + fs.write(tmp, rec.position); + fs.write2(upd + NEWLINE); } }; - fs.$callback = function() { - for (var i = 0; i < length; i++) { - var item = filter[i]; - if (item.insert && !item.count) { - item.builder.$insertcallback && item.builder.$insertcallback(item.insert, item.builder.$repository || EMPTYOBJECT); - var tmp = self.insert(item.insert); - tmp.$callback = item.builder.$callback; - tmp.$options.log = item.builder.$options.log; - } else { - item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); - } - } + fs.ondocuments = function() { + filters.compare2(JSON.parse('[' + fs.docs + ']', jsonparser), update, updateflush); + }; + fs.$callback = function() { + filters.done(); fs = null; self.$writting = false; self.next(0); @@ -1631,74 +1579,70 @@ DP.$update_inmemory = function() { } var filter = self.pending_update.splice(0); - var length = filter.length; var change = false; + var filters = new NoSQLReader(); - for (var i = 0; i < length; i++) { - var fil = filter[i]; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - } + for (var i = 0; i < filter.length; i++) + filters.add(filter[i].builder, true); return self.$inmemory(function() { - var data = self.inmemory['#']; - - for (var a = 0, al = data.length; a < al; a++) { - - var doc = data[a]; - - for (var i = 0; i < length; i++) { - - var item = filter[i]; - var builder = item.builder; - item.filter.index = j; - var output = item.compare(doc, item.filter, j); - if (output) { - - var e = item.keys ? 'modify' : 'update'; - var old = self.$events[e] ? CLONE(output) : 0; - - builder.$options.backup && builder.$backupdoc(doc); - - if (item.keys) { - for (var j = 0, jl = item.keys.length; j < jl; j++) { - var val = item.doc[item.keys[j]]; - if (val !== undefined) { - if (typeof(val) === 'function') - doc[item.keys[j]] = val(doc[item.keys[j]], doc); - else - doc[item.keys[j]] = val; + var old; + + var update = function(docs, doc, dindex, f, findex) { + + var fil = filter[findex]; + var e = fil.keys ? 'modify' : 'update'; + + if (!old) + old = self.$events[e] ? CLONE(doc) : 0; + + if (f.first) + f.canceled = true; + + if (fil.keys) { + for (var j = 0; j < fil.keys.length; j++) { + var key = fil.keys[j]; + var val = fil.doc[key]; + if (val !== undefined) { + if (typeof(val) === 'function') + doc[key] = val(doc[key], doc); + else if (fil.inc && fil.inc[key]) { + switch (fil.inc[key]) { + case '+': + doc[key] = (doc[key] || 0) + val; + break; + case '-': + doc[key] = (doc[key] || 0) - val; + break; + case '*': + doc[key] = (doc[key] || 0) + val; + break; + case '/': + doc[key] = (doc[key] || 0) / val; + break; } - } - } else - doc = typeof(item.doc) === 'function' ? item.doc(doc) : item.doc; - - self.$events[e] && self.emit(e, doc, old); - item.count++; - if (!change) - change = true; + } else + doc[key] = val; + } } - } - } + } else + docs[dindex] = typeof(f.doc) === 'function' ? fil.doc(doc) : fil.doc; - self.$save(); + self.$events[e] && self.emit(e, doc, old); + f.builder.$options.backup && f.builder.$backupdoc(old); + return 1; + }; - for (var i = 0; i < length; i++) { - var item = filter[i]; - if (item.insert && !item.count) { - item.builder.$insertcallback && item.builder.$insertcallback(item.insert, item.builder.$repository || EMPTYOBJECT); - var tmp = self.insert(item.insert); - tmp.$callback = item.builder.$callback; - tmp.$options.log = item.builder.$options.log; - } else { - var e = item.keys ? 'modify' : 'update'; - item.count && self.$events[e] && self.emit(e, item.doc); - item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); - } - } + var updateflush = function(docs, doc) { + !change && (change = true); + self.$events.update && self.emit('update', doc, old); + old = null; + }; + filters.compare2(self.inmemory['#'], update, updateflush); + + change && self.$save(); setImmediate(function() { self.next(0); change && self.$events.change && self.emit('change', 'update'); @@ -1750,24 +1694,8 @@ DP.$reader2 = function(filename, items, callback, reader) { } } - var filter = items; - var length = filter.length; - var first = true; - var indexer = 0; - - for (var i = 0; i < length; i++) { - var fil = filter[i]; - if (!fil.builder.$options.first || fil.builder.$options.sort) - first = false; - fil.scalarcount = 0; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - } - - if (first && length > 1) - first = false; - var fs = new NoSQLStream(self.filename); + var filters = new NoSQLReader(items); if (self.buffersize) fs.buffersize = self.buffersize; @@ -1776,144 +1704,11 @@ DP.$reader2 = function(filename, items, callback, reader) { fs.buffercount = self.buffercount; fs.ondocuments = function() { - - var docs = JSON.parse('[' + fs.docs + ']', jsonparser); - var val; - - for (var j = 0; j < docs.length; j++) { - var json = docs[j]; - indexer++; - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - item.filter.index = indexer; - var output = item.compare(json, item.filter, indexer); - if (!output) - continue; - - item.count++; - - if (!builder.$inlinesort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) - continue; - - item.counter++; - - if (item.type) - continue; - - switch (builder.$options.scalar) { - case 'count': - item.scalar = item.scalar ? item.scalar + 1 : 1; - break; - case 'sum': - val = output[builder.$options.scalarfield] || 0; - item.scalar = item.scalar ? item.scalar + val : val; - break; - case 'min': - val = output[builder.$options.scalarfield] || 0; - if (val != null) { - if (item.scalar) { - if (item.scalar > val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'max': - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar) { - if (item.scalar < val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'avg': - val = output[builder.$options.scalarfield]; - if (val != null) { - item.scalar = item.scalar ? item.scalar + val : val; - item.scalarcount++; - } - break; - case 'group': - !item.scalar && (item.scalar = {}); - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar[val]) - item.scalar[val]++; - else - item.scalar[val] = 1; - } - break; - default: - if (builder.$inlinesort) - nosqlinlinesorter(item, builder, output); - else if (item.response) - item.response.push(output); - else - item.response = [output]; - break; - } - - if (first) - return false; - } - } + return filters.compare(JSON.parse('[' + fs.docs + ']', jsonparser)); }; fs.$callback = function() { - - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - var output; - - if (builder.$options.scalar || !builder.$options.sort) { - - if (builder.$options.scalar) - output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; - else if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - continue; - } - - if (item.count) { - if (builder.$options.sort.name) { - if (!builder.$inlinesort || builder.$options.take !== item.response.length) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - } else if (builder.$options.sort === null) - item.response.random(); - else - item.response.sort(builder.$options.sort); - - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (!builder.$inlinesort && builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); - } - - if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else { - if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - } - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - builder.done(); - } - + filters.done(); fs = null; callback(); }; @@ -1929,7 +1724,6 @@ DP.$reader2 = function(filename, items, callback, reader) { DP.$reader3 = function() { var self = this; - self.step = 11; if (!self.pending_reader2.length) { @@ -1939,24 +1733,8 @@ DP.$reader3 = function() { self.$reading = true; - var filter = self.pending_reader2.splice(0); - var length = filter.length; - var first = true; - var indexer = 0; - - for (var i = 0; i < length; i++) { - var fil = filter[i]; - if (!fil.builder.$options.first) - first = false; - fil.scalarcount = 0; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - } - - if (first && length > 1) - first = false; - var fs = new NoSQLStream(self.filename); + var filters = new NoSQLReader(self.pending_reader2.splice(0)); if (self.buffersize) fs.buffersize = self.buffersize; @@ -1965,159 +1743,11 @@ DP.$reader3 = function() { fs.buffercount = self.buffercount; fs.ondocuments = function() { - - var docs = JSON.parse('[' + fs.docs + ']', jsonparser); - var val; - - for (var j = 0; j < docs.length; j++) { - var json = docs[j]; - indexer++; - - var done = true; - - for (var i = 0; i < length; i++) { - - var item = filter[i]; - - if (item.done) - continue; - else if (done) - done = false; - - var builder = item.builder; - item.filter.index = indexer; - - var output = item.compare(json, item.filter, indexer); - if (!output) - continue; - - item.count++; - - if (!builder.$inlinesort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) - continue; - - item.counter++; - - if (!builder.$inlinesort && !item.done) - item.done = builder.$options.take && builder.$options.take <= item.counter; - - if (item.type) - continue; - - switch (builder.$options.scalar) { - case 'count': - item.scalar = item.scalar ? item.scalar + 1 : 1; - break; - case 'sum': - val = output[builder.$options.scalarfield] || 0; - item.scalar = item.scalar ? item.scalar + val : val; - break; - case 'min': - val = output[builder.$options.scalarfield] || 0; - if (val != null) { - if (item.scalar) { - if (item.scalar > val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'max': - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar) { - if (item.scalar < val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'avg': - val = output[builder.$options.scalarfield]; - if (val != null) { - item.scalar = item.scalar ? item.scalar + val : val; - item.scalarcount++; - } - break; - case 'group': - !item.scalar && (item.scalar = {}); - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar[val]) - item.scalar[val]++; - else - item.scalar[val] = 1; - } - break; - default: - if (builder.$inlinesort) - nosqlinlinesorter(item, builder, output); - else if (item.response) - item.response.push(output); - else - item.response = [output]; - break; - } - - if (first) - return false; - } - - if (done) - return false; - } + return filters.compare(JSON.parse('[' + fs.docs + ']', jsonparser)); }; fs.$callback = function() { - - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - var output; - - if (builder.$options.scalar || !builder.$options.sort) { - - if (builder.$options.scalar) - output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; - else if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - continue; - } - - if (item.count) { - if (builder.$options.sort.name) { - if (!builder.$inlinesort || builder.$options.take !== item.response.length) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - } else if (builder.$options.sort === null) - item.response.random(); - else - item.response.sort(builder.$options.sort); - - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (!builder.$inlinesort && builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); - } - - if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - builder.done(); - } - + filters.done(); self.$reading = false; fs = null; self.next(0); @@ -2230,235 +1860,57 @@ function nosqlresort(arr, builder, doc) { } DP.$reader2_inmemory = function(items, callback) { + var self = this; + var filters = new NoSQLReader(items); + return self.$inmemory(function() { + filters.compare(self.inmemory['#']); + filters.done(); + callback(); + }); +}; + +DP.$remove = function() { var self = this; - var filter = items; - var length = filter.length; - var name = '#'; + self.step = 3; - for (var i = 0; i < length; i++) { - var fil = filter[i]; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; + if (!self.pending_remove.length) { + self.next(0); + return; } - return self.$inmemory(function() { + self.$writting = true; - var data = self.inmemory[name]; - var val; - - for (var j = 0, jl = data.length; j < jl; j++) { - var json = data[j]; - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - item.filter.index = j; - var output = item.compare(U.clone(json), item.filter, j); - if (!output) - continue; + var fs = new NoSQLStream(self.filename); + var filter = self.pending_remove.splice(0); + var filters = new NoSQLReader(filter); + var change = false; - item.count++; + if (self.buffersize) + fs.buffersize = self.buffersize; - if (!builder.$options.sort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) - continue; + if (self.buffercount) + fs.buffercount = self.buffercount; - item.counter++; + var remove = function(docs, d, dindex, f) { + var rec = fs.docsbuffer[dindex]; + f.builder.$options.backup && f.builder.$backupdoc(rec.doc); + return 1; + }; - if (item.type) - continue; + var removeflush = function(docs, d, dindex) { + var rec = fs.docsbuffer[dindex]; + !change && (change = true); + self.$events.remove && self.emit('remove', d); + fs.write(fs.remchar + rec.doc.substring(1) + NEWLINE, rec.position); + }; - switch (builder.$options.scalar) { - case 'count': - item.scalar = item.scalar ? item.scalar + 1 : 1; - break; - case 'sum': - val = json[builder.$options.scalarfield] || 0; - item.scalar = item.scalar ? item.scalar + val : val; - break; - case 'min': - val = json[builder.$options.scalarfield] || 0; - if (val != null) { - if (item.scalar) { - if (item.scalar > val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'max': - val = json[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar) { - if (item.scalar < val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'avg': - val = json[builder.$options.scalarfield]; - if (val != null) { - item.scalar = item.scalar ? item.scalar + val : 0; - if (item.scalarcount) - item.scalarcount++; - else - item.scalarcount = 1; - } - break; - case 'group': - !item.scalar && (item.scalar = {}); - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar[val]) - item.scalar[val]++; - else - item.scalar[val] = 1; - } - break; - - default: - if (item.response) - item.response.push(output); - else - item.response = [output]; - break; - } - } - } - - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - var output; - - if (builder.$options.scalar || !builder.$options.sort) { - - if (builder.$options.scalar) - output = builder.$options.scalar === 'avg' ? item.scalar / item.counter : item.scalar; - else if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - continue; - } - - if (item.count) { - if (builder.$options.sort.name) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - else if (builder.$options.sort === EMPTYOBJECT) - item.response.random(); - else - item.response.sort(builder.$options.sort); - - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); - } - - if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - builder.done(); - } - - callback(); - }); -}; - -DP.$remove = function() { - - var self = this; - self.step = 3; - - if (!self.pending_remove.length) { - self.next(0); - return; - } - - self.$writting = true; - - var fs = new NoSQLStream(self.filename); - var filter = self.pending_remove.splice(0); - var length = filter.length; - var change = false; - var indexer = 0; - var backup = false; - - if (self.buffersize) - fs.buffersize = self.buffersize; - - if (self.buffercount) - fs.buffercount = self.buffercount; - - for (var i = 0; i < length; i++) { - var fil = filter[i]; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - if (fil.backup || fil.builder.$options.backup) - backup = true; - } - - fs.ondocuments = function() { - - var docs = JSON.parse('[' + fs.docs + ']', jsonparser); - - for (var j = 0; j < docs.length; j++) { - - indexer++; - var removed = false; - var doc = docs[j]; - var rec = fs.docsbuffer[j]; - - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.filter.index = indexer; - var output = item.compare(doc, item.filter, indexer); - if (output) { - removed = true; - doc = output; - break; - } - } - - if (removed) { - - if (backup) { - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.backup && item.backup.write(rec.doc + NEWLINE); - item.builder.$options.backup && item.builder.$backupdoc(rec.doc); - } - } - - if (!change) - change = true; - - item.count++; - self.$events.remove && self.emit('remove', doc); - fs.write('-' + rec.doc.substring(1) + NEWLINE, rec.position); - } - } - }; + fs.ondocuments = function() { + filters.compare2(JSON.parse('[' + fs.docs + ']', jsonparser), remove, removeflush); + }; fs.$callback = function() { - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); - } - + filters.done(); fs = null; self.$writting = false; self.next(0); @@ -2565,57 +2017,29 @@ DP.$remove_inmemory = function() { return self; } - var filter = self.pending_remove.splice(0); - var length = filter.length; var change = false; - - for (var i = 0; i < length; i++) { - var fil = filter[i]; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - } + var filters = new NoSQLReader(self.pending_remove.splice(0)); return self.$inmemory(function() { + var cache = self.inmemory['#'].slice(0); - var data = self.inmemory['#']; - var arr = []; - - for (var j = 0, jl = data.length; j < jl; j++) { - var json = data[j]; - var removed = false; - - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.filter.index = j; - if (item.compare(json, item.filter, j)) { - removed = true; - break; - } - } - - if (removed) { - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.backup && item.backup.write(JSON.stringify(json)); - item.count++; - } - change = true; - self.$events.remove && self.emit('remove', json); - } else - arr.push(json); - } - - if (change) { - self.inmemory['#'] = arr; - self.$save(); - } + var remove = function(docs, d, dindex, f) { + f.builder.$options.backup && f.builder.$backupdoc(d); + return 1; + }; - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count); - } + var removeflush = function(docs, d) { + !change && (change = true); + self.$events.remove && self.emit('remove', d); + var data = self.inmemory['#']; + var index = data.indexOf(d); + if (index !== -1) + self.inmemory['#'].splice(index, index + 1); + }; + filters.compare2(cache, remove, removeflush); + change && self.$save(); + filters.done(); self.next(0); change && self.$events.change && self.emit('change', 'remove'); }); @@ -3003,6 +2427,16 @@ DatabaseBuilder.prototype.empty = function(name) { return self; }; +DatabaseBuilder.prototype.map = function(name, code) { + var self = this; + var data = { name: name, code: code }; + if (self.$options.mappers) + self.$options.mappers.push(data); + else + self.$options.mappers = [data]; + return self; +}; + DatabaseBuilder.prototype.backup = function(user) { if (this.db.filenameBackup) this.$options.backup = typeof(user) === 'string' ? user : 'unknown'; @@ -3374,6 +2808,18 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { self.$inlinesort = !!(opt.take && opt.sort && opt.sort !== null); self.$limit = (opt.take || 0) + (opt.skip || 0); var key = opt.id ? self.db.name + '_' + opt.id : code.hash(); + + if (opt.mappers) { + var tmp = ''; + self.$mappers = []; + for (var i = 0; i < opt.mappers.length; i++) { + var m = opt.mappers[i]; + tmp += ('doc.{0}=item.builder.$mappers[\'{1}\'](doc,R);'.format(m.name, i)); + self.$mappers.push(new Function('doc', 'repository', code)); + } + self.$mappersexec = new Function('doc', 'R', tmp.join('')); + } + return CACHE[key] ? CACHE[key] : (CACHE[key] = new Function('doc', '$F', 'index', code)); }; @@ -5892,7 +5338,7 @@ TP.remove = function() { var self = this; self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); - self.pending_remove.push({ builder: builder, count: 0 }); + self.pending_remove.push(builder); setImmediate(next_operation, self, 3); return builder; }; @@ -5906,7 +5352,7 @@ TP.listing = function(builder) { builder = new DatabaseBuilder(self); builder.$options.listing = true; builder.$take = builder.$options.take = 100; - self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -5918,7 +5364,7 @@ TP.find = function(builder) { builder.db = self; else builder = new DatabaseBuilder(self); - self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -5929,7 +5375,7 @@ TP.find2 = function(builder) { builder.db = self; else builder = new DatabaseBuilder(self); - self.pending_reader2.push({ builder: builder, count: 0, counter: 0 }); + self.pending_reader2.push(builder); setImmediate(next_operation, self, 11); return builder; }; @@ -6038,7 +5484,8 @@ TP.count = function() { var self = this; self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); - self.pending_reader.push({ builder: builder, count: 0, type: 1 }); + builder.$options.readertype = 1; + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -6048,7 +5495,7 @@ TP.one = function() { self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); builder.first(); - self.pending_reader.push({ builder: builder, count: 0 }); + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -6058,7 +5505,7 @@ TP.one2 = function() { self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); builder.first(); - self.pending_reader2.push({ builder: builder, count: 0 }); + self.pending_reader2.push(builder); setImmediate(next_operation, self, 11); return builder; }; @@ -6068,7 +5515,7 @@ TP.top = function(max) { self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); builder.take(max); - self.pending_reader.push({ builder: builder, count: 0, counter: 0 }); + self.pending_reader.push(builder); setImmediate(next_operation, self, 4); return builder; }; @@ -6202,42 +5649,13 @@ TP.$reader = function() { self.$reading = true; - var filter = self.pending_reader.splice(0); - var length = filter.length; - var first = true; - var indexer = 0; - var keys = {}; - var keyscount = 0; - - for (var i = 0; i < length; i++) { - var fil = filter[i]; - if (!fil.builder.$options.first || fil.builder.$options.sort) - first = false; - if (fil.builder.$keys == null && keys) - keys = null; - else if (keys) { - if (fil.builder.$options.fields) { - for (var j = 0; j < fil.builder.$keys.length; j++) { - keyscount++; - keys[fil.builder.$keys[j]] = 1; - } - } else - keys = null; - } - - fil.scalarcount = 0; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - } - - if (first && length > 1) - first = false; - var fs = new NoSQLStream(self.filename); + var filters = new NoSQLReader(self.pending_reader.splice(0)); var data = {}; + var indexer = 0; fs.divider = '\n'; - data.keys = keys && keyscount ? Object.keys(keys) : self.$keys; + data.keys = self.$keys; if (self.buffersize) fs.buffersize = self.buffersize; @@ -6248,146 +5666,19 @@ TP.$reader = function() { fs.ondocuments = function() { var lines = fs.docs.split(fs.divider); - var val; + var arr = []; for (var j = indexer ? 0 : 1; j < lines.length; j++) { - data.line = lines[j].split('|'); data.index = indexer++; - - indexer++; - - var obj = self.parseData(data); - - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - item.filter.index = indexer; - - var output = item.compare(obj, item.filter, indexer); - if (!output) - continue; - - item.count++; - - if (!builder.$inlinesort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) - continue; - - item.counter++; - - if (item.type) - continue; - - switch (builder.$options.scalar) { - case 'count': - item.scalar = item.scalar ? item.scalar + 1 : 1; - break; - case 'sum': - val = output[builder.$options.scalarfield] || 0; - item.scalar = item.scalar ? item.scalar + val : val; - break; - case 'min': - val = output[builder.$options.scalarfield] || 0; - if (val != null) { - if (item.scalar) { - if (item.scalar > val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'max': - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar) { - if (item.scalar < val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'avg': - val = output[builder.$options.scalarfield]; - if (val != null) { - item.scalar = item.scalar ? item.scalar + val : val; - item.scalarcount++; - } - break; - case 'group': - !item.scalar && (item.scalar = {}); - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar[val]) - item.scalar[val]++; - else - item.scalar[val] = 1; - } - break; - default: - if (builder.$inlinesort) - nosqlinlinesorter(item, builder, output); - else if (item.response) - item.response.push(output); - else - item.response = [output]; - break; - } - - if (first) - return false; - } + arr.push(self.parseData(data)); } + + return filters.compare(arr, jsonparser); }; fs.$callback = function() { - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - var output; - - if (builder.$options.scalar || !builder.$options.sort) { - - if (builder.$options.scalar) - output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; - else if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - continue; - } - - if (item.count) { - if (builder.$options.sort.name) { - if (!builder.$inlinesort || builder.$options.take !== item.response.length) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - } else if (builder.$options.sort === null) - item.response.random(); - else - item.response.sort(builder.$options.sort); - - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (!builder.$inlinesort && builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); - } - - if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - builder.done(); - } - + filters.done(); fs = null; self.$reading = false; self.next(0); @@ -6410,42 +5701,13 @@ TP.$reader3 = function() { self.$reading = true; - var filter = self.pending_reader2.splice(0); - var length = filter.length; - var first = true; - var indexer = 0; - var keys = {}; - var keyscount = 0; - - for (var i = 0; i < length; i++) { - var fil = filter[i]; - if (!fil.builder.$options.first || fil.builder.$options.sort) - first = false; - if (fil.builder.$keys == null && keys) - keys = null; - else if (keys) { - if (fil.builder.$options.fields) { - for (var j = 0; j < fil.builder.$keys.length; j++) { - keyscount++; - keys[fil.builder.$keys[j]] = 1; - } - } else - keys = null; - } - - fil.scalarcount = 0; - fil.compare = fil.builder.compile(); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - } - - if (first && length > 1) - first = false; - var fs = new NoSQLStream(self.filename); + var filters = new NoSQLReader(self.pending_reader2.splice(0)); var data = {}; + var indexer = 0; fs.divider = '\n'; - data.keys = keys && keyscount ? Object.keys(keys) : self.$keys; + data.keys = self.$keys; if (self.buffersize) fs.buffersize = self.buffersize; @@ -6456,152 +5718,23 @@ TP.$reader3 = function() { fs.ondocuments = function() { var lines = fs.docs.split(fs.divider); - var val; + var arr = []; for (var j = 0; j < lines.length; j++) { - data.line = lines[j].split('|'); - if (!TABLERECORD[data.line[0]]) continue; - data.index = indexer++; - indexer++; - - var obj = self.parseData(data); - - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - item.filter.index = indexer; - - var output = item.compare(obj, item.filter, indexer); - if (!output) - continue; - - item.count++; - - if (!builder.$inlinesort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) - continue; - - item.counter++; - - if (item.type) - continue; - - switch (builder.$options.scalar) { - case 'count': - item.scalar = item.scalar ? item.scalar + 1 : 1; - break; - case 'sum': - val = output[builder.$options.scalarfield] || 0; - item.scalar = item.scalar ? item.scalar + val : val; - break; - case 'min': - val = output[builder.$options.scalarfield] || 0; - if (val != null) { - if (item.scalar) { - if (item.scalar > val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'max': - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar) { - if (item.scalar < val) - item.scalar = val; - } else - item.scalar = val; - } - break; - case 'avg': - val = output[builder.$options.scalarfield]; - if (val != null) { - item.scalar = item.scalar ? item.scalar + val : val; - item.scalarcount++; - } - break; - case 'group': - !item.scalar && (item.scalar = {}); - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar[val]) - item.scalar[val]++; - else - item.scalar[val] = 1; - } - break; - default: - if (builder.$inlinesort) - nosqlinlinesorter(item, builder, output); - else if (item.response) - item.response.push(output); - else - item.response = [output]; - break; - } - - if (first) - return false; - } + arr.push(self.parseData(data)); } + + return filters.compare(arr, jsonparser); }; fs.$callback = function() { - - for (var i = 0; i < length; i++) { - var item = filter[i]; - var builder = item.builder; - var output; - - if (builder.$options.scalar || !builder.$options.sort) { - - if (builder.$options.scalar) - output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; - else if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - continue; - } - - if (item.count) { - if (builder.$options.sort.name) { - if (!builder.$inlinesort || builder.$options.take !== item.response.length) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - } else if (builder.$options.sort === null) - item.response.random(); - else - item.response.sort(builder.$options.sort); - - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (!builder.$inlinesort && builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); - } - - if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(null, builder, output), item.type === 1 ? item.count : output, item.count); - builder.done(); - } - - self.$reading = false; + filters.done(); fs = null; + self.$reading = false; self.next(0); }; @@ -6621,174 +5754,115 @@ TP.$update = function() { self.$writting = true; + var fs = new NoSQLStream(self.filename); var filter = self.pending_update.splice(0); - var length = filter.length; - var backup = false; - var filters = 0; + var filters = new NoSQLReader(); var change = false; - var keys = {}; - var keyscount = 0; - - for (var i = 0; i < length; i++) { - var fil = filter[i]; - fil.compare = fil.builder.compile(true); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - - if (fil.backup || fil.builder.$options.backup) - backup = true; - - if (fil.builder.$keys == null) - keys = null; - else { - for (var j = 0; j < fil.builder.$keys.length; j++) { - keyscount++; - keys[fil.builder.$keys[j]] = 1; - } - } + var indexer = 0; + var data = { keys: self.$keys }; - } + for (var i = 0; i < filter.length; i++) + filters.add(filter[i].builder, true); - var indexer = 0; - var fs = new NoSQLStream(self.filename); fs.divider = '\n'; - var data = {}; - data.keys = keys && keyscount ? Object.keys(keys) : self.$keys; - if (self.buffersize) fs.buffersize = self.buffersize; if (self.buffercount) fs.buffercount = self.buffercount; - fs.ondocuments = function() { - - var lines = fs.docs.split(fs.divider); - var val; + var update = function(docs, doc, dindex, f, findex) { - for (var a = indexer ? 0 : 1; a < lines.length; a++) { + var rec = fs.docsbuffer[dindex]; - data.line = lines[a].split('|'); - data.length = lines[a].length; - data.index = indexer++; + var fil = filter[findex]; + var e = fil.keys ? 'modify' : 'update'; + var old = self.$events[e] ? CLONE(doc) : 0; - var is = false; - var rec = fs.docsbuffer[a]; - var doc = self.parseData(data, data.keys === self.$keys ? EMPTYOBJECT : null); + if (f.first) + f.canceled = true; - for (var i = 0; i < length; i++) { - - var item = filter[i]; - if (item.skip) - continue; - - item.filter.index = indexer; - - var output = item.compare(doc, item.filter, indexer); - if (output) { - - if (item.filter.options.first) { - item.skip = true; - filters++; - } - - if (data.keys !== self.$keys) { - var tmp = data.keys; - data.keys = self.$keys; - output = self.parseData(data, output); - data.keys = tmp; - } - - var e = item.keys ? 'modify' : 'update'; - var old = self.$events[e] ? CLONE(output) : 0; - - if (item.keys) { - for (var j = 0; j < item.keys.length; j++) { - var key = item.keys[j]; - var val = item.doc[key]; - if (val !== undefined) { - if (typeof(val) === 'function') - output[key] = val(output[key], output); - else if (item.inc && item.inc[key]) { - switch (item.inc[key]) { - case '+': - output[key] = (output[key] || 0) + val; - break; - case '-': - output[key] = (output[key] || 0) - val; - break; - case '*': - output[key] = (output[key] || 0) + val; - break; - case '/': - output[key] = (output[key] || 0) / val; - break; - } - } else - output[key] = val; - } + if (fil.keys) { + for (var j = 0; j < fil.keys.length; j++) { + var key = fil.keys[j]; + var val = fil.doc[key]; + if (val !== undefined) { + if (typeof(val) === 'function') + doc[key] = val(doc[key], doc); + else if (fil.inc && fil.inc[key]) { + switch (fil.inc[key]) { + case '+': + doc[key] = (doc[key] || 0) + val; + break; + case '-': + doc[key] = (doc[key] || 0) - val; + break; + case '*': + doc[key] = (doc[key] || 0) + val; + break; + case '/': + doc[key] = (doc[key] || 0) / val; + break; } } else - output = typeof(item.doc) === 'function' ? item.doc(output) : item.doc; - - self.$events[e] && self.emit(e, output, old); - item.count++; - doc = output; - is = true; + doc[key] = val; } } + } else + docs[dindex] = typeof(f.doc) === 'function' ? fil.doc(doc) : fil.doc; - if (is) { + self.$events[e] && self.emit(e, doc, old); + f.builder.$options.backup && f.builder.$backupdoc(rec.doc); + }; - if (backup) { - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.backup && item.backup.write(rec.doc + NEWLINE); - item.builder.$options.backup && item.builder.$backupdoc(rec.doc); - } - } + var updateflush = function(docs, doc, dindex) { - var upd = self.stringify(doc, null, rec.length); - if (upd === rec.doc) - continue; + doc = docs[dindex]; - if (!change) - change = true; + var rec = fs.docsbuffer[dindex]; + var upd = self.stringify(doc, null, rec.length); - var b = Buffer.byteLength(upd); - if (rec.length === b) { - fs.write(upd + NEWLINE, rec.position); - } else { - var tmp = fs.remchar + rec.doc.substring(1) + NEWLINE; - fs.write(tmp, rec.position); - fs.write2(upd + NEWLINE); - } - } + if (upd === rec.doc) + return; + + !change && (change = true); - if (filters === length) - return false; + var b = Buffer.byteLength(upd); + if (rec.length === b) { + fs.write(upd + NEWLINE, rec.position); + } else { + var tmp = fs.remchar + rec.doc.substring(1) + NEWLINE; + fs.write(tmp, rec.position); + fs.write2(upd + NEWLINE); } + }; - fs.$callback = function() { + fs.ondocuments = function() { - for (var i = 0; i < length; i++) { - var item = filter[i]; - if (item.insert && !item.count) { - item.builder.$insertcallback && item.builder.$insertcallback(item.insert, item.builder.$repository || EMPTYOBJECT); - var tmp = self.insert(item.insert); - tmp.$callback = item.builder.$callback; - tmp.$options.log = item.builder.$options.log; - } else { - item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); - } + var lines = fs.docs.split(fs.divider); + var arr = []; + + if (!indexer) + arr.push(EMPTYOBJECT); + + for (var a = indexer ? 0 : 1; a < lines.length; a++) { + data.line = lines[a].split('|'); + data.length = lines[a].length; + data.index = indexer++; + arr.push(self.parseData(data, EMPTYOBJECT)); } + filters.compare2(arr, update, updateflush); + }; + + + fs.$callback = function() { + filters.done(); fs = null; self.$writting = false; self.next(0); - change && self.$events.change && self.emit('change', 'update'); }; @@ -6810,12 +5884,9 @@ TP.$remove = function() { var fs = new NoSQLStream(self.filename); var filter = self.pending_remove.splice(0); - var length = filter.length; + var filters = new NoSQLReader(filter); var change = false; var indexer = 0; - var backup = false; - var keys = {}; - var keyscount = 0; fs.divider = '\n'; @@ -6825,78 +5896,37 @@ TP.$remove = function() { if (self.buffercount) fs.buffercount = self.buffercount; - for (var i = 0; i < length; i++) { - var fil = filter[i]; - fil.compare = fil.builder.compile(true); - fil.filter = { repository: fil.builder.$repository, options: fil.builder.$options, arg: fil.builder.$params, fn: fil.builder.$functions }; - if (fil.backup || fil.builder.$options.backup) - backup = true; - if (fil.builder.$keys == null) - keys = null; - else { - for (var j = 0; j < fil.builder.$keys.length; j++) { - keyscount++; - keys[fil.builder.$keys[j]] = 1; - } - } - } + var data = { keys: self.$keys }; - var data = {}; - data.keys = keys && keyscount ? Object.keys(keys) : self.$keys; + var remove = function(docs, d, dindex, f) { + var rec = fs.docsbuffer[dindex + 1]; + f.builder.$options.backup && f.builder.$backupdoc(rec.doc); + return 1; + }; + + var removeflush = function(docs, d, dindex) { + var rec = fs.docsbuffer[dindex + 1]; + !change && (change = true); + self.$events.remove && self.emit('remove', d); + fs.write(fs.remchar + rec.doc.substring(1) + NEWLINE, rec.position); + }; fs.ondocuments = function() { var lines = fs.docs.split(fs.divider); + var arr = []; for (var a = indexer ? 0 : 1; a < lines.length; a++) { - data.line = lines[a].split('|'); data.index = indexer++; - - indexer++; - var removed = false; - var doc = self.parseData(data); - var rec = fs.docsbuffer[a]; - - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.filter.index = indexer; - var output = item.compare(doc, item.filter, indexer); - if (output) { - removed = true; - doc = output; - break; - } - } - - if (removed) { - - if (backup) { - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.backup && item.backup.write(rec.doc + NEWLINE); - item.builder.$options.backup && item.builder.$backupdoc(rec.doc); - } - } - - if (!change) - change = true; - - item.count++; - self.$events.remove && self.emit('remove', doc); - fs.write(fs.remchar + rec.doc.substring(1) + NEWLINE, rec.position); - } + arr.push(self.parseData(data)); } + + filters.compare2(arr, remove, removeflush); }; fs.$callback = function() { - - for (var i = 0; i < length; i++) { - var item = filter[i]; - item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); - } - + filters.done(); fs = null; self.$writting = false; self.next(0); @@ -7127,8 +6157,10 @@ TP.parseData = function(data, cache) { var esc = data.line[0] === '*'; var val, alloc; - if (cache && data.keys.length === data.line.length - 2) - alloc = data.line[data.line.length - 1].length - 1; + if (cache && data.keys.length === data.line.length - 2) { + // alloc = data.line[data.line.length - 1].length - 1; + alloc = data.line[data.line.length - 1].length; + } for (var i = 0; i < data.keys.length; i++) { var key = data.keys[i]; @@ -7171,7 +6203,7 @@ TP.parseData = function(data, cache) { } } - alloc && (obj.$$alloc = { size: alloc, length: data.length }); + alloc >= 0 && (obj.$$alloc = { size: alloc, length: data.length }); return obj; }; @@ -7224,7 +6256,7 @@ TP.stringify = function(doc, insert, byteslen) { if (doc.$$alloc) { var l = output.length; var a = doc.$$alloc; - if (l < a.length) { + if (l <= a.length) { var s = (a.length - l) - 1; if (s > 0) { output += '|'.padRight(s, '.'); @@ -7353,132 +6385,227 @@ function jsonparser(key, value) { // item.filter = builder.makefilter(); // item.index = DOCUMENT_COUNTER; -exports.compare = function(item, obj) { +function NoSQLReader(builder) { + var self = this; + self.builders = []; + self.canceled = 0; + builder && self.add(builder); +} + +NoSQLReader.prototype.add = function(builder, noTrimmer) { + var self = this; + if (builder instanceof Array) { + for (var i = 0; i < builder.length; i++) + self.add(builder[i]); + } else { + var item = {}; + item.scalarcount = 0; + item.count = 0; + item.counter = 0; + item.builder = builder; + item.compare = builder.compile(noTrimmer); + item.filter = builder.makefilter(); + item.first = builder.$options.first && !builder.$options.sort; + self.builders.push(item); + } + return self; +}; - var val; - var builder = item.builder; +NoSQLReader.prototype.compare2 = function(docs, custom, done) { + var self = this; + for (var i = 0; i < docs.length; i++) { - item.filter.index = item.index; + var doc = docs[i]; - var output = item.compare(obj, item.filter, item.index); - if (!output) - return; + if (self.builders.length === self.canceled) + return false; - item.count++; + var is = false; - if (!builder.$inlinesort && ((builder.$options.skip && builder.$options.skip >= item.count) || (builder.$options.take && builder.$options.take <= item.counter))) - return; + for (var j = 0; j < self.builders.length; j++) { - item.counter++; + var item = self.builders[j]; + if (item.canceled) + continue; - if (!builder.$inlinesort && !item.done) - item.done = builder.$options.take && builder.$options.take <= item.counter; + var output = item.compare(doc, item.filter, j); + if (!output) + continue; - if (item.type) - return; + item.is = false; - switch (builder.$options.scalar) { - case 'count': - item.scalar = item.scalar ? item.scalar + 1 : 1; - break; - case 'sum': - val = output[builder.$options.scalarfield] || 0; - item.scalar = item.scalar ? item.scalar + val : val; - break; - case 'min': - val = output[builder.$options.scalarfield] || 0; - if (val != null) { - if (item.scalar) { - if (item.scalar > val) - item.scalar = val; - } else - item.scalar = val; + // @TODO: add "first" checking + if (custom) { + !is && (is = true); + item.count++; + var canceled = item.canceled; + var c = custom(docs, output, i, item, j); + if (!canceled && item.canceled) + self.canceled++; + if (c === 1) + break; + else + continue; } - break; - case 'max': - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar) { - if (item.scalar < val) - item.scalar = val; - } else - item.scalar = val; + } + + is && done && done(docs, doc, i, self.builders); + } +}; + +NoSQLReader.prototype.compare = function(docs) { + + var self = this; + for (var i = 0; i < docs.length; i++) { + + var doc = docs[i]; + + if (self.builders.length === self.canceled) + return false; + + for (var j = 0; j < self.builders.length; j++) { + + var item = self.builders[j]; + if (item.canceled) + continue; + + var output = item.compare(doc, item.filter, j); + if (!output) + continue; + + var b = item.builder; + item.count++; + + if (!b.$inlinesort && ((b.$options.skip && b.$options.skip >= item.count) || (b.$options.take && b.$options.take <= item.counter))) + continue; + + item.counter++; + + if (!b.$inlinesort && !item.done) { + item.done = b.$options.take && b.$options.take <= item.counter; + if (item.done) + continue; } - break; - case 'avg': - val = output[builder.$options.scalarfield]; - if (val != null) { - item.scalar = item.scalar ? item.scalar + val : val; - item.scalarcount++; + + if (b.$options.readertype) + continue; + + b.$mappersexec && b.$mappersexec(doc, item.filter); + + var val; + + switch (b.$options.scalar) { + case 'count': + item.scalar = item.scalar ? item.scalar + 1 : 1; + break; + case 'sum': + val = output[b.$options.scalarfield] || 0; + item.scalar = item.scalar ? item.scalar + val : val; + break; + case 'min': + val = output[b.$options.scalarfield] || 0; + if (val != null) { + if (item.scalar) { + if (item.scalar > val) + item.scalar = val; + } else + item.scalar = val; + } + break; + case 'max': + val = output[b.$options.scalarfield]; + if (val != null) { + if (item.scalar) { + if (item.scalar < val) + item.scalar = val; + } else + item.scalar = val; + } + break; + case 'avg': + val = output[b.$options.scalarfield]; + if (val != null) { + item.scalar = item.scalar ? item.scalar + val : val; + item.scalarcount++; + } + break; + case 'group': + !item.scalar && (item.scalar = {}); + val = output[b.$options.scalarfield]; + if (val != null) { + if (item.scalar[val]) + item.scalar[val]++; + else + item.scalar[val] = 1; + } + break; + default: + if (b.$inlinesort) + nosqlinlinesorter(item, b, output); + else if (item.response) + item.response.push(output); + else + item.response = [output]; + break; } - break; - case 'group': - !item.scalar && (item.scalar = {}); - val = output[builder.$options.scalarfield]; - if (val != null) { - if (item.scalar[val]) - item.scalar[val]++; - else - item.scalar[val] = 1; + + if (item.first) { + item.canceled = true; + self.canceled++; } - break; - default: - if (builder.$inlinesort) - nosqlinlinesorter(item, builder, output); - else if (item.response) - item.response.push(output); - else - item.response = [output]; - break; + } } - - return item.first ? false : true; }; -exports.callback = function(item, err) { +NoSQLReader.prototype.done = function() { - var builder = item.builder; - var output; + var self = this; + for (var i = 0; i < self.builders.length; i++) { - if (builder.$options.scalar || !builder.$options.sort) { + var item = self.builders[i]; + var builder = item.builder; + var output; - if (builder.$options.scalar) - output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; - else if (builder.$options.first) + if (builder.$options.scalar || !builder.$options.sort) { + if (builder.$options.scalar) + output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; + else if (builder.$options.first) + output = item.response ? item.response[0] : undefined; + else if (builder.$options.listing) + output = listing(builder, item); + else + output = item.response || []; + builder.$callback2(errorhandling(null, builder, output), builder.$options.readertype === 1 ? item.count : output, item.count); + continue; + } + + if (item.count) { + if (builder.$options.sort.name) { + if (!builder.$inlinesort || builder.$options.take !== item.response.length) + item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); + } else if (builder.$options.sort === null) + item.response.random(); + else + item.response.sort(builder.$options.sort); + + if (builder.$options.skip && builder.$options.take) + item.response = item.response.splice(builder.$options.skip, builder.$options.take); + else if (builder.$options.skip) + item.response = item.response.splice(builder.$options.skip); + else if (!builder.$inlinesort && builder.$options.take) + item.response = item.response.splice(0, builder.$options.take); + } + + if (builder.$options.first) output = item.response ? item.response[0] : undefined; else if (builder.$options.listing) output = listing(builder, item); else output = item.response || []; - builder.$callback2(errorhandling(err, builder, output), item.type === 1 ? item.count : output, item.count); - return; + builder.$callback2(errorhandling(null, builder, output), builder.$options.readertype === 1 ? item.count : output, item.count); + builder.done(); } +}; - if (item.count) { - if (builder.$options.sort.name) { - if (!builder.$inlinesort || builder.$options.take !== item.response.length) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - } else if (builder.$options.sort === null) - item.response.random(); - else - item.response.sort(builder.$options.sort); - - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (!builder.$inlinesort && builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); - } - - if (builder.$options.first) - output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) - output = listing(builder, item); - else - output = item.response || []; - - builder.$callback2(errorhandling(err, builder, output), item.type === 1 ? item.count : output, item.count); - builder.done(); -}; \ No newline at end of file +exports.NoSQLReader = NoSQLReader; \ No newline at end of file From b193f67883ceb46529c4238c50a8ea574da3d4a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 23 Aug 2018 20:58:50 +0200 Subject: [PATCH 013/217] Fixed `table.modify()` / `table.update()` --- nosql.js | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/nosql.js b/nosql.js index 9a8c670ad..e840132f4 100755 --- a/nosql.js +++ b/nosql.js @@ -1103,6 +1103,7 @@ DP.remove = function() { self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); self.pending_remove.push(builder); + builder.$options.readertype = 1; setImmediate(next_operation, self, 3); return builder; }; @@ -5299,6 +5300,7 @@ TP.update = function(doc, insert) { var self = this; self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); + builder.$options.readertype = 1; self.pending_update.push({ builder: builder, doc: doc, count: 0, insert: insert === true ? doc : insert }); setImmediate(next_operation, self, 2); return builder; @@ -5328,6 +5330,7 @@ TP.modify = function(doc, insert) { break; } } + builder.$options.readertype = 1; self.pending_update.push({ builder: builder, doc: data, count: 0, keys: keys, inc: inc, insert: insert === true ? data : insert }); setImmediate(next_operation, self, 2); } @@ -5339,6 +5342,7 @@ TP.remove = function() { self.readonly && self.throwReadonly(); var builder = new DatabaseBuilder(self); self.pending_remove.push(builder); + builder.$options.readertype = 1; setImmediate(next_operation, self, 3); return builder; }; @@ -6377,14 +6381,6 @@ function jsonparser(key, value) { return typeof(value) === 'string' && value.isJSONDate() ? new Date(value) : value; } -// Item requirements: -// item.first = false; -// item.scalarcount = 0; -// item.builder = builder; -// item.compare = builder.compile(); -// item.filter = builder.makefilter(); -// item.index = DOCUMENT_COUNTER; - function NoSQLReader(builder) { var self = this; self.builders = []; From 0d4639564fa9efe50e9a4ef3bfad88a54697ed58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 23 Aug 2018 21:28:25 +0200 Subject: [PATCH 014/217] Fixed file modifications. --- nosql.js | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/nosql.js b/nosql.js index e840132f4..a7e515839 100755 --- a/nosql.js +++ b/nosql.js @@ -6429,20 +6429,21 @@ NoSQLReader.prototype.compare2 = function(docs, custom, done) { continue; item.is = false; + !is && (is = true); + item.count++; + var canceled = item.canceled; + var c = custom(docs, output, i, item, j); - // @TODO: add "first" checking - if (custom) { - !is && (is = true); - item.count++; - var canceled = item.canceled; - var c = custom(docs, output, i, item, j); - if (!canceled && item.canceled) - self.canceled++; - if (c === 1) - break; - else - continue; - } + if (item.first) { + item.canceled = true; + self.canceled++; + } else if (!canceled && item.canceled) + self.canceled++; + + if (c === 1) + break; + else + continue; } is && done && done(docs, doc, i, self.builders); @@ -6477,11 +6478,8 @@ NoSQLReader.prototype.compare = function(docs) { item.counter++; - if (!b.$inlinesort && !item.done) { + if (!b.$inlinesort && !item.done) item.done = b.$options.take && b.$options.take <= item.counter; - if (item.done) - continue; - } if (b.$options.readertype) continue; From d4c05e827cc4225237e9937ebde6404c4a50955a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 23 Aug 2018 22:19:17 +0200 Subject: [PATCH 015/217] New improvements. --- nosql.js | 42 +++++++++++++++++++++++++ nosqlcrawler.js | 83 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 nosqlcrawler.js diff --git a/nosql.js b/nosql.js index a7e515839..29ff8c55e 100755 --- a/nosql.js +++ b/nosql.js @@ -5066,6 +5066,48 @@ SP.listing = function(beg, end, callback) { return self; }; +SP.find = function(beg, end, threads) { + var self = this; + + if (!threads) + threads = 1; + + var builder = new DatabaseBuilder(self); + var filters = new NoSQLReader([builder]); + + self.listing(beg, end, function(err, storage) { + + var count = (storage.length / threads) >> 0; + var opt = { cwd: F.directory }; + var filename = module.filename.replace(/\.js$/, '') + 'crawler.js'; + var counter = 0; + var finish = threads; + + for (var i = 0; i < threads; i++) { + var fork = require('child_process').fork(filename, EMPTYARRAY, opt); + var files = (i === threads - 1) ? storage : storage.splice(0, count); + fork.send({ TYPE: 'init', files: files, builder: builder.stringify() }); + fork.on('message', function(msg) { + counter += msg.count; + msg.response && msg.response.length && filters.compare(msg.response); + finish--; + if (finish === 0) { + filters.builders[0].count = counter; + filters.done(); + } + }); + } + }); + + return builder; +}; + +SP.count = function(beg, end, threads) { + var builder = this.find(beg, end, threads); + builder.$options.readertype = 1; + return builder; +}; + SP.scan = function(beg, end, mapreduce, callback, reverse) { var self = this; diff --git a/nosqlcrawler.js b/nosqlcrawler.js new file mode 100644 index 000000000..bc2998931 --- /dev/null +++ b/nosqlcrawler.js @@ -0,0 +1,83 @@ +// Copyright 2018 (c) Peter Širka +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +/** + * @module NoSQL Crawler + * @version 1.0.0 + */ + +var filename = module.filename.replace(/nosqlcrawler\.js$/, ''); + +require(filename + 'index.js'); +const NoSQLStream = require(filename + 'nosqlstream.js'); + +global.NOW = global.DATETIME = new Date(); + +function killprocess() { + process.exit(0); +} + +setInterval(function() { + global.NOW = global.DATETIME = new Date(); +}, 30000); + +process.on('disconnect', killprocess); +process.on('close', killprocess); +process.on('exit', killprocess); +process.on('message', function(msg) { + + // msg.builder; + // msg.filename or msg.data + + var builder = new framework_nosql.DatabaseBuilder(); + builder.parse(msg.builder); + + var filters = new framework_nosql.NoSQLReader([builder]); + + msg.files.wait(function(item, next) { + find(item.filename, filters, next); + }, function() { + var item = filters.builders[0]; + process.send({ response: item.response, count: item.count }); + setTimeout(() => killprocess(), 1000); + }); + +}); + +function find(filename, filters, next) { + + var fs = new NoSQLStream(filename); + + fs.ondocuments = function() { + return filters.compare(JSON.parse('[' + fs.docs + ']', jsonparser)); + }; + + fs.$callback = function() { + fs = null; + next(); + }; + + fs.openread(); +} + +function jsonparser(key, value) { + return typeof(value) === 'string' && value.isJSONDate() ? new Date(value) : value; +} From c9a330339e4edc8e09fd2f368c44534f725a7a12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 23 Aug 2018 22:20:45 +0200 Subject: [PATCH 016/217] Added new change. --- changes.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changes.txt b/changes.txt index 813b5e95f..443b22595 100755 --- a/changes.txt +++ b/changes.txt @@ -1,6 +1,9 @@ ======= 3.0.1 - added: CSS variables support default values `border-radius: $radius || 10px` +- added: NoSQL storage `.find()` + `.count()` with multiple thread support + +- improved: NoSQL reader - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function From 0ab820b973ad59f3bd97f975163c492b3ea32f3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 23 Aug 2018 23:35:04 +0200 Subject: [PATCH 017/217] Fixed mapping. --- nosql.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nosql.js b/nosql.js index 29ff8c55e..ebbb2402f 100755 --- a/nosql.js +++ b/nosql.js @@ -2815,10 +2815,11 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { self.$mappers = []; for (var i = 0; i < opt.mappers.length; i++) { var m = opt.mappers[i]; - tmp += ('doc.{0}=item.builder.$mappers[\'{1}\'](doc,R);'.format(m.name, i)); - self.$mappers.push(new Function('doc', 'repository', code)); + tmp += ('doc.{0}=item.builder.$mappers[{1}](doc,item.filter.repository,item.filter.repository);'.format(m.name, i)); + opt.fields && opt.fields.push(m.name); + self.$mappers.push(new Function('doc', 'repository', 'R', m.code.lastIndexOf('return ') === -1 ? ('return ' + m.code) : m.code)); } - self.$mappersexec = new Function('doc', 'R', tmp.join('')); + self.$mappersexec = new Function('doc', 'item', tmp); } return CACHE[key] ? CACHE[key] : (CACHE[key] = new Function('doc', '$F', 'index', code)); @@ -6526,7 +6527,7 @@ NoSQLReader.prototype.compare = function(docs) { if (b.$options.readertype) continue; - b.$mappersexec && b.$mappersexec(doc, item.filter); + b.$mappersexec && b.$mappersexec(output, item); var val; From d2fdbec5cff31555f1cb8188330cc98b2aa9102c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 24 Aug 2018 00:01:15 +0200 Subject: [PATCH 018/217] New improvement. --- nosql.js | 7 +- utils.js | 345 +++++++++++++++++++++++++++++++------------------------ 2 files changed, 203 insertions(+), 149 deletions(-) diff --git a/nosql.js b/nosql.js index ebbb2402f..51f477d4b 100755 --- a/nosql.js +++ b/nosql.js @@ -5074,10 +5074,10 @@ SP.find = function(beg, end, threads) { threads = 1; var builder = new DatabaseBuilder(self); - var filters = new NoSQLReader([builder]); self.listing(beg, end, function(err, storage) { + var filters = new NoSQLReader([builder]); var count = (storage.length / threads) >> 0; var opt = { cwd: F.directory }; var filename = module.filename.replace(/\.js$/, '') + 'crawler.js'; @@ -5109,6 +5109,10 @@ SP.count = function(beg, end, threads) { return builder; }; +SP.scalar = function(type, field) { + return this.find().scalar(type, field); +}; + SP.scan = function(beg, end, mapreduce, callback, reverse) { var self = this; @@ -6528,7 +6532,6 @@ NoSQLReader.prototype.compare = function(docs) { continue; b.$mappersexec && b.$mappersexec(output, item); - var val; switch (b.$options.scalar) { diff --git a/utils.js b/utils.js index cd566f4c6..a8b35f079 100755 --- a/utils.js +++ b/utils.js @@ -664,14 +664,16 @@ function ProxyAgent(options) { self.requests = []; } -ProxyAgent.prototype.createConnection = function(pending) { +const PAP = ProxyAgent.prototype; + +PAP.createConnection = function(pending) { var self = this; self.createSocket(pending, function(socket) { pending.request.onSocket(socket); }); }; -ProxyAgent.prototype.createSocket = function(options, callback) { +PAP.createSocket = function(options, callback) { var self = this; var proxy = self.options.proxy; @@ -719,13 +721,13 @@ function proxyagent_response(res) { res.upgrade = true; } -ProxyAgent.prototype.addRequest = function(req, options) { +PAP.addRequest = function(req, options) { this.createConnection({ host: options.host, port: options.port, request: req }); }; function createSecureSocket(options, callback) { var self = this; - ProxyAgent.prototype.createSocket.call(self, options, function(socket) { + PAP.createSocket.call(self, options, function(socket) { PROXYTLS.servername = self.options.uri.hostname; PROXYTLS.headers = self.options.uri.headers; PROXYTLS.socket = socket; @@ -2655,7 +2657,7 @@ exports.ls2 = function(path, callback, filter) { ls(path, callback, true, filter); }; -Date.prototype.add = function(type, value) { +DP.add = function(type, value) { var self = this; @@ -2719,13 +2721,15 @@ Date.prototype.add = function(type, value) { return dt; }; +const DP = Date.prototype; + /** * Date difference * @param {Date/Number/String} date Optional. * @param {String} type Date type: minutes, seconds, hours, days, months, years * @return {Number} */ -Date.prototype.diff = function(date, type) { +DP.diff = function(date, type) { if (arguments.length === 1) { type = date; @@ -2779,7 +2783,7 @@ Date.prototype.diff = function(date, type) { return NaN; }; -Date.prototype.extend = function(date) { +DP.extend = function(date) { var dt = new Date(this); var match = date.match(regexpDATE); @@ -2856,7 +2860,7 @@ Date.prototype.extend = function(date) { * @param {Date} date * @return {Number} Results: -1 = current date is earlier than @date, 0 = current date is same as @date, 1 = current date is later than @date */ -Date.prototype.compare = function(date) { +DP.compare = function(date) { var self = this; var r = self.getTime() - date.getTime(); @@ -2892,7 +2896,7 @@ Date.compare = function(d1, d2) { * @param {String} format * @return {String} */ -Date.prototype.format = function(format, resource) { +DP.format = function(format, resource) { if (!format) return this.getUTCFullYear() + '-' + (this.getUTCMonth() + 1).toString().padLeft(2, '0') + '-' + this.getUTCDate().toString().padLeft(2, '0') + 'T' + this.getUTCHours().toString().padLeft(2, '0') + ':' + this.getUTCMinutes().toString().padLeft(2, '0') + ':' + this.getUTCSeconds().toString().padLeft(2, '0') + '.' + this.getUTCMilliseconds().toString().padLeft(3, '0') + 'Z'; @@ -2978,22 +2982,24 @@ exports.$pmam = function(value) { return value >= 12 ? value - 12 : value; }; -Date.prototype.toUTC = function(ticks) { +DP.toUTC = function(ticks) { var dt = this.getTime() + this.getTimezoneOffset() * 60000; return ticks ? dt : new Date(dt); }; // +v2.2.0 parses JSON dates as dates and this is the fallback for backward compatibility -Date.prototype.parseDate = function() { +DP.parseDate = function() { return this; }; -String.prototype.isJSONDate = function() { +const SP = String.prototype; + +SP.isJSONDate = function() { var l = this.length - 1; return l > 22 && l < 30 && this[l] === 'Z' && this[10] === 'T' && this[4] === '-' && this[13] === ':' && this[16] === ':'; }; -String.prototype.ROOT = function(noremap) { +SP.ROOT = function(noremap) { var str = this; @@ -3018,26 +3024,26 @@ function $urlmaker(text) { return F.config['default-root'] ? F.config['default-root'] : (c || ''); } -if (!String.prototype.trim) { - String.prototype.trim = function() { +if (!SP.trim) { + SP.trim = function() { return this.replace(regexpTRIM, ''); }; } -if (!String.prototype.replaceAt) { - String.prototype.replaceAt = function(index, character) { +if (!SP.replaceAt) { + SP.replaceAt = function(index, character) { return this.substr(0, index) + character + this.substr(index + character.length); }; } /** * Checks if the string starts with the text - * @see {@link http://docs.totaljs.com/String.prototype/#String.prototype.startsWith|Documentation} + * @see {@link http://docs.totaljs.com/SP/#SP.startsWith|Documentation} * @param {String} text Text to find. * @param {Boolean/Number} ignoreCase Ingore case sensitive or position in the string. * @return {Boolean} */ -String.prototype.startsWith = function(text, ignoreCase) { +SP.startsWith = function(text, ignoreCase) { var self = this; var length = text.length; var tmp; @@ -3057,12 +3063,12 @@ String.prototype.startsWith = function(text, ignoreCase) { /** * Checks if the string ends with the text - * @see {@link http://docs.totaljs.com/String.prototype/#String.prototype.endsWith|Documentation} + * @see {@link http://docs.totaljs.com/SP/#SP.endsWith|Documentation} * @param {String} text Text to find. * @param {Boolean/Number} ignoreCase Ingore case sensitive or position in the string. * @return {Boolean} */ -String.prototype.endsWith = function(text, ignoreCase) { +SP.endsWith = function(text, ignoreCase) { var self = this; var length = text.length; var tmp; @@ -3080,7 +3086,7 @@ String.prototype.endsWith = function(text, ignoreCase) { return tmp.length === length && tmp === text; }; -String.prototype.replacer = function(find, text) { +SP.replacer = function(find, text) { var self = this; var beg = self.indexOf(find); return beg === -1 ? self : (self.substring(0, beg) + text + self.substring(beg + find.length)); @@ -3092,7 +3098,7 @@ String.prototype.replacer = function(find, text) { * @param {String} salt Optional, salt. * @return {String} */ -String.prototype.hash = function(type, salt) { +SP.hash = function(type, salt) { var str = salt ? this + salt : this; switch (type) { case 'md5': @@ -3112,7 +3118,7 @@ String.prototype.hash = function(type, salt) { } }; -String.prototype.crc32 = function(unsigned) { +SP.crc32 = function(unsigned) { var crc = -1; for (var i = 0, length = this.length; i < length; i++) crc = (crc >>> 8) ^ CRC32TABLE[(crc ^ this.charCodeAt(i)) & 0xFF]; @@ -3132,7 +3138,7 @@ function string_hash(s, convert) { return hash; } -String.prototype.count = function(text) { +SP.count = function(text) { var index = 0; var count = 0; do { @@ -3143,19 +3149,19 @@ String.prototype.count = function(text) { return count; }; -String.prototype.parseXML = function() { +SP.parseXML = function() { return F.onParseXML(this); }; -String.prototype.parseJSON = function(date) { +SP.parseJSON = function(date) { return exports.parseJSON(this, date); }; -String.prototype.parseQuery = function() { +SP.parseQuery = function() { return exports.parseQuery(this); }; -String.prototype.parseTerminal = function(fields, fn, skip, take) { +SP.parseTerminal = function(fields, fn, skip, take) { var lines = this.split('\n'); @@ -3273,7 +3279,7 @@ function parseTerminal2(lines, fn, skip, take) { } } -String.prototype.parseDate = function() { +SP.parseDate = function() { var self = this.trim(); var lc = self.charCodeAt(self.length - 1); @@ -3366,7 +3372,7 @@ String.prototype.parseDate = function() { return new Date(parsed[0], parsed[1] - 1, parsed[2], parsed[3], parsed[4] - NOW.getTimezoneOffset(), parsed[5]); }; -String.prototype.parseDateExpiration = function() { +SP.parseDateExpiration = function() { var self = this; var arr = self.split(' '); @@ -3385,7 +3391,7 @@ String.prototype.parseDateExpiration = function() { return dt; }; -String.prototype.contains = function(value, mustAll) { +SP.contains = function(value, mustAll) { var str = this; if (typeof(value) === 'string') @@ -3408,7 +3414,7 @@ String.prototype.contains = function(value, mustAll) { * @param {String} value * @return {Number} */ -String.prototype.localeCompare2 = function(value) { +SP.localeCompare2 = function(value) { return COMPARER(this, value); }; @@ -3418,7 +3424,7 @@ String.prototype.localeCompare2 = function(value) { * @onerr {Function} error handling * @return {Object} */ -String.prototype.parseConfig = function(def, onerr) { +SP.parseConfig = function(def, onerr) { if (typeof(def) === 'function') { onerr = def; @@ -3506,7 +3512,7 @@ String.prototype.parseConfig = function(def, onerr) { return obj; }; -String.prototype.format = function() { +SP.format = function() { var arg = arguments; return this.replace(regexpSTRINGFORMAT, function(text) { var value = arg[+text.substring(1, text.length - 1)]; @@ -3514,7 +3520,7 @@ String.prototype.format = function() { }); }; -String.prototype.encode = function() { +SP.encode = function() { var output = ''; for (var i = 0, length = this.length; i < length; i++) { var c = this[i]; @@ -3542,7 +3548,7 @@ String.prototype.encode = function() { return output; }; -String.prototype.decode = function() { +SP.decode = function() { return this.replace(regexpDECODE, function(s) { if (s.charAt(1) !== '#') return ALPHA_INDEX[s] || s; @@ -3551,15 +3557,15 @@ String.prototype.decode = function() { }); }; -String.prototype.urlEncode = function() { +SP.urlEncode = function() { return encodeURIComponent(this); }; -String.prototype.urlDecode = function() { +SP.urlDecode = function() { return decodeURIComponent(this); }; -String.prototype.arg = function(obj) { +SP.arg = function(obj) { return this.replace(regexpARG, function(text) { // Is double? var l = text[1] === '{' ? 2 : 1; @@ -3568,7 +3574,7 @@ String.prototype.arg = function(obj) { }); }; -String.prototype.params = function(obj) { +SP.params = function(obj) { OBSOLETE('String.params()', 'The method is deprecated instead of it use F.viewCompile() or String.format().'); @@ -3639,14 +3645,14 @@ String.prototype.params = function(obj) { }); }; -String.prototype.max = function(length, chars) { +SP.max = function(length, chars) { var str = this; if (typeof(chars) !== 'string') chars = '...'; return str.length > length ? str.substring(0, length - chars.length) + chars : str; }; -String.prototype.isJSON = function() { +SP.isJSON = function() { var self = this; if (self.length <= 1) return false; @@ -3673,48 +3679,48 @@ String.prototype.isJSON = function() { return (a === '"' && b === '"') || (a === '[' && b === ']') || (a === '{' && b === '}') || (a.charCodeAt(0) > 47 && b.charCodeAt(0) < 57); }; -String.prototype.isURL = function() { +SP.isURL = function() { return this.length <= 7 ? false : F.validators.url.test(this); }; -String.prototype.isZIP = function() { +SP.isZIP = function() { return F.validators.zip.test(this); }; -String.prototype.isEmail = function() { +SP.isEmail = function() { return this.length <= 4 ? false : F.validators.email.test(this); }; -String.prototype.isPhone = function() { +SP.isPhone = function() { return this.length < 6 ? false : F.validators.phone.test(this); }; -String.prototype.isUID = function() { +SP.isUID = function() { return this.length < 18 ? false : F.validators.uid.test(this); }; -String.prototype.parseInt = function(def) { +SP.parseInt = function(def) { var str = this.trim(); var num = +str; return isNaN(num) ? (def || 0) : num; }; -String.prototype.parseInt2 = function(def) { +SP.parseInt2 = function(def) { var num = this.match(regexpINTEGER); return num ? +num[0] : def || 0; }; -String.prototype.parseFloat2 = function(def) { +SP.parseFloat2 = function(def) { var num = this.match(regexpFLOAT); return num ? +num[0].toString().replace(/,/g, '.') : def || 0; }; -String.prototype.parseBool = String.prototype.parseBoolean = function() { +SP.parseBool = SP.parseBoolean = function() { var self = this.toLowerCase(); return self === 'true' || self === '1' || self === 'on'; }; -String.prototype.parseFloat = function(def) { +SP.parseFloat = function(def) { var str = this.trim(); if (str.indexOf(',') !== -1) str = str.replace(',', '.'); @@ -3722,7 +3728,7 @@ String.prototype.parseFloat = function(def) { return isNaN(num) ? (def || 0) : num; }; -String.prototype.capitalize = function(first) { +SP.capitalize = function(first) { if (first) return this[0].toUpperCase() + this.substring(1); @@ -3743,7 +3749,7 @@ String.prototype.capitalize = function(first) { }; -String.prototype.toUnicode = function() { +SP.toUnicode = function() { var result = ''; var self = this; var length = self.length; @@ -3756,35 +3762,35 @@ String.prototype.toUnicode = function() { return result; }; -String.prototype.fromUnicode = function() { +SP.fromUnicode = function() { return unescape(this.replace(regexpUNICODE, (match, v) => String.fromCharCode(parseInt(v, 16)))); }; -String.prototype.sha1 = function(salt) { +SP.sha1 = function(salt) { var hash = Crypto.createHash('sha1'); hash.update(this + (salt || ''), ENCODING); return hash.digest('hex'); }; -String.prototype.sha256 = function(salt) { +SP.sha256 = function(salt) { var hash = Crypto.createHash('sha256'); hash.update(this + (salt || ''), ENCODING); return hash.digest('hex'); }; -String.prototype.sha512 = function(salt) { +SP.sha512 = function(salt) { var hash = Crypto.createHash('sha512'); hash.update(this + (salt || ''), ENCODING); return hash.digest('hex'); }; -String.prototype.md5 = function(salt) { +SP.md5 = function(salt) { var hash = Crypto.createHash('md5'); hash.update(this + (salt || ''), ENCODING); return hash.digest('hex'); }; -String.prototype.toSearch = function() { +SP.toSearch = function() { var str = this.replace(regexpSEARCH, '').trim().toLowerCase().removeDiacritics(); var buf = []; var prev = ''; @@ -3801,7 +3807,7 @@ String.prototype.toSearch = function() { return buf.join(''); }; -String.prototype.toKeywords = String.prototype.keywords = function(forSearch, alternative, max_count, max_length, min_length) { +SP.toKeywords = SP.keywords = function(forSearch, alternative, max_count, max_length, min_length) { return exports.keywords(this, forSearch, alternative, max_count, max_length, min_length); }; @@ -3812,7 +3818,7 @@ function checksum(val) { return sum; } -String.prototype.encrypt = function(key, isUnique, secret) { +SP.encrypt = function(key, isUnique, secret) { var str = '0' + this; var data_count = str.length; var key_count = key.length; @@ -3839,7 +3845,7 @@ String.prototype.encrypt = function(key, isUnique, secret) { return (sum + checksum((secret || F.config.secret) + key)) + '-' + str; }; -String.prototype.decrypt = function(key, secret) { +SP.decrypt = function(key, secret) { var index = this.indexOf('-'); if (index === -1) @@ -3912,7 +3918,7 @@ exports.decryptUID = function(val, key) { return exports.encryptUID(raw, key) === val ? raw : null; }; -String.prototype.base64ToFile = function(filename, callback) { +SP.base64ToFile = function(filename, callback) { var self = this; var index = self.indexOf(','); if (index === -1) @@ -3923,7 +3929,7 @@ String.prototype.base64ToFile = function(filename, callback) { return this; }; -String.prototype.base64ToBuffer = function() { +SP.base64ToBuffer = function() { var self = this; var index = self.indexOf(','); @@ -3935,17 +3941,17 @@ String.prototype.base64ToBuffer = function() { return exports.createBuffer(self.substring(index), 'base64'); }; -String.prototype.base64ContentType = function() { +SP.base64ContentType = function() { var self = this; var index = self.indexOf(';'); return index === -1 ? '' : self.substring(5, index); }; -String.prototype.removeDiacritics = function() { +SP.removeDiacritics = function() { return exports.removeDiacritics(this); }; -String.prototype.indent = function(max, c) { +SP.indent = function(max, c) { var plus = ''; if (c === undefined) c = ' '; @@ -3954,7 +3960,7 @@ String.prototype.indent = function(max, c) { return plus + this; }; -String.prototype.isNumber = function(isDecimal) { +SP.isNumber = function(isDecimal) { var self = this; var length = self.length; @@ -3981,8 +3987,8 @@ String.prototype.isNumber = function(isDecimal) { return true; }; -if (!String.prototype.padLeft) { - String.prototype.padLeft = function(max, c) { +if (!SP.padLeft) { + SP.padLeft = function(max, c) { var self = this; var len = max - self.length; if (len < 0) @@ -3996,8 +4002,8 @@ if (!String.prototype.padLeft) { } -if (!String.prototype.padRight) { - String.prototype.padRight = function(max, c) { +if (!SP.padRight) { + SP.padRight = function(max, c) { var self = this; var len = max - self.length; if (len < 0) @@ -4010,7 +4016,7 @@ if (!String.prototype.padRight) { }; } -String.prototype.insert = function(index, value) { +SP.insert = function(index, value) { var str = this; var a = str.substring(0, index); var b = value.toString() + str.substring(index); @@ -4022,7 +4028,7 @@ String.prototype.insert = function(index, value) { * @param {Number} max A maximum length, default: 60 and optional. * @return {String} */ -String.prototype.slug = String.prototype.toSlug = String.prototype.toLinker = String.prototype.linker = function(max) { +SP.slug = SP.toSlug = SP.toLinker = SP.linker = function(max) { max = max || 60; var self = this.trim().toLowerCase().removeDiacritics(); @@ -4049,11 +4055,11 @@ String.prototype.slug = String.prototype.toSlug = String.prototype.toLinker = St return builder[l] === '-' ? builder.substring(0, l) : builder; }; -String.prototype.pluralize = function(zero, one, few, other) { +SP.pluralize = function(zero, one, few, other) { return this.parseInt().pluralize(zero, one, few, other); }; -String.prototype.isBoolean = function() { +SP.isBoolean = function() { var self = this.toLowerCase(); return (self === 'true' || self === 'false') ? true : false; }; @@ -4062,11 +4068,11 @@ String.prototype.isBoolean = function() { * Check if the string contains only letters and numbers. * @return {Boolean} */ -String.prototype.isAlphaNumeric = function() { +SP.isAlphaNumeric = function() { return regexpALPHA.test(this); }; -String.prototype.soundex = function() { +SP.soundex = function() { var arr = this.toLowerCase().split(''); var first = arr.shift(); @@ -4090,23 +4096,25 @@ String.prototype.soundex = function() { * Remove all Html Tags from a string * @return {string} */ -String.prototype.removeTags = function() { +SP.removeTags = function() { return this.replace(regexpTags, ''); }; -Number.prototype.floor = function(decimals) { +const NP = Number.prototype; + +NP.floor = function(decimals) { return Math.floor(this * Math.pow(10, decimals)) / Math.pow(10, decimals); }; -Number.prototype.padLeft = function(max, c) { +NP.padLeft = function(max, c) { return this.toString().padLeft(max, c || '0'); }; -Number.prototype.padRight = function(max, c) { +NP.padRight = function(max, c) { return this.toString().padRight(max, c || '0'); }; -Number.prototype.round = function(precision) { +NP.round = function(precision) { var m = Math.pow(10, precision) || 1; return Math.round(this * m) / m; }; @@ -4117,7 +4125,7 @@ Number.prototype.round = function(precision) { * @param {Function} callback * @return {Number} */ -Number.prototype.async = function(fn, callback) { +NP.async = function(fn, callback) { var number = this; if (number) fn(number--, () => setImmediate(() => number.async(fn, callback))); @@ -4133,7 +4141,7 @@ Number.prototype.async = function(fn, callback) { * @param {String} separatorDecimal Decimal separator, default '.' if number separator is ',' or ' '. * @return {String} */ -Number.prototype.format = function(decimals, separator, separatorDecimal) { +NP.format = function(decimals, separator, separatorDecimal) { var self = this; @@ -4184,7 +4192,7 @@ Number.prototype.format = function(decimals, separator, separatorDecimal) { return minus + output + (dec.length ? separatorDecimal + dec : ''); }; -Number.prototype.add = function(value, decimals) { +NP.add = function(value, decimals) { if (value == null) return this; @@ -4254,7 +4262,7 @@ Number.prototype.add = function(value, decimals) { return num; }; -Number.prototype.format2 = function(format) { +NP.format2 = function(format) { var index = 0; var num = this.toString(); var beg = 0; @@ -4363,7 +4371,7 @@ Number.prototype.format2 = function(format) { return this.format(output); }; -Number.prototype.pluralize = function(zero, one, few, other) { +NP.pluralize = function(zero, one, few, other) { var num = this; var value = ''; @@ -4386,14 +4394,14 @@ Number.prototype.pluralize = function(zero, one, few, other) { return num.format(format) + value.replace(format, ''); }; -Number.prototype.hex = function(length) { +NP.hex = function(length) { var str = this.toString(16).toUpperCase(); while(str.length < length) str = '0' + str; return str; }; -Number.prototype.VAT = function(percentage, decimals, includedVAT) { +NP.VAT = function(percentage, decimals, includedVAT) { var num = this; var type = typeof(decimals); @@ -4415,25 +4423,25 @@ Number.prototype.VAT = function(percentage, decimals, includedVAT) { return includedVAT ? (num / ((percentage / 100) + 1)).floor(decimals) : (num * ((percentage / 100) + 1)).floor(decimals); }; -Number.prototype.discount = function(percentage, decimals) { +NP.discount = function(percentage, decimals) { var num = this; if (decimals === undefined) decimals = 2; return (num - (num / 100) * percentage).floor(decimals); }; -Number.prototype.parseDate = function(plus) { +NP.parseDate = function(plus) { return new Date(this + (plus || 0)); }; -if (!Number.prototype.toRad) { - Number.prototype.toRad = function () { +if (!NP.toRad) { + NP.toRad = function () { return this * Math.PI / 180; }; } -Number.prototype.filesize = function(decimals, type) { +NP.filesize = function(decimals, type) { if (typeof(decimals) === 'string') { var tmp = type; @@ -4501,12 +4509,14 @@ function filesizehelper(number, count) { return number; } +var AP = Array.prototype; + /** * Take items from array * @param {Number} count * @return {Array} */ -Array.prototype.take = function(count) { +AP.take = function(count) { var arr = []; var self = this; var length = self.length; @@ -4524,7 +4534,7 @@ Array.prototype.take = function(count) { * @param {Boolean} rewrite Default: false. * @return {Array} Returns self */ -Array.prototype.extend = function(obj, rewrite) { +AP.extend = function(obj, rewrite) { var isFn = typeof(obj) === 'function'; for (var i = 0, length = this.length; i < length; i++) { if (isFn) @@ -4540,7 +4550,7 @@ Array.prototype.extend = function(obj, rewrite) { * @param {Object} def Default value. * @return {Object} */ -Array.prototype.first = function(def) { +AP.first = function(def) { var item = this[0]; return item === undefined ? def : item; }; @@ -4550,7 +4560,7 @@ Array.prototype.first = function(def) { * @param {String} name Optional, property name. * @return {Object} */ -Array.prototype.toObject = function(name) { +AP.toObject = function(name) { var self = this; var obj = {}; @@ -4572,7 +4582,7 @@ Array.prototype.toObject = function(name) { * @param {Array} b Second array. * @param {Function(itemA, itemB, indexA, indexB)} executor */ -Array.prototype.compare = function(id, b, executor) { +AP.compare = function(id, b, executor) { var a = this; var ak = {}; @@ -4634,7 +4644,7 @@ Array.prototype.compare = function(id, b, executor) { * @param {Boolean} remove Optional, remove item from this array if the item doesn't exist int arr (default: false). * @return {Array} */ -Array.prototype.pair = function(property, arr, fn, remove) { +AP.pair = function(property, arr, fn, remove) { if (property instanceof Array) { var tmp = property; @@ -4678,12 +4688,12 @@ Array.prototype.pair = function(property, arr, fn, remove) { * @param {Object} def Default value. * @return {Object} */ -Array.prototype.last = function(def) { +AP.last = function(def) { var item = this[this.length - 1]; return item === undefined ? def : item; }; -Array.prototype.quicksort = Array.prototype.orderBy = function(name, asc) { +AP.quicksort = AP.orderBy = function(name, asc) { var length = this.length; if (!length || length === 1) @@ -4750,7 +4760,7 @@ Array.prototype.quicksort = Array.prototype.orderBy = function(name, asc) { return self; }; -Array.prototype.trim = function() { +AP.trim = function() { var self = this; var output = []; for (var i = 0, length = self.length; i < length; i++) { @@ -4766,7 +4776,7 @@ Array.prototype.trim = function() { * @param {Number} count * @return {Array} */ -Array.prototype.skip = function(count) { +AP.skip = function(count) { var arr = []; var self = this; var length = self.length; @@ -4781,7 +4791,7 @@ Array.prototype.skip = function(count) { * @param {Object} value Optional. * @return {Array} */ -Array.prototype.where = function(cb, value) { +AP.where = function(cb, value) { var self = this; var selected = []; @@ -4812,7 +4822,7 @@ Array.prototype.where = function(cb, value) { * @param {Object} value Optional. * @return {Array} */ -Array.prototype.findItem = Array.prototype.find = function(cb, value) { +AP.findItem = AP.find = function(cb, value) { var self = this; var index = self.findIndex(cb, value); if (index === -1) @@ -4820,7 +4830,7 @@ Array.prototype.findItem = Array.prototype.find = function(cb, value) { return self[index]; }; -Array.prototype.findIndex = function(cb, value) { +AP.findIndex = function(cb, value) { var self = this; var isFN = typeof(cb) === 'function'; @@ -4853,7 +4863,7 @@ Array.prototype.findIndex = function(cb, value) { * @param {Object} value Optional. * @return {Array} */ -Array.prototype.remove = function(cb, value) { +AP.remove = function(cb, value) { var self = this; var arr = []; @@ -4877,7 +4887,7 @@ Array.prototype.remove = function(cb, value) { return arr; }; -Array.prototype.wait = Array.prototype.waitFor = function(onItem, callback, thread, tmp) { +AP.wait = AP.waitFor = function(onItem, callback, thread, tmp) { var self = this; var init = false; @@ -4931,7 +4941,7 @@ function next_wait(self, onItem, callback, thread, tmp) { * @param {Function} callback Optional * @return {Array} */ -Array.prototype.async = function(thread, callback, pending) { +AP.async = function(thread, callback, pending) { var self = this; @@ -4970,13 +4980,13 @@ Array.prototype.async = function(thread, callback, pending) { return self; }; -Array.prototype.randomize = function() { +AP.randomize = function() { OBSOLETE('Array.randomize()', 'Use Array.random().'); return this.random(); }; // Fisher-Yates shuffle -Array.prototype.random = function() { +AP.random = function() { for (var i = this.length - 1; i > 0; i--) { var j = Math.floor(Math.random() * (i + 1)); var temp = this[i]; @@ -4986,7 +4996,7 @@ Array.prototype.random = function() { return this; }; -Array.prototype.limit = function(max, fn, callback, index) { +AP.limit = function(max, fn, callback, index) { if (index === undefined) index = 0; @@ -5031,7 +5041,7 @@ Array.prototype.limit = function(max, fn, callback, index) { * Get unique elements from Array * @return {[type]} [description] */ -Array.prototype.unique = function(property) { +AP.unique = function(property) { var self = this; var result = []; @@ -5210,21 +5220,23 @@ Async.prototype = { } }; -Async.prototype.__proto__ = Object.create(Events.EventEmitter.prototype, { +const ACP = Async.prototype; + +ACP.__proto__ = Object.create(Events.EventEmitter.prototype, { constructor: { value: Async, enumberable: false } }); -Async.prototype.reload = function() { +ACP.reload = function() { var self = this; self.tasksAll = Object.keys(self.tasksPending); self.emit('percentage', self.percentage); return self; }; -Async.prototype.cancel = function(name) { +ACP.cancel = function(name) { var self = this; @@ -5250,7 +5262,7 @@ Async.prototype.cancel = function(name) { return true; }; -Async.prototype.await = function(name, fn, cb) { +ACP.await = function(name, fn, cb) { var self = this; @@ -5273,7 +5285,7 @@ Async.prototype.await = function(name, fn, cb) { return true; }; -Async.prototype.wait = function(name, waitingFor, fn, cb) { +ACP.wait = function(name, waitingFor, fn, cb) { var self = this; @@ -5296,34 +5308,34 @@ Async.prototype.wait = function(name, waitingFor, fn, cb) { return true; }; -Async.prototype.complete = function(fn) { +ACP.complete = function(fn) { return this.run(fn); }; -Async.prototype.run = function(fn) { +ACP.run = function(fn) { this._isRunning = true; fn && this.onComplete.push(fn); this.refresh(); return this; }; -Async.prototype.isRunning = function(name) { +ACP.isRunning = function(name) { if (!name) return this._isRunning; var task = this.tasksPending[name]; return task ? task.isRunning === 1 : false; }; -Async.prototype.isWaiting = function(name) { +ACP.isWaiting = function(name) { var task = this.tasksPending[name]; return task ? task.isRunning === 0 : false; }; -Async.prototype.isPending = function(name) { +ACP.isPending = function(name) { return this.tasksPending[name] ? true : false; }; -Async.prototype.timeout = function(name, timeout) { +ACP.timeout = function(name, timeout) { if (timeout) this.tasksTimeout[name] = timeout; else @@ -5331,7 +5343,7 @@ Async.prototype.timeout = function(name, timeout) { return this; }; -Async.prototype.refresh = function(name) { +ACP.refresh = function(name) { var self = this; @@ -5398,14 +5410,16 @@ function FileList() { this.advanced = false; } -FileList.prototype.reset = function() { +const FLP = FileList.prototype; + +FLP.reset = function() { this.file.length = 0; this.directory.length = 0; this.pendingDirectory.length = 0; return this; }; -FileList.prototype.walk = function(directory) { +FLP.walk = function(directory) { var self = this; @@ -5427,7 +5441,7 @@ FileList.prototype.walk = function(directory) { }); }; -FileList.prototype.stat = function(path) { +FLP.stat = function(path) { var self = this; Fs.stat(path, function(err, stats) { @@ -5448,11 +5462,11 @@ FileList.prototype.stat = function(path) { }); }; -FileList.prototype.clean = function(path) { +FLP.clean = function(path) { return path[path.length - 1] === '/' ? path : path + '/'; }; -FileList.prototype.next = function() { +FLP.next = function() { var self = this; if (self.pending.length) { @@ -5856,7 +5870,9 @@ function EventEmitter2() { this.$events = {}; } -EventEmitter2.prototype.emit = function(name, a, b, c, d, e, f, g) { +const EE2P = EventEmitter2.prototype; + +EE2P.emit = function(name, a, b, c, d, e, f, g) { var evt = this.$events[name]; if (evt) { var clean = false; @@ -5876,7 +5892,7 @@ EventEmitter2.prototype.emit = function(name, a, b, c, d, e, f, g) { return this; }; -EventEmitter2.prototype.on = function(name, fn) { +EE2P.on = function(name, fn) { if (this.$events[name]) this.$events[name].push(fn); else @@ -5884,12 +5900,12 @@ EventEmitter2.prototype.on = function(name, fn) { return this; }; -EventEmitter2.prototype.once = function(name, fn) { +EE2P.once = function(name, fn) { fn.$once = true; return this.on(name, fn); }; -EventEmitter2.prototype.removeListener = function(name, fn) { +EE2P.removeListener = function(name, fn) { var evt = this.$events[name]; if (evt) { evt = evt.remove(n => n === fn); @@ -5901,7 +5917,7 @@ EventEmitter2.prototype.removeListener = function(name, fn) { return this; }; -EventEmitter2.prototype.removeAllListeners = function(name) { +EE2P.removeAllListeners = function(name) { if (name === true) this.$events = EMPTYOBJECT; else if (name) @@ -5928,7 +5944,9 @@ function Chunker(name, max) { this.filename = F.path.temp(this.filename); } -Chunker.prototype.append = Chunker.prototype.write = function(obj) { +const CHP = Chunker.prototype; + +CHP.append = CHP.write = function(obj) { var self = this; self.stack.push(obj); @@ -5956,7 +5974,7 @@ Chunker.prototype.append = Chunker.prototype.write = function(obj) { return self; }; -Chunker.prototype.end = function() { +CHP.end = function() { var self = this; var tmp = self.stack.length; if (tmp) { @@ -5979,7 +5997,7 @@ Chunker.prototype.end = function() { return self; }; -Chunker.prototype.each = function(onItem, onEnd, indexer) { +CHP.each = function(onItem, onEnd, indexer) { var self = this; @@ -5999,7 +6017,7 @@ Chunker.prototype.each = function(onItem, onEnd, indexer) { return self; }; -Chunker.prototype.read = function(index, callback) { +CHP.read = function(index, callback) { var self = this; if (self.flushing) { @@ -6034,7 +6052,7 @@ Chunker.prototype.read = function(index, callback) { return self; }; -Chunker.prototype.clear = function() { +CHP.clear = function() { var files = []; for (var i = 0; i < this.index; i++) files.push(this.filename + i + '.chunker'); @@ -6042,7 +6060,7 @@ Chunker.prototype.clear = function() { return this; }; -Chunker.prototype.destroy = function() { +CHP.destroy = function() { this.clear(); this.indexer = 0; this.flushing = 0; @@ -6079,13 +6097,14 @@ function Callback(count, callback) { this.pending = count; this.$callback = callback; } +const CP = Callback.prototype; -Callback.prototype.done = function(callback) { +CP.done = function(callback) { this.$callback = callback; return this; }; -Callback.prototype.next = function() { +CP.next = function() { var self = this; self.pending--; if (!self.pending && self.$callback) { @@ -6101,5 +6120,37 @@ exports.Callback = function(count, callback) { return new Callback(count, callback); }; +function Reader() {} +const RP = Reader.prototype; + +RP.push = function(data) { + if (data == null || !data.length) + this.reader.done(); + else + this.reader.compare(data); + return this; +}; + +RP.find = function() { + var self = this; + var builder = new framework_nosql.DatabaseBuilder(); + self.reader = new framework_nosql.NoSQLReader(builder); + return builder; +}; + +RP.count = function() { + var builder = this.find(); + builder.$options.readertype = 1; + return builder; +}; + +RP.scalar = function(type, field) { + return this.find().scalar(type, field); +}; + +exports.reader = function() { + return new Reader(); +}; + global.WAIT = exports.wait; !global.F && require('./index'); \ No newline at end of file From d7da68c32f10d3e97216439988ed99585f31b609 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 24 Aug 2018 00:02:50 +0200 Subject: [PATCH 019/217] New improvements again. --- changes.txt | 3 ++- utils.js | 10 ++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/changes.txt b/changes.txt index 443b22595..e309a609e 100755 --- a/changes.txt +++ b/changes.txt @@ -1,7 +1,8 @@ ======= 3.0.1 - added: CSS variables support default values `border-radius: $radius || 10px` -- added: NoSQL storage `.find()` + `.count()` with multiple thread support +- added: NoSQL storage `.find()` + `.count()` + '.scalar(type, field)' with multiple thread support +- added: `U.reader()` - improved: NoSQL reader diff --git a/utils.js b/utils.js index a8b35f079..3c3822dd0 100755 --- a/utils.js +++ b/utils.js @@ -99,6 +99,10 @@ for (var i=0; i 22 && l < 30 && this[l] === 'Z' && this[10] === 'T' && this[4] === '-' && this[13] === ':' && this[16] === ':'; @@ -4100,8 +4100,6 @@ SP.removeTags = function() { return this.replace(regexpTags, ''); }; -const NP = Number.prototype; - NP.floor = function(decimals) { return Math.floor(this * Math.pow(10, decimals)) / Math.pow(10, decimals); }; From b2066a2ff4d9933fdf8b29b68beae9bb82b45c28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 24 Aug 2018 09:04:19 +0200 Subject: [PATCH 020/217] Updated bundles. --- bundles.js | 9 +++++++-- changes.txt | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/bundles.js b/bundles.js index f390944cf..e04a46443 100755 --- a/bundles.js +++ b/bundles.js @@ -6,6 +6,7 @@ const CONSOLE = process.argv.indexOf('restart') === -1; const META = {}; const INTERNAL = { '/sitemap': 1, '/versions': 1, '/workflows': 1, '/dependencies': 1, '/config': 1, '/config-release': 1, '/config-debug': 1 }; const isWindows = require('os').platform().substring(0, 3).toLowerCase() === 'win'; +const REGAPPEND = /\/--[a-z0-9]+/i; META.version = 1; META.created = new Date(); @@ -112,11 +113,15 @@ exports.make = function(callback) { for (var i = 0, length = files.length; i < length; i++) { var file = files[i].substring(Length); var type = 0; - if (file.startsWith(F.config['directory-databases'])) type = 1; else if (file.startsWith(F.config['directory-public'])) type = 2; + else if (REGAPPEND.test(file)) { + file = file.replace(/\/--/g, '/'); + type = 3; + } + Files.push({ name: file, filename: files[i], type: type }); } @@ -216,7 +221,7 @@ function copyFiles(files, callback) { var filename = Path.join(path, file.name); var exists = false; var ext = U.getExtension(file.name); - var append = false; + var append = file.type === 3; try { exists = Fs.statSync(filename) != null; diff --git a/changes.txt b/changes.txt index e309a609e..b8f85dfc5 100755 --- a/changes.txt +++ b/changes.txt @@ -3,6 +3,7 @@ - added: CSS variables support default values `border-radius: $radius || 10px` - added: NoSQL storage `.find()` + `.count()` + '.scalar(type, field)' with multiple thread support - added: `U.reader()` +- added: `bundles` supports merging files between bundle and project, project file must start with e.g. `--name.js` - improved: NoSQL reader From 48aa3625f60f14024519ecb1c561ca4bb631e5d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 24 Aug 2018 20:24:21 +0200 Subject: [PATCH 021/217] Fixed filtering. --- nosql.js | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/nosql.js b/nosql.js index 51f477d4b..76b3ad46c 100755 --- a/nosql.js +++ b/nosql.js @@ -21,7 +21,7 @@ /** * @module NoSQL - * @version 3.0.0 + * @version 3.0.1 */ 'use strict'; @@ -1356,12 +1356,12 @@ DP.$inmemory = function(callback) { var arr = data.toString('utf8').split('\n'); for (var i = 0, length = arr.length; i < length; i++) { var item = arr[i]; - if (!item) - continue; - try { - item = JSON.parse(item.trim(), jsonparser); - item && self.inmemory[view].push(item); - } catch (e) {} + if (item) { + try { + item = JSON.parse(item.trim(), jsonparser); + item && self.inmemory[view].push(item); + } catch (e) {} + } } callback(); @@ -1646,6 +1646,7 @@ DP.$update_inmemory = function() { change && self.$save(); setImmediate(function() { self.next(0); + filters.done(); change && self.$events.change && self.emit('change', 'update'); }); }); @@ -1862,8 +1863,9 @@ function nosqlresort(arr, builder, doc) { DP.$reader2_inmemory = function(items, callback) { var self = this; - var filters = new NoSQLReader(items); return self.$inmemory(function() { + var filters = new NoSQLReader(items); + filters.clone = true; filters.compare(self.inmemory['#']); filters.done(); callback(); @@ -2255,7 +2257,8 @@ DatabaseBuilder.prototype.$callback2 = function(err, response, count, repository if (err || !self.$join) { self.$options.log && self.log(); self.$done && setImmediate(self.$done); - return self.$callback(err, response, count, repository); + self.$callback && self.$callback(err, response, count, repository); + return; } self.$response = response; @@ -2280,8 +2283,8 @@ DatabaseBuilder.prototype.$callback2 = function(err, response, count, repository } self.$options.log && self.log(); - self.$callback(err, response, count, repository); self.$done && setImmediate(self.$done); + self.$callback && self.$callback(err, response, count, repository); }); return self; @@ -6502,7 +6505,7 @@ NoSQLReader.prototype.compare = function(docs) { var self = this; for (var i = 0; i < docs.length; i++) { - var doc = docs[i]; + var doc = self.clone ? U.clone(docs[i]) : docs[i]; if (self.builders.length === self.canceled) return false; From 5e434c898b7208d653b2a2f510a302accc8d290e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 25 Aug 2018 11:26:57 +0200 Subject: [PATCH 022/217] Improved WebSocket. --- index.js | 90 ++++++++++++++++++++++++++++------------------ websocketclient.js | 3 ++ 2 files changed, 58 insertions(+), 35 deletions(-) diff --git a/index.js b/index.js index 2c666b859..f5516e3ac 100755 --- a/index.js +++ b/index.js @@ -7451,26 +7451,20 @@ F.$websocketcontinue = function(req, path) { var auth = F.onAuthorize; if (auth) { auth.call(F, req, req.websocket, req.flags, function(isLogged, user) { - if (user) req.user = user; - var route = F.lookup_websocket(req, req.websocket.uri.pathname, isLogged ? 1 : 2); if (route) { F.$websocketcontinue_process(route, req, path); - } else { - req.websocket.close(); - req.connection.destroy(); - } + } else + req.websocket.$close(4001, '401: unauthorized'); }); } else { var route = F.lookup_websocket(req, req.websocket.uri.pathname, 0); if (route) { F.$websocketcontinue_process(route, req, path); - } else { - req.websocket.close(); - req.connection.destroy(); - } + } else + req.websocket.$close(4004, '404: not found'); } }; @@ -7479,8 +7473,7 @@ F.$websocketcontinue_process = function(route, req, path) { var socket = req.websocket; if (!socket.prepare(route.flags, route.protocols, route.allow, route.length)) { - socket.close(); - req.connection.destroy(); + socket.$close(4001, '401: unauthorized'); return; } @@ -13583,6 +13576,28 @@ WebSocketClient.prototype.cookie = function(name) { return this.req.cookie(name); }; +WebSocketClient.prototype.$close = function(code, message) { + + var self = this; + + if ((self.req.headers['user-agent'] || '').indexOf('Total.js') !== -1) { + self.close(); + return; + } + + var header = SOCKET_RESPONSE.format(self.$websocket_key(self.req)); + self.socket.write(U.createBuffer(header, 'binary')); + self.ready = true; + self.close(message, code); + + setTimeout(function(self) { + self.req = null; + self.socket = null; + }, 1000, self); + + return self; +}; + WebSocketClient.prototype.prepare = function(flags, protocols, allow, length) { flags = flags || EMPTYARRAY; @@ -13623,6 +13638,7 @@ WebSocketClient.prototype.prepare = function(flags, protocols, allow, length) { var header = protocols.length ? (compress ? SOCKET_RESPONSE_PROTOCOL_COMPRESS : SOCKET_RESPONSE_PROTOCOL).format(self.$websocket_key(self.req), protocols.join(', ')) : (compress ? SOCKET_RESPONSE_COMPRESS : SOCKET_RESPONSE).format(self.$websocket_key(self.req)); self.socket.write(U.createBuffer(header, 'binary')); + self.ready = true; if (compress) { self.inflatepending = []; @@ -13630,10 +13646,10 @@ WebSocketClient.prototype.prepare = function(flags, protocols, allow, length) { self.inflate = Zlib.createInflateRaw(WEBSOCKET_COMPRESS_OPTIONS); self.inflate.$websocket = self; self.inflate.on('error', function() { - if (self.$uerror) - return; - self.$uerror = true; - self.close('Unexpected error'); + if (!self.$uerror) { + self.$uerror = true; + self.close('Invalid data', 1003); + } }); self.inflate.on('data', websocket_inflate); @@ -13642,10 +13658,10 @@ WebSocketClient.prototype.prepare = function(flags, protocols, allow, length) { self.deflate = Zlib.createDeflateRaw(WEBSOCKET_COMPRESS_OPTIONS); self.deflate.$websocket = self; self.deflate.on('error', function() { - if (self.$uerror) - return; - self.$uerror = true; - self.close('Unexpected error'); + if (!self.$uerror) { + self.$uerror = true; + self.close('Invalid data', 1003); + } }); self.deflate.on('data', websocket_deflate); } @@ -13833,9 +13849,8 @@ WebSocketClient.prototype.$parse = function() { // total message length (data + header) var mlength = index + length; - // ??? if (mlength > this.length) { - this.close('Maximum request length exceeded.'); + this.close('Frame is too large', 1009); return; } @@ -13963,7 +13978,7 @@ WebSocketClient.prototype.parseInflate = function() { self.inflatelock = false; if (data.length > self.length) { - self.close('Maximum request length exceeded.'); + self.close('Frame is too large', 1009); return; } @@ -14062,14 +14077,14 @@ WebSocketClient.prototype.sendDeflate = function() { self.deflatelock = true; self.deflate.write(buf); self.deflate.flush(function() { - if (!self.deflatechunks) - return; - var data = buffer_concat(self.deflatechunks, self.deflatechunkslength); - data = data.slice(0, data.length - 4); - self.deflatelock = false; - self.deflatechunks = null; - self.socket.write(U.getWebSocketFrame(0, data, self.type === 1 ? 0x02 : 0x01, true)); - self.sendDeflate(); + if (self.deflatechunks) { + var data = buffer_concat(self.deflatechunks, self.deflatechunkslength); + data = data.slice(0, data.length - 4); + self.deflatelock = false; + self.deflatechunks = null; + self.socket.write(U.getWebSocketFrame(0, data, self.type === 1 ? 0x02 : 0x01, true)); + self.sendDeflate(); + } }); } }; @@ -14093,11 +14108,16 @@ WebSocketClient.prototype.ping = function() { * @return {WebSocketClient} */ WebSocketClient.prototype.close = function(message, code) { - if (!this.isClosed) { - this.isClosed = true; - this.socket.end(U.getWebSocketFrame(code || 1000, message ? (F.config['default-websocket-encodedecode'] ? encodeURIComponent(message) : message) : '', 0x08)); + var self = this; + if (!self.isClosed) { + self.isClosed = true; + if (self.ready) + self.socket.end(U.getWebSocketFrame(code || 1000, message ? (F.config['default-websocket-encodedecode'] ? encodeURIComponent(message) : message) : '', 0x08)); + else + self.socket.end(); + self.req.connection.destroy(); } - return this; + return self; }; /** diff --git a/websocketclient.js b/websocketclient.js index 5cbb89827..5ecce29b6 100644 --- a/websocketclient.js +++ b/websocketclient.js @@ -69,6 +69,7 @@ WebSocketClient.prototype.connect = function(url, protocol, origin) { options.path = url.path; options.query = url.query; options.headers = {}; + options.headers['User-Agent'] = 'Total.js/v' + F.version_header; options.headers['Sec-WebSocket-Version'] = '13'; options.headers['Sec-WebSocket-Key'] = key; options.headers['Sec-Websocket-Extensions'] = (self.options.compress ? 'permessage-deflate, ' : '') + 'client_max_window_bits'; @@ -157,6 +158,8 @@ function websocket_close() { ws.closed = true; ws.$onclose(); ws.options.reconnect && setTimeout(function(ws) { + ws.isClosed = false; + ws._isClosed = false; ws.reconnect++; ws.connect(ws.url, ws.protocol, ws.origin); }, ws.options.reconnect, ws); From 30dea7b6ac4bf955ad62eb0e3c235091693c90ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 25 Aug 2018 11:34:15 +0200 Subject: [PATCH 023/217] Updated `debug.js` by changing `debug.pid`. --- debug.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/debug.js b/debug.js index d7363e4f8..609821efa 100644 --- a/debug.js +++ b/debug.js @@ -21,7 +21,7 @@ /** * @module FrameworkDebug - * @version 3.0.0 + * @version 3.0.1 */ const Path = require('path'); @@ -91,6 +91,7 @@ function runwatching() { const isRELOAD = !!options.livereload; const SPEED = isRELOAD ? 1000 : 1500; const ARGV = CLONE(process.argv); + const PIDNAME = FILENAME.replace(/\.js$/, '.pid'); function copyFile(oldname, newname, callback) { var writer = Fs.createWriteStream(newname); @@ -149,6 +150,7 @@ function runwatching() { var WS = null; var speed = isRELOAD ? 1000 : 4000; + blacklist['/' + PIDNAME] = 1; blacklist['/debug.pid'] = 1; blacklist['/debug.js'] = 1; blacklist['/bundle.json'] = 1; @@ -408,7 +410,7 @@ function runwatching() { console.log(prefix.substring(8) + 'DEBUG PID: ' + process.pid + ' (v' + VERSION + ')'); - pid = Path.join(directory, 'debug.pid'); + pid = Path.join(directory, PIDNAME); Fs.writeFileSync(pid, process.pid); setInterval(function() { @@ -434,7 +436,7 @@ function runwatching() { refresh_directory(); } - var filename = Path.join(directory, 'debug.pid'); + var filename = Path.join(directory, PIDNAME); if (Fs.existsSync(filename)) { Fs.unlinkSync(filename); setTimeout(app, 3500); From b2953d8d38ec7c7f89d41809c9f22ce88a6b1d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 25 Aug 2018 11:34:19 +0200 Subject: [PATCH 024/217] Added new changes. --- changes.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index b8f85dfc5..69b618c0e 100755 --- a/changes.txt +++ b/changes.txt @@ -5,12 +5,15 @@ - added: `U.reader()` - added: `bundles` supports merging files between bundle and project, project file must start with e.g. `--name.js` -- improved: NoSQL reader +- updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function - fixed: `F.wait()` in the test mode - fixed: `LOCALIZE()` for nested directories +- fixed: sending of error handling when WebSocketClient is starting (for example: `unauthorized`) + +- improved: NoSQL reader ======= 3.0.0 From 02912d2ae787f0408a438bfc70f922b0f4dc954f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 26 Aug 2018 13:40:31 +0200 Subject: [PATCH 025/217] Fixed `.remove()` method in `TABLE`. --- nosql.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index 76b3ad46c..ada884b06 100755 --- a/nosql.js +++ b/nosql.js @@ -5953,13 +5953,13 @@ TP.$remove = function() { var data = { keys: self.$keys }; var remove = function(docs, d, dindex, f) { - var rec = fs.docsbuffer[dindex + 1]; + var rec = fs.docsbuffer[dindex]; f.builder.$options.backup && f.builder.$backupdoc(rec.doc); return 1; }; var removeflush = function(docs, d, dindex) { - var rec = fs.docsbuffer[dindex + 1]; + var rec = fs.docsbuffer[dindex]; !change && (change = true); self.$events.remove && self.emit('remove', d); fs.write(fs.remchar + rec.doc.substring(1) + NEWLINE, rec.position); @@ -5970,6 +5970,9 @@ TP.$remove = function() { var lines = fs.docs.split(fs.divider); var arr = []; + if (!indexer) + arr.push(EMPTYOBJECT); + for (var a = indexer ? 0 : 1; a < lines.length; a++) { data.line = lines[a].split('|'); data.index = indexer++; From 112b815c3126a2bc1e0db36c91826b69c9e9f2d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 26 Aug 2018 20:26:06 +0200 Subject: [PATCH 026/217] Removed GraphDB. --- graphdb.js | 2582 ---------------------------------------------------- 1 file changed, 2582 deletions(-) delete mode 100644 graphdb.js diff --git a/graphdb.js b/graphdb.js deleted file mode 100644 index 1917c48f3..000000000 --- a/graphdb.js +++ /dev/null @@ -1,2582 +0,0 @@ -// Copyright 2012-2018 (c) Peter Širka -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -/** - * @module FrameworkGraphDB - * @version 1.0.0 - */ - -const Fs = require('fs'); -const Zlib = require('zlib'); - -const ZLIBOPTIONS = { level: Zlib.constants.Z_FULL_FLUSH, memLevel: Zlib.constants.Z_BEST_COMPRESSION, strategy: Zlib.constants.Z_DEFAULT_STRATEGY }; -const VERSION = 1; -const DOCUMENTSIZE = 1000; -const PAGESIZE = 20; -const PAGELIMIT = 50; -const DATAOFFSET = 17; -const EMPTYBUFFER = U.createBufferSize(1); -const HEADERSIZE = 7000; -const DELAY = 100; -const REGTUNESCAPE = /%7C|%0D|%0A/g; -const REGTESCAPETEST = /\||\n|\r/; -const REGTESCAPE = /\||\n|\r/g; -const BOOLEAN = { '1': 1, 'true': 1, 'on': 1 }; -const DatabaseBuilder = framework_nosql.DatabaseBuilder; - -// STATES -const STATE_UNCOMPRESSED = 1; -const STATE_COMPRESSED = 2; -const STATE_REMOVED = 255; - -// META -const META_PAGE_ADD = 100; -const META_CLASSESRELATIONS = 101; -const META_PAGE_ADD3 = 102; -const META_RELATIONPAGEINDEX = 103; - -// OPERATIONS -const NEXT_READY = 1; -const NEXT_INSERT = 2; -const NEXT_RELATION = 3; -const NEXT_UPDATE = 4; -const NEXT_FIND = 5; -const NEXT_REMOVE = 6; -const NEXT_RESIZE = 7; -const NEXT_CONTINUE = 100; - -// TYPES -const TYPE_CLASS = 1; -const TYPE_RELATION = 2; -const TYPE_RELATION_DOCUMENT = 3; - -var IMPORTATOPERATIONS = 0; - -function GraphDB(name) { - - F.path.verify('databases'); - - var self = this; - self.name = name; - self.filename = F.path.databases(name + '.gdb'); - self.filenameBackup = self.filename.replace(/\.gdb$/, '.gdp-backup'); - self.ready = false; - - self.$classes = {}; - self.$relations = {}; - self.$events = {}; - - self.header = {}; - - self.pending = {}; - self.pending.insert = []; - self.pending.find = []; - self.pending.update = []; - self.pending.remove = []; - self.pending.relation = []; - self.pending.meta = []; - - self.states = {}; - self.states.resize = false; - self.states.insert = false; - self.states.read = false; - self.states.remove = false; - self.states.update = false; - - F.path.verify('databases'); - // t.open(); - - self.cb_error = function(err) { - err && console.log(err); - }; - - self.cb_next = function(value) { - self.next(value); - }; - - F.grapdbinstance = true; - self.open(); -} - -var GP = GraphDB.prototype; - -// ==== DB:HEADER (7000b) -// name (30b) = from: 0 -// version (1b) = from: 30 -// pages (4b) = from: 31 -// pagesize (2b) = from: 35 -// pagelimit (2b) = from: 37 -// documents (4b) = from: 39 -// documentsize (2b) = from: 43 -// classindex (1b) = from: 45 -// relationindex (1b) = from: 46 -// relationnodeindex = from: 47 -// classes + relations = from: 51 - -// ==== DB:PAGE (20b) -// type (1b) = from: 0 -// index (1b) = from: 1 -// documents (2b) = from: 2 -// freeslots (1b) = from: 4 -// parentindex (4b) = from: 5 - -// ==== DB:DOCUMENT (SIZE) -// type (1b) = from: 0 -// index (1b) = from: 1 -// state (1b) = from: 2 -// pageindex (4b) = from: 3 -// relationindex (4b) = from: 7 (it's for relations between two documents in TYPE_RELATION page) -// parentindex (4b) = from: 11 -// size/count (2b) = from: 15 -// data = from: 17 - -// Creates new page -function addPage(self, type, index, parentindex, callback) { - - // @type - // 1: classes - // 2: relations - // 3: relations private - - // @index - // index of value - - // Add a new page - self.header.pages++; - - var indexer = self.header.pages; - var buffer = []; - var page = U.createBufferSize(self.header.pagesize); - - // console.log('CREATING PAGE:', TYPES[type], indexer, type, index); - - page.writeUInt8(type, 0); // type (1:class, 2:relation, 3:private) - page.writeUInt8(index, 1); // index - page.writeUInt16LE(0, 2); // documents - page.writeUInt8(0, 4); // freeslots - page.writeUInt32LE(parentindex, 5); // parentindex - - buffer.push(page); - - for (var i = 0; i < self.header.pagelimit; i++) { - var doc = U.createBufferSize(self.header.documentsize); - doc.writeUInt8(type, 0); - doc.writeUInt8(index, 1); - doc.writeUInt8(STATE_REMOVED, 2); - doc.writeUInt32LE(self.header.pages, 3); - doc.writeUInt32LE(0, 7); // continuerindex - doc.writeUInt32LE(0, 11); // parentindex - doc.writeUInt16LE(0, 15); // size/count - buffer.push(doc); - } - - buffer = Buffer.concat(buffer); - - var offset = offsetPage(self, indexer); - - Fs.write(self.fd, buffer, 0, buffer.length, offset, function(err) { - err && self.error(err, 'createPage.write'); - !err && updMeta(self, type === TYPE_RELATION_DOCUMENT ? META_PAGE_ADD3 : META_PAGE_ADD); - callback && callback(err, indexer); - }); - - return indexer; -} - -function addNodeFree(self, meta, callback) { - - if (!meta.type.findfreeslots) { - addNode(self, meta, callback); - return; - } - - findDocumentFree(self, meta.type.pageindex, function(err, documentindex, pageindex) { - - if (!documentindex) { - meta.type.findfreeslots = false; - addNode(self, meta, callback); - return; - } - - var buffer = U.createBufferSize(self.header.documentsize); - buffer.writeUInt8(meta.typeid, 0); // type - buffer.writeUInt8(meta.type.index, 1); // index - buffer.writeUInt32LE(pageindex, 3); // pageindex - buffer.writeUInt8(meta.state || STATE_UNCOMPRESSED, 2); // state - buffer.writeUInt32LE(meta.relationindex || 0, 7); // relationindex - buffer.writeUInt32LE(meta.parentindex || 0, 11); // parentindex - buffer.writeUInt16LE(meta.size, 15); - meta.data && meta.data.copy(buffer, DATAOFFSET); - - Fs.write(self.fd, buffer, 0, buffer.length, offsetDocument(self, documentindex), function() { - meta.type.locked = false; - callback(null, documentindex, pageindex); - }); - }); -} - -function addNode(self, meta, callback) { - - // meta.typeid (1 CLASS, 2 RELATION) - // meta.type (link to type class/relation) - // meta.state - // meta.parentindex - // meta.relationindex - // meta.size - // meta.buffer - - var buf = U.createBufferSize(self.header.pagesize); - var offset = offsetPage(self, meta.type.pageindex); - - meta.type.locked = true; - - Fs.read(self.fd, buf, 0, buf.length, offset, function(err) { - - if (err) - throw err; - - if (buf[0] !== meta.typeid) - throw new Error('Not a class page'); - - if (!meta.type.private && buf[1] !== meta.type.index) - throw new Error('Not same class type'); - - // type : buf[0] - // index : buf[1] - // documents : buf.readUInt16LE(2) - // freeslots : buf[4] - // parentindex : readUInt32LE(5) - - var buffer = U.createBufferSize(self.header.documentsize); - buffer.writeUInt8(buf[0], 0); // type - buffer.writeUInt8(meta.type.index, 1); // index - buffer.writeUInt32LE(meta.type.pageindex, 3); // pageindex - buffer.writeUInt8(meta.state || STATE_UNCOMPRESSED, 2); // state - buffer.writeUInt32LE(meta.relationindex || 0, 7); // relationindex - buffer.writeUInt32LE(meta.parentindex || 0, 11); // parentindex - buffer.writeUInt16LE(meta.size, 15); - meta.data && meta.data.copy(buffer, DATAOFFSET); - - var documents = buf.readUInt16LE(2); - var documentsbuf = U.createBufferSize(2); - - documents++; - documentsbuf.writeUInt16LE(documents); - - Fs.write(self.fd, documentsbuf, 0, documentsbuf.length, offset + 2, function(err) { - - err && console.log('addNode.write.meta', err); - Fs.write(self.fd, buffer, 0, buffer.length, offset + self.header.pagesize + ((documents - 1) * self.header.documentsize), function(err) { - - err && console.log('addNode.write.data', err); - - // type (1b) = from: 0 - // index (1b) = from: 1 - // state (1b) = from: 2 - // pageindex (4b) = from: 3 - // continuerindex (4b) = from: 7 - // parentindex (4b) = from: 11 - // size/count (2b) = from: 15 - // data = from: 17 - - // We must create a new page - if (documents + 1 > self.header.pagelimit) { - addPage(self, meta.typeid, meta.type.index, meta.type.pageindex, function(err, index) { - - var documentindex = getDocumentIndex(self, meta.type.pageindex, documents); - meta.type.documentindex = documentindex; - meta.type.pageindex = index; - meta.type.locked = false; - - // Problem with classes - // meta.type.index = 0; - - if (meta.type.private) - self.header.relationpageindex = index; - - updMeta(self, meta.type.private ? META_RELATIONPAGEINDEX : META_CLASSESRELATIONS); - callback(null, documentindex, index); - }); - } else { - var documentindex = getDocumentIndex(self, meta.type.pageindex, documents); - meta.type.locked = false; - meta.type.documentindex = documentindex; - callback(null, documentindex, meta.type.pageindex); - } - }); - }); - }); -} - -function addDocument(self, cls, value, callback) { - - // meta.typeid (1 CLASS, 2 RELATION) - // meta.type (link to type class/relation) - // meta.state - // meta.parentindex - // meta.relationindex - // meta.size - // meta.data - - var meta = {}; - meta.type = cls; - meta.typeid = TYPE_CLASS; - meta.state = 1; - meta.parentindex = 0; - meta.relationindex = 0; - meta.data = U.createBuffer(stringifyData(cls.schema, value)); - meta.size = meta.data.length; - - var limit = self.header.documentsize - DATAOFFSET; - - if (meta.data.length > limit) { - Zlib.deflate(meta.data, ZLIBOPTIONS, function(err, buf) { - if (err || buf.length > limit) - callback(new Error('GraphDB: Data too long'), 0); - else { - meta.state = STATE_COMPRESSED; - meta.data = buf; - meta.size = buf.length; - addNodeFree(self, meta, callback); - } - }); - } else - addNodeFree(self, meta, callback); -} - -function addRelation(self, relation, indexA, indexB, callback) { - - // Workflow: - // Has "A" relation nodes? - // Has "B" relation nodes? - // Create "A" relation with "B" - // Create "B" relation with "A" - // Register relation to global relations - - var tasks = []; - var relA = null; - var relB = null; - - var tmprelation = { index: relation.index, pageindex: 0, documentindex: 0, locked: false, private: true }; - - tasks.push(function(next) { - self.read(indexA, function(err, doc, relid) { - if (doc) { - relA = relid; - next(); - } else { - tasks = null; - next = null; - callback(new Error('GraphDB: Node (A) "{0}" not exists.'.format(indexA))); - } - }); - }); - - tasks.push(function(next) { - self.read(indexB, function(err, doc, relid) { - if (doc) { - relB = relid; - next(); - } else { - tasks = null; - next = null; - callback(new Error('GraphDB: Node (B) "{0}" not exists.'.format(indexB))); - } - }); - }); - - tasks.push(function(next) { - - if (relA == 0) { - next(); - return; - } - - checkRelation(self, relation, relA, indexB, function(err, is) { - if (is) { - tasks = null; - next = null; - callback(new Error('GraphDB: Same relation already exists between nodes (A) "{0}" and (B) "{1}".'.format(indexA, indexB))); - } else - next(); - }); - }); - - // Obtaining indexA a relation document - tasks.push(function(next) { - - if (F.isKilled) - return; - - IMPORTATOPERATIONS++; - - if (relA) - next(); - else { - addRelationDocument(self, relation, indexA, function(err, index) { - relA = index; - next(); - }, true); - } - }); - - // Obtaining indexB a relation document - tasks.push(function(next) { - - if (F.isKilled) - return; - - if (relB) - next(); - else { - addRelationDocument(self, relation, indexB, function(err, index) { - relB = index; - next(); - }, true); - } - }); - - // Push "indexB" relation to "indexA" - tasks.push(function(next) { - tmprelation.documentindex = relA; - tmprelation.pageindex = self.header.relationpageindex; - pushRelationDocument(self, relA, tmprelation, indexB, true, function(err, index) { - // Updated relation, document was full - if (relA !== index) { - relA = index; - updDocumentRelation(self, indexA, relA, next); - } else - next(); - }, true); - }); - - tasks.push(function(next) { - tmprelation.documentindex = relB; - tmprelation.pageindex = self.header.relationpageindex; - pushRelationDocument(self, relB, tmprelation, indexA, false, function(err, index) { - // Updated relation, document was full - if (relB !== index) { - relB = index; - updDocumentRelation(self, indexB, relB, next); - } else - next(); - }, true); - }); - - tasks.push(function(next) { - // console.log('PUSH COMMON', relation.documentindex, indexA); - pushRelationDocument(self, relation.documentindex, relation, indexA, true, next); - }); - - tasks.async(function() { - IMPORTATOPERATIONS--; - // console.log('REL ====', relA, relB); - callback(null, true); - }); -} - -function remRelation(self, relation, indexA, indexB, callback) { - - var tasks = []; - var relA = null; - var relB = null; - - tasks.push(function(next) { - self.read(indexA, function(err, doc, relid) { - if (doc) { - relA = relid; - next(); - } else { - tasks = null; - next = null; - callback(new Error('GraphDB: Node (A) "{0}" not exists.'.format(indexA))); - } - }); - }); - - tasks.push(function(next) { - self.read(indexB, function(err, doc, relid) { - if (doc) { - relB = relid; - next(); - } else { - tasks = null; - next = null; - callback(new Error('GraphDB: Node (B) "{0}" not exists.'.format(indexB))); - } - }); - }); - - tasks.async(function() { - - if (F.isKilled) - return; - - IMPORTATOPERATIONS++; - remRelationLink(self, relA, indexB, function(err, countA) { - remRelationLink(self, relB, indexA, function(err, countB) { - remRelationLink(self, relation.documentindex, indexA, function(err, countC) { - IMPORTATOPERATIONS--; - callback(null, (countA + countB + countC) > 1); - }); - }); - }); - }); -} - -function remRelationLink(self, index, documentindex, callback, nochild, counter) { - - var buf = U.createBufferSize(self.header.documentsize); - var offset = offsetDocument(self, index); - - !counter && (counter = 0); - - Fs.read(self.fd, buf, 0, buf.length, offset, function() { - - // type (1b) = from: 0 - // index (1b) = from: 1 - // state (1b) = from: 2 - // pageindex (4b) = from: 3 - // relationindex (4b) = from: 7 (it's for relations between two documents in TYPE_RELATION page) - // parentindex (4b) = from: 11 - // size/count (2b) = from: 15 - // data = from: 17 - - if ((buf[0] !== TYPE_RELATION && buf[0] !== TYPE_RELATION_DOCUMENT) || (buf[2] === STATE_REMOVED)) { - callback(null, counter); - return; - } - - var relid = buf.readUInt32LE(7); - var count = buf.readUInt16LE(15); - var arr = []; - var is = false; - - for (var i = 0; i < count; i++) { - var off = DATAOFFSET + (i * 6); - var obj = {}; - obj.INDEX = buf[off]; - obj.INIT = buf[off + 1]; - obj.ID = buf.readUInt32LE(off + 2); - if (obj.ID === documentindex && obj.INIT === 1) - is = true; - else - arr.push(obj); - } - - if (is) { - count = arr.length; - for (var i = 0; i < count; i++) { - var off = DATAOFFSET + (i * 6); - var obj = arr[i]; - buf.writeUInt8(obj.INDEX, off); - buf.writeUInt8(obj.INIT, off + 1); - buf.writeUInt32LE(obj.ID, off + 2); - } - buf.writeUInt16LE(count, 15); - buf.fill(EMPTYBUFFER, DATAOFFSET + ((count + 1) * 6)); - Fs.write(self.fd, buf, 0, buf.length, offset, function() { - counter++; - if (relid && !nochild) - setImmediate(remRelationLink, self, relid, documentindex, callback, null, counter); - else - callback(null, counter); - }); - } else if (relid && !nochild) - setImmediate(remRelationLink, self, relid, documentindex, callback, null, counter); - else - callback(null, counter); - }); -} - -// Traverses all RELATIONS documents and remove specific "documentindex" -function remRelationAll(self, index, documentindex, callback, counter) { - - var buf = U.createBufferSize(self.header.pagelimit * self.header.documentsize); - var offset = offsetDocument(self, index); - - !counter && (counter = 0); - - Fs.read(self.fd, buf, 0, buf.length, offset, function(err, size) { - - if (err || !size) { - callback(null, counter); - return; - } - - // type (1b) = from: 0 - // index (1b) = from: 1 - // state (1b) = from: 2 - // pageindex (4b) = from: 3 - // relationindex (4b) = from: 7 (it's for relations between two documents in TYPE_RELATION page) - // parentindex (4b) = from: 11 - // size/count (2b) = from: 15 - // data = from: 17 - - var removed = []; - - while (true) { - - if (!buf.length) - break; - - index++; - - var data = buf.slice(0, self.header.documentsize); - - if ((data[0] !== TYPE_RELATION && data[0] !== TYPE_RELATION_DOCUMENT) || (data[2] === STATE_REMOVED)) { - buf = buf.slice(self.header.documentsize); - continue; - } - - var count = data.readUInt16LE(15); - var arr = []; - var is = false; - - for (var i = 0; i < count; i++) { - var off = DATAOFFSET + (i * 6); - var obj = {}; - obj.INDEX = data[off]; - obj.INIT = data[off + 1]; - obj.ID = data.readUInt32LE(off + 2); - if (obj.ID === documentindex) - is = true; - else - arr.push(obj); - } - - if (is) { - - var newcount = arr.length; - - for (var i = 0; i < newcount; i++) { - var off = DATAOFFSET + (i * 6); - var obj = arr[i]; - data.writeUInt8(obj.INDEX, off); - data.writeUInt8(obj.INIT, off + 1); - data.writeUInt32LE(obj.ID, off + 2); - } - - data.writeUInt16LE(newcount, 15); - data.fill(EMPTYBUFFER, DATAOFFSET + ((newcount + 1) * 6)); - - removed.push({ index: index - 1, buf: data }); - } - - buf = buf.slice(self.header.documentsize); - } - - if (!removed.length) { - setImmediate(remRelationAll, self, index, documentindex, callback, counter); - return; - } - - counter += removed.length; - removed.wait(function(item, next) { - Fs.write(self.fd, item.buf, 0, item.buf.length, offsetDocument(self, item.index), next); - }, function() { - setImmediate(remRelationAll, self, index, documentindex, callback, counter); - }); - - }); -} - -function addRelationDocument(self, relation, index, callback, between) { - - // meta.typeid (1 CLASS, 2 RELATION, 3 PRIVATE RELATION) - // meta.type (link to type class/relation) - // meta.state - // meta.parentindex - // meta.relationindex - // meta.size - // meta.data - - var meta = {}; - meta.typeid = between ? TYPE_RELATION_DOCUMENT : TYPE_RELATION; - meta.type = between ? { index: 0, pageindex: self.header.relationpageindex, documentindex: index, locked: false, private: true } : relation; - meta.state = 1; - meta.parentindex = 0; - meta.relationindex = 0; - meta.size = 0; - - // Creates a new node - addNode(self, meta, function(err, relationindex) { - - // Updates exiting document by updating relation index - updDocumentRelation(self, index, relationindex, function(err) { - // Returns a new relation index - callback(err, relationindex); - }); - }); -} - -function findDocumentFree(self, pageindex, callback, ready) { - - var offset = offsetPage(self, pageindex); - var buf = U.createBufferSize(self.header.pagesize); - - Fs.read(self.fd, buf, 0, buf.length, offset, function() { - - // ==== DB:PAGE (20b) - // type (1b) = from: 0 - // index (1b) = from: 1 - // documents (2b) = from: 2 - // freeslots (1b) = from: 4 - // parentindex (4b) = from: 5 - - var relid = buf.readUInt32LE(5); - if (!relid) { - if (!ready) { - callback(null, 0); - return; - } - } - - // First page is the last page saved in meta therefore is needed to perform recursive with "ready" - if (!ready) { - findDocumentFree(self, relid, callback, true); - return; - } - - var documents = buf.readUInt16LE(2); - if (documents >= self.header.pagelimit) { - // Finds in parent if exists - if (relid) - findDocumentFree(self, relid, callback, true); - else - callback(null, 0); - return; - } - - // Finds a free document slot - var index = getDocumentIndex(self, pageindex) - 1; - var buffer = U.createBufferSize(self.header.pagelimit * self.header.documentsize); - - Fs.read(self.fd, buffer, 0, buffer.length, offset + self.header.pagesize, function() { - while (true) { - index++; - var data = buffer.slice(0, self.header.documentsize); - if (!data.length) - break; - - if (data[2] === STATE_REMOVED) { - - if (F.isKilled) - return; - - updPageMeta(self, pageindex, function(err, buf) { - buf.writeUInt16LE(documents + 1, 2); - setImmediate(callback, null, index, pageindex); - }); - buffer = buffer.slice(self.header.documentsize); - return; - } - } - - if (relid) - findDocumentFree(self, relid, callback, true); - else - callback(null, 0); - - }); - }); -} - -// Finds a free space for new relation in "pushRelationDocument" -function findRelationDocument(self, relid, callback) { - - if (!relid) { - callback(null, 0); - return; - } - - var offset = offsetDocument(self, relid); - var buf = U.createBufferSize(self.header.documentsize); - - Fs.read(self.fd, buf, 0, buf.length, offset, function(err, size) { - - if (err || !size) { - callback(err, 0); - return; - } - - var count = buf.readUInt16LE(15); - if (count + 1 > self.header.relationlimit) { - // Checks if the relation index has next relation - - if (relid === buf.readUInt32LE(7)) - return; - - relid = buf.readUInt32LE(7); - if (relid) - setImmediate(findRelationDocument, self, relid, callback); - else - callback(null, 0); - } else { - // Free space for this relation - callback(null, relid); - } - }); -} - -// Pushs "documentindex" to "index" document (document with all relations) -function pushRelationDocument(self, index, relation, documentindex, initializator, callback, between, recovered) { - - var offset = offsetDocument(self, index); - var buf = U.createBufferSize(self.header.documentsize); - - Fs.read(self.fd, buf, 0, buf.length, offset, function() { - - // type (1b) = from: 0 - // index (1b) = from: 1 - // state (1b) = from: 2 - // pageindex (4b) = from: 3 - // relationindex (4b) = from: 7 (it's for relations between two documents in TYPE_RELATION page) - // parentindex (4b) = from: 11 - // size/count (2b) = from: 15 - // data = from: 17 - - var count = buf.readUInt16LE(15); - if (count + 1 > self.header.relationlimit) { - findRelationDocument(self, buf.readUInt32LE(7), function(err, newindex) { - - // Is some relation document exist? - if (newindex && !recovered) { - pushRelationDocument(self, newindex, relation, documentindex, initializator, callback, between, true); - return; - } - - // meta.typeid (1 CLASS, 2 RELATION) - // meta.type (link to type class/relation) - // meta.state - // meta.parentindex - // meta.relationindex - // meta.size - // meta.buffer - - var meta = {}; - meta.typeid = relation.private ? TYPE_RELATION_DOCUMENT : TYPE_RELATION; - meta.type = relation; - meta.state = STATE_UNCOMPRESSED; - meta.parentindex = 0; - meta.relationindex = index; - meta.size = 0; - - addNode(self, meta, function(err, docindex, pageindex) { - relation.pageindex = pageindex; - relation.documentindex = docindex; - updDocumentRelation(self, relation.documentindex, index, function() { - updDocumentParent(self, index, relation.documentindex, function() { - pushRelationDocument(self, relation.documentindex, relation, documentindex, initializator, callback, between); - }); - }); - }); - }); - - } else { - - var buffer = U.createBufferSize(6); - buffer.writeUInt8(relation.index, 0); - buffer.writeUInt8(initializator ? 1 : 0, 1); - buffer.writeUInt32LE(documentindex, 2); - buffer.copy(buf, DATAOFFSET + (count * 6)); - buf.writeUInt16LE(count + 1, 15); - - if (buf[2] === STATE_REMOVED) { - // We must update counts of documents in the page meta - var pageindex = Math.ceil(index / self.header.pagelimit); - updPageMeta(self, pageindex, function(err, buf) { - - // type (1b) = from: 0 - // index (1b) = from: 1 - // documents (2b) = from: 2 - // freeslots (1b) = from: 4 - // parentindex (4b) = from: 5 - - buf.writeUInt16LE(buf.readUInt16LE(2) + 1, 2); - setImmediate(function() { - Fs.write(self.fd, buf, 0, buf.length, offset, function(err) { - err && self.error(err, 'pushRelationDocument.read.write'); - callback(null, index); - }); - }); - }); - - buf.writeUInt8(STATE_UNCOMPRESSED, 2); - - } else { - // DONE - Fs.write(self.fd, buf, 0, buf.length, offset, function(err) { - err && self.error(err, 'pushRelationDocument.read.write'); - callback(null, index); - }); - } - } - - }); -} - -function updDocumentRelation(self, index, relationindex, callback) { - - if (index === relationindex) - throw new Error('FET'); - - var offset = offsetDocument(self, index); - var buf = U.createBufferSize(4); - buf.writeUInt32LE(relationindex); - Fs.write(self.fd, buf, 0, buf.length, offset + 7, callback); -} - -function updDocumentParent(self, index, parentindex, callback) { - var offset = offsetDocument(self, index); - var buf = U.createBufferSize(4); - buf.writeUInt32LE(parentindex); - Fs.write(self.fd, buf, 0, buf.length, offset + 11, callback); -} - -function updPageMeta(self, index, fn) { - var offset = offsetPage(self, index); - var buf = U.createBufferSize(self.header.pagesize); - Fs.read(self.fd, buf, 0, buf.length, offset, function() { - fn(null, buf); - Fs.write(self.fd, buf, 0, buf.length, offset, self.cb_error); - }); -} - -function remDocument(self) { - if (!self.ready || self.states.remove || !self.pending.remove.length || F.isKilled) - return; - self.states.remove = true; - var doc = self.pending.remove.shift(); - IMPORTATOPERATIONS++; - remRelationAll(self, doc.id, doc.id, function() { - remDocumentAll(self, doc.id, function(err, count) { - IMPORTATOPERATIONS--; - self.states.remove = false; - doc.callback && doc.callback(err, count); - setImmediate(self.cb_next, NEXT_REMOVE); - }); - }); -} - -function remDocumentAll(self, index, callback, count) { - - var offset = offsetDocument(self, index); - var buf = U.createBufferSize(17); - - // type (1b) = from: 0 - // index (1b) = from: 1 - // state (1b) = from: 2 - // pageindex (4b) = from: 3 - // relationindex (4b) = from: 7 (it's for relations between two documents in TYPE_RELATION page) - // parentindex (4b) = from: 11 - // size/count (2b) = from: 15 - // data = from: 17 - - if (!count) - count = 0; - - Fs.read(self.fd, buf, 0, buf.length, offset, function() { - - var relid = buf.readUInt32LE(7); - - if (buf[2] === STATE_REMOVED) { - if (relid) - remDocumentAll(self, relid, callback, count); - else - callback(null, count); - return; - } - - buf.writeUInt8(STATE_REMOVED, 2); - buf.writeUInt16LE(0, 15); - - if (buf[0] === TYPE_CLASS) - self.$classes[buf[1]].findfreeslots = true; - - var pageindex = buf.readUInt32LE(3); - - Fs.write(self.fd, buf, 0, buf.length, offset, function() { - - // Updates "documents" in the current page - updPageMeta(self, pageindex, function(err, buf) { - - // type (1b) = from: 0 - // index (1b) = from: 1 - // documents (2b) = from: 2 - // freeslots (1b) = from: 4 - // parentindex (4b) = from: 5 - - var documents = buf.readUInt16LE(2); - buf.writeUInt16LE(documents > 0 ? documents - 1 : documents, 2); - count++; - - setImmediate(function() { - if (relid) - remDocumentAll(self, relid, callback, count); - else - callback(null, count); - }); - }); - }); - }); -} - -function offsetPage(self, index) { - return HEADERSIZE + ((index - 1) * (self.header.pagesize + (self.header.pagelimit * self.header.documentsize))); -} - -function offsetDocument(self, index) { - var page = Math.ceil(index / self.header.pagelimit); - var offset = page * self.header.pagesize; - return HEADERSIZE + offset + ((index - 1) * self.header.documentsize); -} - -function getIndexPage(self, offset) { - return ((offset - HEADERSIZE) / (self.header.pagesize + (self.header.pagelimit * self.header.documentsize))); -} - -function getDocumentIndex(self, pageindex, count) { - return ((pageindex - 1) * self.header.pagelimit) + (count || 1); -} - -function checkRelation(self, relation, indexA, indexB, callback) { - - self.read(indexA, function(err, docs, relid) { - - if (docs) { - for (var i = 0; i < docs.length; i++) { - var doc = docs[i]; - if (doc.ID === indexB && (relation.both || doc.INIT)) { - callback(null, true); - return; - } - } - } - - if (relid) - setImmediate(checkRelation, self, relation, relid, indexB, callback); - else - callback(null, false); - }); -} - -function updMeta(self, type) { - var buf; - switch (type) { - - case META_PAGE_ADD: - buf = U.createBufferSize(4); - buf.writeUInt32LE(self.header.pages); - Fs.write(self.fd, buf, 0, buf.length, 31, self.cb_error); - break; - - case META_PAGE_ADD3: - buf = U.createBufferSize(4); - buf.writeUInt32LE(self.header.pages, 0); - Fs.write(self.fd, buf, 0, buf.length, 31, function() { - buf.writeUInt32LE(self.header.relationpageindex, 0); - Fs.write(self.fd, buf, 0, buf.length, 47, self.cb_error); - }); - break; - - case META_RELATIONPAGEINDEX: - buf = U.createBufferSize(4); - buf.writeUInt32LE(self.header.relationpageindex, 0); - Fs.write(self.fd, buf, 0, buf.length, 47, self.cb_error); - break; - - case META_CLASSESRELATIONS: - - var obj = {}; - obj.c = []; // classes - obj.r = []; // relations - - for (var i = 0; i < self.header.classindex; i++) { - var item = self.$classes[i + 1]; - obj.c.push({ n: item.name, i: item.index, p: item.pageindex, r: item.schema.raw, d: item.documentindex }); - } - - for (var i = 0; i < self.header.relationindex; i++) { - var item = self.$relations[i + 1]; - obj.r.push({ n: item.name, i: item.index, p: item.pageindex, b: item.both ? 1 :0, d: item.documentindex }); - } - - buf = U.createBufferSize(HEADERSIZE - 45); - buf.writeUInt8(self.header.classindex, 0); - buf.writeUInt8(self.header.relationindex, 1); - buf.writeUInt32LE(self.header.relationpageindex, 2); - buf.write(JSON.stringify(obj), 6); - Fs.write(self.fd, buf, 0, buf.length, 45, self.cb_error); - break; - } -} - -function insDocument(self) { - - if (!self.ready || self.states.insert || !self.pending.insert.length || F.isKilled) - return; - - var doc = self.pending.insert.shift(); - if (doc) { - - var cls = self.$classes[doc.name]; - if (cls == null) { - doc.callback(new Error('GraphDB: Class "{0}" not found.'.format(doc.name))); - return; - } - - if (cls.locked || !cls.ready) { - self.pending.insert.push(doc); - setTimeout(self.cb_next, DELAY, NEXT_INSERT); - return; - } - - self.states.insert = true; - - addDocument(self, cls, doc.value, function(err, id) { - // setTimeout(insDocument, 100, self); - self.states.insert = false; - setImmediate(insDocument, self); - doc.callback(err, id); - }); - } -} - -function updDocument(self) { - - if (!self.ready || self.states.update || !self.pending.update.length || F.isKilled) - return; - - var upd = self.pending.update.shift(); - if (upd) { - self.states.update = true; - - var offset = offsetDocument(self, upd.id); - var buf = U.createBufferSize(self.header.documentsize); - - Fs.read(self.fd, buf, 0, buf.length, offset, function(err, size) { - - if (err) { - self.states.update = false; - upd.callback(err); - setImmediate(self.cb_next, NEXT_UPDATE); - return; - } - - if (!size) { - upd.callback(null, 0); - self.states.update = false; - setImmediate(self.cb_next, NEXT_UPDATE); - return; - } - - var save = function(err) { - self.states.update = false; - !err && Fs.write(self.fd, buf, 0, buf.length, offset, self.cb_error); - upd.callback(err, err ? 0 : 1); - setImmediate(self.cb_next, NEXT_UPDATE); - }; - - var data = buf.slice(DATAOFFSET, buf.readUInt16LE(15) + DATAOFFSET); - var limit = self.header.documentsize - DATAOFFSET; - var schema = self.$classes[buf[1]].schema; - var doc; - - if (buf[2] === STATE_COMPRESSED) { - Zlib.inflate(data, ZLIBOPTIONS, function(err, buffer) { - doc = parseData(schema, buffer.toString('utf8').split('|')); - buffer = U.createBuffer(stringifyData(schema, upd.fn(doc, upd.value))); - if (buffer.length > limit) { - Zlib.deflate(buffer, ZLIBOPTIONS, function(err, buffer) { - if (buffer.length <= limit) { - buf.writeUInt16LE(buffer.length, 15); - buf.writeUInt8(STATE_COMPRESSED, 2); - buffer.copy(buf, DATAOFFSET); - save(); - } else - save(new Error('GraphDB: Data too long')); - }); - } else { - buf.writeUInt16LE(buffer.length, 15); - buf.writeUInt8(STATE_UNCOMPRESSED, 2); - buffer.copy(buf, DATAOFFSET); - save(); - } - }); - } else { - doc = parseData(schema, data.toString('utf8').split('|')); - var o = stringifyData(schema, upd.fn(doc, upd.value)); - var buffer = U.createBuffer(o); - if (buffer.length > limit) { - Zlib.deflate(buffer, ZLIBOPTIONS, function(err, buffer) { - if (buffer.length <= limit) { - buf.writeUInt16LE(buffer.length, 15); - buf.writeUInt8(STATE_COMPRESSED, 2); - buffer.copy(buf, DATAOFFSET); - save(); - } else - save(new Error('GraphDB: Data too long')); - }); - } else { - buf.writeUInt16LE(buffer.length, 15); - buf.writeUInt8(STATE_UNCOMPRESSED, 2); - buffer.copy(buf, DATAOFFSET); - save(); - } - } - }); - } -} - -function insRelation(self) { - - if (!self.ready || self.states.relation) - return; - - var doc = self.pending.relation.shift(); - if (doc) { - - var rel = self.$relations[doc.name]; - if (rel == null) { - doc.callback(new Error('GraphDB: Relation "{0}" not found.'.format(doc.name))); - return; - } - - if (rel.locked || !rel.ready) { - self.pending.relation.push(doc); - setTimeout(insRelation, DELAY, self); - return; - } - - self.states.relation = true; - - if (doc.connect) { - addRelation(self, rel, doc.indexA, doc.indexB, function(err, id) { - self.states.relation = false; - doc.callback(err, id); - setImmediate(insRelation, self); - }); - } else { - remRelation(self, rel, doc.indexA, doc.indexB, function(err, id) { - self.states.relation = false; - doc.callback(err, id); - setImmediate(insRelation, self); - }); - } - } -} - -GP.create = function(filename, documentsize, callback) { - var self = this; - Fs.unlink(filename, function() { - var buf = U.createBufferSize(HEADERSIZE); - buf.write('Total.js GraphDB embedded', 0); - buf.writeUInt8(VERSION, 30); // version - buf.writeUInt32LE(0, 31); // pages - buf.writeUInt16LE(PAGESIZE, 35); // pagesize - buf.writeUInt16LE(PAGELIMIT, 37); // pagelimit - buf.writeUInt32LE(0, 39); // documents - buf.writeUInt16LE(documentsize, 43); // documentsize - buf.writeUInt8(0, 45); // classindex - buf.writeUInt8(0, 46); // relationindex - buf.writeUInt8(0, 47); // relationpageindex - buf.write('{"c":[],"r":[]}', 51); // classes and relations - Fs.open(filename, 'w', function(err, fd) { - Fs.write(fd, buf, 0, buf.length, 0, function(err) { - err && self.error(err, 'create'); - Fs.close(fd, function() { - callback && callback(); - }); - }); - }); - }); - return self; -}; - -GP.open = function() { - var self = this; - Fs.stat(self.filename, function(err, stat) { - if (err) { - // file not found - self.create(self.filename, DOCUMENTSIZE, () => self.open()); - } else { - self.header.size = stat.size; - Fs.open(self.filename, 'r+', function(err, fd) { - self.fd = fd; - err && self.error(err, 'open'); - var buf = U.createBufferSize(HEADERSIZE); - Fs.read(self.fd, buf, 0, buf.length, 0, function() { - - self.header.pages = buf.readUInt32LE(31); - self.header.pagesize = buf.readUInt16LE(35); - self.header.pagelimit = buf.readUInt16LE(37); - self.header.documents = buf.readUInt32LE(39); - self.header.documentsize = buf.readUInt16LE(43); - - var size = F.config['graphdb.' + self.name] || DOCUMENTSIZE; - if (size > self.header.documentsize) { - setTimeout(function() { - self.next(NEXT_RESIZE); - }, DELAY); - } - - self.header.relationlimit = ((self.header.documentsize - DATAOFFSET) / 6) >> 0; - self.header.classindex = buf[45]; - self.header.relationindex = buf[46]; - self.header.relationpageindex = buf.readUInt32LE(47); - - var data = buf.slice(51, buf.indexOf(EMPTYBUFFER, 51)).toString('utf8'); - var meta = data.parseJSON(true); - - for (var i = 0; i < meta.c.length; i++) { - var item = meta.c[i]; - self.class(item.n, item.r, item); - } - - for (var i = 0; i < meta.r.length; i++) { - var item = meta.r[i]; - self.relation(item.n, item.b === 1, item); - } - - !self.header.relationpageindex && addPage(self, TYPE_RELATION_DOCUMENT, 0, 0, function(err, index) { - self.header.relationpageindex = index; - }); - - self.ready = true; - self.next(NEXT_READY); - }); - }); - } - }); - return self; -}; - -GP.next = function(type) { - - var self = this; - var tmp; - - switch (type) { - case NEXT_READY: - for (var i = 0; i < self.pending.meta.length; i++) { - tmp = self.pending.meta[i]; - if (tmp.type === TYPE_CLASS) - self.class(tmp.name, tmp.data); - else - self.relation(tmp.name, tmp.data); - } - self.emit('ready'); - break; - - case NEXT_RESIZE: - - clearTimeout(self.$resizedelay); - self.$resizedelay = setTimeout(function() { - if (!self.states.resize) { - self.ready = false; - self.states.resize = true; - var size = (F.config['graphdb.' + self.name] || DOCUMENTSIZE); - var meta = { documentsize: size > self.header.documentsize ? size : self.header.documentsize }; - var keys = Object.keys(self.$classes); - - for (var i = 0; i < keys.length; i++) { - var key = keys[i]; - var cls = self.$classes[key]; - if (cls.$resize) { - !meta.classes && (meta.classes = {}); - meta.classes[cls.index] = cls.$resize; - cls.$resize = null; - } - } - self.resize(meta, function() { - self.states.resize = false; - self.ready = true; - setImmediate(self.cb_next, NEXT_CONTINUE); - }); - } - }, DELAY); - - break; - - case NEXT_INSERT: - insDocument(self); - break; - case NEXT_RELATION: - insRelation(self); - break; - case NEXT_UPDATE: - updDocument(self); - break; - case NEXT_REMOVE: - remDocument(self); - break; - case NEXT_FIND: - if (self.pending.find.length) { - tmp = self.pending.find.shift(); - $find(self, tmp.name, tmp.builder, tmp.reverse); - } - break; - } -}; - -GP.class = function(name, meta, data) { - - var self = this; - - if (!self.ready && !data) { - self.pending.meta.push({ name: name, data: meta, type: 1 }); - return self; - } - - var item = self.$classes[name]; - var save = false; - - if (item == null) { - - item = {}; - item.locked = false; - - if (data) { - item.ready = true; - item.name = name; - item.index = data.i; - item.pageindex = data.p; - item.documentindex = data.d; - item.findfreeslots = true; - } else { - self.header.classindex++; - item.name = name; - item.index = self.header.classindex; - item.ready = false; - item.pageindex = addPage(self, TYPE_CLASS, item.index, 0, function() { - item.ready = true; - }); - item.documentindex = getDocumentIndex(self, item.pageindex); - save = true; - } - - item.schema = parseSchema(meta); - self.$classes[item.name] = self.$classes[item.index] = item; - - } else { - var newschema = parseSchema(meta); - var raw = item.schema.raw; - if (raw !== newschema.raw) { - item.$resize = newschema; - self.next(NEXT_RESIZE); - } - } - - save && updMeta(self, META_CLASSESRELATIONS); - return self; -}; - -GP.relation = function(name, both, data) { - - var self = this; - - if (!self.ready && !data) { - self.pending.meta.push({ name: name, data: both, type: 2 }); - return self; - } - - var self = this; - var item = self.$relations[name]; - var save = false; - - if (item == null) { - - item = {}; - item.ready = true; - item.locked = false; - - if (data) { - item.name = name; - item.index = data.i; - item.pageindex = data.p; - item.documentindex = data.d; - item.both = both; - } else { - self.header.relationindex++; - item.name = name; - item.index = self.header.relationindex; - item.ready = false; - item.both = both; - item.pageindex = addPage(self, TYPE_RELATION, item.index, 0, function() { - item.ready = true; - }); - item.documentindex = getDocumentIndex(self, item.pageindex); - save = true; - } - - self.$relations[item.name] = self.$relations[item.index] = item; - - } else { - // compare - } - - save && updMeta(self, META_CLASSESRELATIONS); - return self; -}; - -GP.emit = function(name, a, b, c, d, e, f, g) { - var evt = this.$events[name]; - if (evt) { - var clean = false; - for (var i = 0, length = evt.length; i < length; i++) { - if (evt[i].$once) - clean = true; - evt[i].call(this, a, b, c, d, e, f, g); - } - if (clean) { - evt = evt.remove(n => n.$once); - if (evt.length) - this.$events[name] = evt; - else - this.$events[name] = undefined; - } - } - return this; -}; - -GP.on = function(name, fn) { - if (this.$ready && (name === 'ready' || name === 'load')) { - fn(); - return this; - } - if (!fn.$once) - this.$free = false; - if (this.$events[name]) - this.$events[name].push(fn); - else - this.$events[name] = [fn]; - return this; -}; - -GP.once = function(name, fn) { - fn.$once = true; - return this.on(name, fn); -}; - -GP.removeListener = function(name, fn) { - var evt = this.$events[name]; - if (evt) { - evt = evt.remove(n => n === fn); - if (evt.length) - this.$events[name] = evt; - else - this.$events[name] = undefined; - } - return this; -}; - -GP.removeAllListeners = function(name) { - if (name === true) - this.$events = EMPTYOBJECT; - else if (name) - this.$events[name] = undefined; - else - this.$events[name] = {}; - return this; -}; - -GP.resize = function(meta, callback) { - - // meta.documentsize - // meta.classes - - var self = this; - var filename = self.filename + '-tmp'; - - self.create(filename, meta.documentsize, function(err) { - - if (err) - throw err; - - Fs.open(filename, 'r+', function(err, fd) { - - var offset = HEADERSIZE; - var newoffset = HEADERSIZE; - var size = self.header.pagesize + (self.header.pagelimit * self.header.documentsize); - var newsize = self.header.pagesize + (self.header.pagelimit * meta.documentsize); - var pageindex = 0; - var totaldocuments = 0; - - var finish = function() { - - var buf = U.createBufferSize(HEADERSIZE); - Fs.read(fd, buf, 0, buf.length, 0, function() { - - // ==== DB:HEADER (7000b) - // name (30b) = from: 0 - // version (1b) = from: 30 - // pages (4b) = from: 31 - // pagesize (2b) = from: 35 - // pagelimit (2b) = from: 37 - // documents (4b) = from: 39 - // documentsize (2b) = from: 43 - // classindex (1b) = from: 45 - // relationindex (1b) = from: 46 - // relationnodeindex = from: 47 - // classes + relations = from: 51 - - // buf. - - buf.writeUInt32LE(pageindex > 0 ? (pageindex - 1) : 0, 31); - buf.writeUInt32LE(totaldocuments, 39); - buf.writeUInt16LE(meta.documentsize, 43); - - var obj = {}; - obj.c = []; // classes - obj.r = []; // relations - - for (var i = 0; i < self.header.classindex; i++) { - var item = self.$classes[i + 1]; - var schema = meta.classes[i + 1]; - obj.c.push({ n: item.name, i: item.index, p: item.pageindex, r: schema ? schema.raw : item.schema.raw, d: item.documentindex }); - } - - for (var i = 0; i < self.header.relationindex; i++) { - var item = self.$relations[i + 1]; - obj.r.push({ n: item.name, i: item.index, p: item.pageindex, b: item.both ? 1 :0, d: item.documentindex }); - } - - buf.writeUInt8(self.header.classindex, 45); - buf.writeUInt8(self.header.relationindex, 46); - buf.writeUInt32LE(self.header.relationpageindex, 47); - buf.write(JSON.stringify(obj), 51); - - Fs.write(fd, buf, 0, buf.length, 0, function() { - // console.log(pageindex, meta.documentsize, totaldocuments); - Fs.close(fd, function() { - Fs.close(self.fd, function() { - Fs.copyFile(self.filename, self.filename.replace(/\.gdb$/, NOW.format('_yyyyMMddHHmm') + '.gdp'), function() { - Fs.rename(self.filename + '-tmp', self.filename, function() { - callback(null); - }); - }); - }); - }); - }); - }); - }; - - var readvalue = function(docbuf, callback) { - var data = docbuf.slice(DATAOFFSET, docbuf.readUInt16LE(15) + DATAOFFSET); - if (docbuf[2] === STATE_COMPRESSED) - Zlib.inflate(data, ZLIBOPTIONS, (err, data) => callback(data ? data.toString('utf8') : '')); - else - callback(data.toString('utf8')); - }; - - var writevalue = function(value, callback) { - var maxsize = meta.documentsize - DATAOFFSET; - var data = U.createBuffer(value); - if (data.length > maxsize) { - Zlib.deflate(data, ZLIBOPTIONS, (err, data) => callback((!data || data.length > maxsize) ? EMPTYBUFFER : data)); - } else - callback(data); - }; - - var process = function() { - - pageindex++; - - // ==== DB:PAGE (20b) - // type (1b) = from: 0 - // index (1b) = from: 1 - // documents (2b) = from: 2 - // freeslots (1b) = from: 4 - // parentindex (4b) = from: 5 - - // ==== DB:DOCUMENT (SIZE) - // type (1b) = from: 0 - // index (1b) = from: 1 - // state (1b) = from: 2 - // pageindex (4b) = from: 3 - // relationindex (4b) = from: 7 (it's for relations between two documents in TYPE_RELATION page) - // parentindex (4b) = from: 11 - // size/count (2b) = from: 15 - // data = from: 17 - - var buf = U.createBufferSize(size); - - Fs.read(self.fd, buf, 0, buf.length, offset, function(err, size) { - - if (!size) { - finish(); - return; - } - - var newbuf = U.createBufferSize(newsize); - - // Copies page info - newbuf.fill(buf, 0, self.header.pagesize); - buf = buf.slice(self.header.pagesize); - - var index = self.header.pagesize; - var documents = 0; - - (self.header.pagelimit).async(function(i, next) { - - // Unexpected problem - if (!buf.length) { - next(); - return; - } - - var docbuf = buf.slice(0, self.header.documentsize); - var typeid = docbuf[0]; - var indexid = docbuf[1]; - - if (docbuf[2] !== STATE_REMOVED) { - totaldocuments++; - documents++; - } - - if (docbuf[2] !== STATE_REMOVED && meta.classes && typeid === TYPE_CLASS && meta.classes[indexid]) { - readvalue(docbuf, function(value) { - - // parseData - // stringifyData - value = stringifyData(meta.classes[indexid], parseData(self.$classes[indexid].schema, value.split('|'))); - - writevalue(value, function(value) { - - if (value === EMPTYBUFFER) { - // BIG PROBLEM - docbuf.writeUInt16LE(0, 15); - docbuf.writeUInt8(STATE_REMOVED, 2); - documents--; - } else { - docbuf.writeUInt16LE(value.length, 15); - docbuf.fill(value, DATAOFFSET, DATAOFFSET + value.length); - } - - newbuf.fill(docbuf, index, index + self.header.documentsize); - index += meta.documentsize; - buf = buf.slice(self.header.documentsize); - next(); - }); - - }); - } else { - newbuf.fill(docbuf, index, index + self.header.documentsize); - index += meta.documentsize; - buf = buf.slice(self.header.documentsize); - next(); - } - - }, function() { - - // Update count of documents - if (newbuf.readUInt16LE(2) !== documents) - newbuf.writeUInt16LE(documents, 2); - - Fs.write(fd, newbuf, 0, newbuf.length, newoffset, function() { - offset += size; - newoffset += newsize; - setImmediate(process); - }); - }); - - }); - }; - - process(); - }); - }); - return self; -}; - - -function $update(doc, value) { - return value; -} - -function $modify(doc, value) { - var keys = Object.keys(value); - for (var i = 0; i < keys.length; i++) { - var key = keys[i]; - - switch (key[0]) { - case '+': - case '-': - case '*': - case '/': - var tmp = key.substring(1); - if (typeof(doc[tmp]) === 'number') { - if (key[0] === '+') - doc[tmp] += value[key]; - else if (key[0] === '-') - doc[tmp] -= value[key]; - else if (key[0] === '*') - doc[tmp] *= value[key]; - else if (key[0] === '/') - doc[tmp] = doc[tmp] / value[key]; - } - break; - default: - if (doc[key] != undefined) - doc[key] = value[key]; - break; - } - } - return doc; -} - -GP.remove = function(id, callback) { - var self = this; - var rem = { id: id, callback: callback || NOOP }; - self.pending.remove.push(rem); - self.next(NEXT_REMOVE); - return self; -}; - -GP.update = function(id, value, callback) { - var self = this; - var upd = { id: id, value: value, fn: typeof(value) === 'function' ? value : $update, callback: callback || NOOP }; - self.pending.update.push(upd); - self.next(NEXT_UPDATE); - return self; -}; - -GP.modify = function(id, value, callback) { - var self = this; - var upd = { id: id, value: value, fn: $modify, callback: callback || NOOP }; - self.pending.update.push(upd); - self.next(NEXT_UPDATE); - return self; -}; - -GP.insert = function(name, value, callback) { - var self = this; - self.pending.insert.push({ name: name, value: value, callback: callback || NOOP }); - self.next(NEXT_INSERT); - return self; -}; - -GP.cursor = function(type, name, callback) { - - var self = this; - var index; - var tmp; - - switch (type) { - case TYPE_CLASS: - tmp = self.$classes[name]; - index = tmp.pageindex; - break; - case TYPE_RELATION: - tmp = self.$relations[name]; - index = tmp.pageindex; - break; - } - - var offset = offsetPage(self, index); - var buf = U.createBufferSize(PAGESIZE); - - Fs.read(self.fd, buf, 0, buf.length, offset, function(err) { - - if (err) { - callback(err); - return; - } - - if (buf[0] !== TYPE_CLASS) { - callback(new Error('Invalid page type')); - return; - } - - if (buf[1] !== tmp.index) { - callback(new Error('Invalid type index')); - return; - } - - var data = {}; - data.count = buf.readUInt16LE(2); - data.parent = buf.readUInt32LE(5); - data.offset = offset; - data.type = buf[0]; - data.index = buf[1]; - data.freeslots = buf[4]; - - data.next = function(callback) { - - if (data.parent == 0) { - callback(new Error('This is the last page'), data); - return; - } - - offset = offsetPage(self, data.parent); - Fs.read(self.fd, buf, 0, buf.length, offset, function(err) { - data.count = buf.readUInt16LE(2); - data.parent = buf.readUInt32LE(5); - data.offset = offset; - data.type = buf[0]; - data.index = buf[1]; - data.freeslots = buf[4]; - data.INDEX = getIndexPage(self, offset) + 1; - callback(err, data); - }); - }; - - data.documents = function(callback) { - - if (!data.count) { - callback(null, EMPTYARRAY); - return; - } - - var index = getIndexPage(self, data.offset) * self.header.pagelimit; - var buffer = U.createBufferSize(self.header.pagelimit * self.header.documentsize); - var offset = data.offset + self.header.pagesize; - var decompress = []; - - index += self.header.pagelimit + 1; - - Fs.read(self.fd, buffer, 0, buffer.length, offset, function(err) { - - if (err) { - callback(err, EMPTYARRAY); - return; - } - - var arr = []; - while (true) { - - if (!buffer.length) - break; - - index--; - var data = buffer.slice(buffer.length - self.header.documentsize); - // index++; - // var data = buffer.slice(0, self.header.documentsize); - if (!data.length) - break; - - // type (1b) = from: 0 - // index (1b) = from: 1 - // state (1b) = from: 2 - // pageindex (4b) = from: 3 - // continuerindex (4b) = from: 7 - // parentindex (4b) = from: 11 - // size/count (2b) = from: 15 - // data = from: 17 - - if (data[2] !== STATE_REMOVED) { - var raw = data.slice(DATAOFFSET, data.readUInt16LE(15) + DATAOFFSET); - if (type === TYPE_CLASS) { - // Document is compressed - if (data[2] === STATE_COMPRESSED) { - var obj = {}; - obj.CLASS = tmp.name; - obj.ID = index; - obj.BUFFER = raw; - decompress.push({ CLASS: tmp, ID: index, BUFFER: raw, index: arr.push(null) }); - } else { - var obj = parseData(tmp.schema, raw.toString('utf8').split('|')); - obj.CLASS = tmp.name; - obj.ID = index; - arr.push(obj); - } - } - } - - buffer = buffer.slice(0, buffer.length - self.header.documentsize); - // buffer = buffer.slice(self.header.documentsize); - } - - if (decompress.length) { - decompress.wait(function(item, next) { - Zlib.inflate(item.BUFFER, ZLIBOPTIONS, function(err, data) { - var obj = parseData(item.CLASS.schema, data.toString('utf8').split('|')); - obj.CLASS = item.CLASS.name; - obj.ID = item.ID; - arr[item.index] = obj; - setImmediate(next); - }); - }, () => callback(null, arr)); - } else - callback(null, arr); - }); - }; - - callback(null, data); - }); -}; - -GP.read = function(index, callback) { - var self = this; - var buf = U.createBufferSize(self.header.documentsize); - Fs.read(self.fd, buf, 0, buf.length, offsetDocument(self, index), function(err) { - - if (err) { - callback(err); - return; - } - - if (buf[2] === STATE_REMOVED) { - callback(null, buf[0] === TYPE_CLASS ? null : EMPTYARRAY); - return; - } - - var tmp; - - switch(buf[0]) { - case TYPE_CLASS: - tmp = self.$classes[buf[1]]; - if (tmp) { - var data = buf.slice(DATAOFFSET, buf.readUInt16LE(15) + DATAOFFSET); - if (buf[2] === STATE_COMPRESSED) { - Zlib.inflate(data, ZLIBOPTIONS, function(err, data) { - data = parseData(tmp.schema, data.toString('utf8').split('|')); - data.ID = index; - data.CLASS = tmp.name; - callback(null, data, buf.readUInt32LE(7), buf.readUInt32LE(11)); - }); - } else { - data = parseData(tmp.schema, data.toString('utf8').split('|')); - data.ID = index; - data.CLASS = tmp.name; - callback(null, data, buf.readUInt32LE(7), buf.readUInt32LE(11)); - } - } else - callback(new Error('GraphDB: invalid document'), null); - break; - case TYPE_RELATION: - tmp = self.$relations[buf[1]]; - if (tmp) { - - var count = buf.readUInt16LE(15); - var arr = []; - for (var i = 0; i < count; i++) { - var off = DATAOFFSET + (i * 6); - arr.push({ RELATION: tmp.name, ID: buf.readUInt32LE(off + 2), INIT: buf[1], INDEX: i }); - } - - callback(null, arr, buf.readUInt32LE(7), buf.readUInt32LE(11), 'RELATION'); - - } else - callback(new Error('GraphDB: invalid document'), null); - break; - - case TYPE_RELATION_DOCUMENT: - - var count = buf.readUInt16LE(15); - var arr = []; - - for (var i = 0; i < count; i++) { - var off = DATAOFFSET + (i * 6); - tmp = self.$relations[buf[off]]; - arr.push({ RELATION: tmp.name, ID: buf.readUInt32LE(off + 2), INIT: buf[off + 1], INDEX: i }); - } - - callback(null, arr, buf.readUInt32LE(7), buf.readUInt32LE(11), 'PRIVATE'); - break; - - default: - callback(null, null); - break; - } - }); -}; - -GP.connect = function(name, indexA, indexB, callback) { - var self = this; - self.pending.relation.push({ name: name, indexA: indexA, indexB: indexB, callback: callback, connect: true }); - self.next(NEXT_RELATION); - return self; -}; - -GP.disconnect = function(name, indexA, indexB, callback) { - var self = this; - self.pending.relation.push({ name: name, indexA: indexA, indexB: indexB, callback: callback }); - self.next(NEXT_RELATION); - return self; -}; - -GP.find = function(cls) { - var self = this; - var builder = new DatabaseBuilder(self); - self.pending.find.push({ name: cls, builder: builder }); - setImmediate(self.cb_next, NEXT_FIND); - return builder; -}; - -GP.find2 = function(cls) { - var self = this; - var builder = new DatabaseBuilder(self); - self.pending.find.push({ name: cls, builder: builder, reverse: true }); - setImmediate(self.cb_next, NEXT_FIND); - return builder; -}; - -GP.scalar = function(cls, type, field) { - var self = this; - var builder = new DatabaseBuilder(self); - builder.scalar(type, field); - self.pending.find.push({ name: cls, builder: builder }); - setImmediate(self.cb_next, NEXT_FIND); - return builder; -}; - -GP.count = function(cls) { - return this.scalar(cls, 'count', 'ID'); -}; - -function GraphDBFilter(db) { - var t = this; - t.db = db; - t.levels = null; -} - -GraphDBFilter.prototype.level = function(num) { - var self = this; - if (self.levels == null) - self.levels = {}; - return self.levels[num] = new DatabaseBuilder(self.db); -}; - -GraphDBFilter.prototype.prepare = function() { - - var self = this; - - if (!self.levels) - return self; - - var arr = Object.keys(self.levels); - - for (var i = 0; i < arr.length; i++) { - var key = arr[i]; - var builder = self.levels[key]; - var filter = {}; - filter.builder = builder; - filter.scalarcount = 0; - filter.filter = builder.makefilter(); - filter.compare = builder.compile(); - filter.index = 0; - filter.count = 0; - filter.counter = 0; - filter.first = builder.$options.first && !builder.$options.sort; - self.levels[key] = filter; - } - - return self; -}; - -GP.graph = function(id, options, callback, filter) { - - if (typeof(options) === 'function') { - callback = options; - options = EMPTYOBJECT; - } else if (!options) - options = EMPTYOBJECT; - - var self = this; - - if (!filter) - filter = new GraphDBFilter(self); - - - self.read(id, function(err, doc, linkid) { - - if (err || !doc) { - callback(err, null, 0); - return; - } - - // options.depth (Int) - // options.relation (String or String Array) - // options.class (String or String Array) - - var relations = null; - var classes = null; - - if (options.relation) { - - var rel; - relations = {}; - - if (options.relation instanceof Array) { - for (var i = 0; i < options.relation.length; i++) { - rel = self.$relations[options.relation[i]]; - if (rel) - relations[rel.name] = rel.both ? 1 : 0; - } - } else { - rel = self.$relations[options.relation]; - if (rel) - relations[rel.name] = rel.both ? 1 : 0; - } - } - - if (options.class) { - - var clstmp; - classes = {}; - - if (options.class instanceof Array) { - for (var i = 0; i < options.class.length; i++) { - clstmp = self.$classes[options.class[i]]; - if (clstmp) - classes[clstmp.name] = 1; - } - } else { - clstmp = self.$classes[options.class]; - if (clstmp) - classes[clstmp.name] = clstmp.index + 1; - } - } - - filter.prepare(); - - var pending = []; - var tmp = {}; - var count = 1; - var sort = false; - - tmp[id] = 1; - - doc.INDEX = 0; - doc.LEVEL = 0; - doc.NODES = []; - - var reader = function(parent, id, depth) { - - if ((options.depth && depth >= options.depth) || (tmp[id])) { - process(); - return; - } - - tmp[id] = 1; - - self.read(id, function(err, links, linkid) { - - if (linkid && !tmp[linkid]) { - pending.push({ id: linkid, parent: parent, depth: depth }); - sort = true; - } - - // because of seeking on HDD - links.quicksort('ID'); - - var fil; - - links.wait(function(item, next) { - - var key = item.ID + '-' + item.RELATION; - - if (tmp[key] || (relations && relations[item.RELATION] == null) || (!options.all && !item.INIT && !relations) || (relations && relations[item.RELATION] === item.TYPE)) - return next(); - - tmp[key] = 1; - - self.read(item.ID, function(err, doc, linkid) { - - if (doc && (!classes || classes[doc.CLASS])) { - - count++; - - doc.INDEX = item.INDEX; - doc.LEVEL = depth + 1; - doc.NODES = []; - - var rel = self.$relations[item.RELATION]; - - if (rel) { - // doc.RELATION = rel.relation; - doc.RELATION = rel.name; - } - - fil = filter.levels ? filter.levels[depth + 1] : null; - - if (fil) { - !fil.response && (fil.response = parent.NODES); - if (!framework_nosql.compare(fil, doc)) - linkid = null; - } else - parent.NODES.push(doc); - - if (linkid && !tmp[linkid]) { - pending.push({ id: linkid, parent: doc, depth: depth + 1 }); - sort = true; - } - } - - next(); - }); - - }, process); - }); - }; - - var process = function() { - - if (pending.length) { - - // because of seeking on HDD - if (sort && pending.length > 1) { - pending.quicksort('id'); - sort = false; - } - - var item = pending.shift(); - reader(item.parent, item.id, item.depth); - - } else { - - if (filter.levels) { - var keys = Object.keys(filter.levels); - for (var i = 0; i < keys.length; i++) { - var f = filter.levels[keys[i]]; - framework_nosql.callback(f); - } - } - - callback(null, doc, count); - } - }; - - linkid && pending.push({ id: linkid, parent: doc, depth: 0 }); - process(); - - }, options.type); - - return filter; -}; - -function $find(self, cls, builder, reverse) { - - var filter = {}; - - filter.builder = builder; - filter.scalarcount = 0; - filter.filter = builder.makefilter(); - filter.compare = builder.compile(); - filter.index = 0; - filter.count = 0; - filter.counter = 0; - filter.first = builder.$options.first && !builder.$options.sort; - - var tmp = self.$classes[cls]; - if (!tmp) { - framework_nosql.callback(filter, 'GraphDB: Class "{0}" is not registered.'.format(cls)); - setImmediate(self.cb_next, NEXT_FIND); - return; - } - - var read = function(err, data) { - - if (err || (!data.count && !data.parent)) { - framework_nosql.callback(filter); - return; - } - - data.documents(function(err, docs) { - for (var i = 0; i < docs.length; i++) { - var doc = docs[i]; - filter.index++; - if ((doc && framework_nosql.compare(filter, doc) === false) || (reverse && filter.done)) { - framework_nosql.callback(filter); - data.next = null; - data.documents = null; - data = null; - setImmediate(self.cb_next, NEXT_FIND); - return; - } - } - data.next(read); - }); - }; - - self.cursor(1, tmp.name, read); -} - -function parseSchema(schema) { - - var obj = {}; - var arr = schema.split('|').trim(); - - obj.meta = {}; - obj.keys = []; - obj.raw = schema; - - for (var i = 0; i < arr.length; i++) { - var arg = arr[i].split(':'); - var type = 0; - switch ((arg[1] || '').toLowerCase().trim()) { - case 'number': - type = 2; - break; - case 'boolean': - case 'bool': - type = 3; - break; - case 'date': - type = 4; - break; - case 'object': - type = 5; - break; - case 'string': - default: - type = 1; - break; - } - var name = arg[0].trim(); - obj.meta[name] = { type: type, pos: i }; - obj.keys.push(name); - } - - return obj; -} - -function stringifyData(schema, doc) { - - var output = ''; - var esc = false; - var size = 0; - - for (var i = 0; i < schema.keys.length; i++) { - var key = schema.keys[i]; - var meta = schema.meta[key]; - var val = doc[key]; - - switch (meta.type) { - case 1: // String - val = val ? val : ''; - size += 4; - break; - case 2: // Number - val = (val || 0); - size += 2; - break; - case 3: // Boolean - val = (val == true ? '1' : '0'); - break; - case 4: // Date - // val = val ? val.toISOString() : ''; - val = val ? val.getTime() : ''; - !val && (size += 13); - break; - case 5: // Object - val = val ? JSON.stringify(val) : ''; - size += 4; - break; - } - - if (!esc && (meta.type === 1 || meta.type === 5)) { - val += ''; - if (REGTESCAPETEST.test(val)) { - esc = true; - val = val.replace(REGTESCAPE, regtescape); - } - } - - output += '|' + val; - } - - return (esc ? '*' : '+') + output; -} - -function parseData(schema, lines, cache) { - - var obj = {}; - var esc = lines === '*'; - var val; - - for (var i = 0; i < schema.keys.length; i++) { - var key = schema.keys[i]; - - if (cache && cache !== EMPTYOBJECT && cache[key] != null) { - obj[key] = cache[key]; - continue; - } - - var meta = schema.meta[key]; - if (meta == null) - continue; - - var pos = meta.pos + 1; - - switch (meta.type) { - case 1: // String - obj[key] = lines[pos]; - if (esc && obj[key]) - obj[key] = obj[key].replace(REGTUNESCAPE, regtescapereverse); - break; - case 2: // Number - val = +lines[pos]; - obj[key] = val < 0 || val > 0 ? val : 0; - break; - case 3: // Boolean - val = lines[pos]; - obj[key] = BOOLEAN[val] == 1; - break; - case 4: // Date - val = lines[pos]; - obj[key] = val ? new Date(val[10] === 'T' ? val : +val) : null; - break; - case 5: // Object - val = lines[pos]; - if (esc && val) - val = val.replace(REGTUNESCAPE, regtescapereverse); - obj[key] = val ? val.parseJSON(true) : null; - break; - } - } - - return obj; -} - -function regtescapereverse(c) { - switch (c) { - case '%0A': - return '\n'; - case '%0D': - return '\r'; - case '%7C': - return '|'; - } - return c; -} - -function regtescape(c) { - switch (c) { - case '\n': - return '%0A'; - case '\r': - return '%0D'; - case '|': - return '%7C'; - } - return c; -} - -exports.load = function(name, size) { - return new GraphDB(name, size); -}; - -exports.getImportantOperations = function() { - return IMPORTATOPERATIONS; -}; \ No newline at end of file From b52bc2111d3d6cbaf1d8df4ddb7b80ea9b2e8416 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 26 Aug 2018 20:26:40 +0200 Subject: [PATCH 027/217] Updated version. --- utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.js b/utils.js index 3c3822dd0..43956b4a7 100755 --- a/utils.js +++ b/utils.js @@ -21,7 +21,7 @@ /** * @module FrameworkUtils - * @version 3.0.0 + * @version 3.0.1 */ 'use strict'; From 82e47b92b18e594607978ec0d4c287762584fa6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 26 Aug 2018 20:27:41 +0200 Subject: [PATCH 028/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e819243bd..261e638ce 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-2", + "version": "3.0.1-3", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 20e8d28fec968d8be09597a6126a43d33604d9ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 27 Aug 2018 13:13:28 +0200 Subject: [PATCH 029/217] Fixed insert after modify/update. --- nosql.js | 99 ++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 71 insertions(+), 28 deletions(-) diff --git a/nosql.js b/nosql.js index ada884b06..45efce0ac 100755 --- a/nosql.js +++ b/nosql.js @@ -1534,9 +1534,7 @@ DP.$update = function() { return; !change && (change = true); - var was = true; - !change && (change = true); if (rec.doc.length === upd.length) { var b = Buffer.byteLength(upd); @@ -1558,10 +1556,26 @@ DP.$update = function() { }; fs.$callback = function() { - filters.done(); + fs = null; self.$writting = false; self.next(0); + + for (var i = 0; i < filters.builders.length; i++) { + var item = filters.builders[i]; + var fil = filter[i]; + if (fil.insert && !item.count) { + item.builder.$insertcallback && item.builder.$insertcallback(fil.insert, item.builder.$repository || EMPTYOBJECT); + var tmp = self.insert(fil.insert); + tmp.$callback = item.builder.$callback; + tmp.$options.log = item.builder.$options.log; + item.builder.$callback = null; + } else { + item.builder.$options.log && item.builder.log(); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.repository); + } + } + change && self.$events.change && self.emit('change', 'update'); }; @@ -1644,9 +1658,23 @@ DP.$update_inmemory = function() { filters.compare2(self.inmemory['#'], update, updateflush); change && self.$save(); + + for (var i = 0; i < filter.length; i++) { + var item = filter[i]; + if (item.insert && !item.count) { + item.builder.$insertcallback && item.builder.$insertcallback(item.insert, item.builder.$repository || EMPTYOBJECT); + var tmp = self.insert(item.insert); + tmp.$callback = item.builder.$callback; + tmp.$options.log = item.builder.$options.log; + item.builder.$callback = null; + } else { + item.builder.$options.log && item.builder.log(); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); + } + } + setImmediate(function() { self.next(0); - filters.done(); change && self.$events.change && self.emit('change', 'update'); }); }); @@ -5890,7 +5918,6 @@ TP.$update = function() { fs.write(tmp, rec.position); fs.write2(upd + NEWLINE); } - }; fs.ondocuments = function() { @@ -5913,10 +5940,26 @@ TP.$update = function() { fs.$callback = function() { - filters.done(); + fs = null; self.$writting = false; self.next(0); + + for (var i = 0; i < filters.builders.length; i++) { + var item = filters.builders[i]; + var fil = filter[i]; + if (fil.insert && !item.count) { + item.builder.$insertcallback && item.builder.$insertcallback(fil.insert, item.builder.$repository || EMPTYOBJECT); + var tmp = self.insert(fil.insert); + tmp.$callback = item.builder.$callback; + tmp.$options.log = item.builder.$options.log; + item.builder.$callback = null; + } else { + item.builder.$options.log && item.builder.log(); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.repository); + } + } + change && self.$events.change && self.emit('change', 'update'); }; @@ -6611,46 +6654,46 @@ NoSQLReader.prototype.done = function() { var item = self.builders[i]; var builder = item.builder; var output; + var opt = builder.$options; - if (builder.$options.scalar || !builder.$options.sort) { - if (builder.$options.scalar) - output = builder.$options.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; - else if (builder.$options.first) + if (opt.scalar || !opt.sort) { + if (opt.scalar) + output = opt.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; + else if (opt.first) output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) + else if (opt.listing) output = listing(builder, item); else output = item.response || []; - builder.$callback2(errorhandling(null, builder, output), builder.$options.readertype === 1 ? item.count : output, item.count); + builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count); continue; } if (item.count) { - if (builder.$options.sort.name) { - if (!builder.$inlinesort || builder.$options.take !== item.response.length) - item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc); - } else if (builder.$options.sort === null) + if (opt.sort.name) { + if (!builder.$inlinesort || opt.take !== item.response.length) + item.response.quicksort(opt.sort.name, opt.sort.asc); + } else if (opt.sort === null) item.response.random(); else - item.response.sort(builder.$options.sort); - - if (builder.$options.skip && builder.$options.take) - item.response = item.response.splice(builder.$options.skip, builder.$options.take); - else if (builder.$options.skip) - item.response = item.response.splice(builder.$options.skip); - else if (!builder.$inlinesort && builder.$options.take) - item.response = item.response.splice(0, builder.$options.take); + item.response.sort(opt.sort); + + if (opt.skip && opt.take) + item.response = item.response.splice(opt.skip, opt.take); + else if (opt.skip) + item.response = item.response.splice(opt.skip); + else if (!builder.$inlinesort && opt.take) + item.response = item.response.splice(0, opt.take); } - if (builder.$options.first) + if (opt.first) output = item.response ? item.response[0] : undefined; - else if (builder.$options.listing) + else if (opt.listing) output = listing(builder, item); else output = item.response || []; - builder.$callback2(errorhandling(null, builder, output), builder.$options.readertype === 1 ? item.count : output, item.count); - builder.done(); + builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count); } }; From 7a79a7fc2313cc44ea2b4f4127e0f6e16cf1d370 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 27 Aug 2018 13:33:03 +0200 Subject: [PATCH 030/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 261e638ce..efb0e00e6 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-3", + "version": "3.0.1-4", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 5776a42b3dac11b6b90ee76aa6e042e0047eda33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 27 Aug 2018 21:24:40 +0200 Subject: [PATCH 031/217] Fixed `versions`. --- changes.txt | 1 + index.js | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/changes.txt b/changes.txt index 69b618c0e..ddf93ca71 100755 --- a/changes.txt +++ b/changes.txt @@ -12,6 +12,7 @@ - fixed: `F.wait()` in the test mode - fixed: `LOCALIZE()` for nested directories - fixed: sending of error handling when WebSocketClient is starting (for example: `unauthorized`) +- fixed: `versions` and `auto` feature with enabled `F.wait()` - improved: NoSQL reader diff --git a/index.js b/index.js index f5516e3ac..7ddf8c653 100755 --- a/index.js +++ b/index.js @@ -8461,9 +8461,19 @@ F.$configure_versions = function(arr, clean) { return F; }; -function makehash(url, callback) { - url = 'http://' + (F.ip === 'auto' ? '0.0.0.0' : F.ip) + ':' + F.port + url; - U.download(url, ['get'], function(err, stream, status) { +function makehash(url, callback, count) { + var target = 'http://' + (F.ip === 'auto' ? '0.0.0.0' : F.ip) + ':' + F.port + url; + U.download(target, ['get'], function(err, stream, status) { + + // Maybe F.wait() + if (status === 503) { + // Unhandled problem + if (count > 60) + callback(''); + else + setTimeout((url, callback, count) => makehash(url, callback, (count || 1) + 1), 1000, url, callback, count); + return; + } if (status !== 200) { callback(''); From 3cb9554cd863b48daa8ab4bc613c41e0df25b768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 27 Aug 2018 21:26:09 +0200 Subject: [PATCH 032/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index efb0e00e6..1e48fa7e2 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-4", + "version": "3.0.1-5", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 1d1256ec828e965f4890417795879ed779726f25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 29 Aug 2018 09:52:29 +0200 Subject: [PATCH 033/217] Added a custom Schema data type. --- builders.js | 12 ++++++++++++ utils.js | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/builders.js b/builders.js index ecedd6515..4b4556c72 100755 --- a/builders.js +++ b/builders.js @@ -643,6 +643,11 @@ SchemaBuilderEntity.prototype.$parse = function(name, value, required, custom) { return result; } + if (value instanceof SchemaBuilderEntity) + result.type = 7; + else + result.type = 10; + return result; } @@ -1880,6 +1885,13 @@ SchemaBuilderEntity.prototype.prepare = function(model, dependencies) { item[property] = null; break; + + case 10: + // custom object type + item[property] = type.raw(val == null ? '' : val.toString()); + if (item[property] === undefined) + item[property] = null; + break; } continue; } diff --git a/utils.js b/utils.js index 43956b4a7..c067fe55e 100755 --- a/utils.js +++ b/utils.js @@ -2227,7 +2227,7 @@ function validate_builder_default(name, value, entity) { var type = typeof(value); - // Enum + KeyValue (8+9) + // Enum + KeyValue + Custom (8+9+10) if (entity.type > 7) return value !== undefined; From 1195ec5c7dd5c13de8ac6b0cf67ac4f5d2a0e3a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 29 Aug 2018 09:52:56 +0200 Subject: [PATCH 034/217] New beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1e48fa7e2..145079f97 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-5", + "version": "3.0.1-6", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From fa2e7d971f9e674c65668d06f7a918127a5b5697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 29 Aug 2018 10:36:33 +0200 Subject: [PATCH 035/217] Added `.bundlesignore` --- bundles.js | 43 +++++++++++++++++++++++++++++++++++++++++++ changes.txt | 1 + 2 files changed, 44 insertions(+) diff --git a/bundles.js b/bundles.js index e04a46443..a8fc63ed5 100755 --- a/bundles.js +++ b/bundles.js @@ -14,6 +14,7 @@ META.total = 'v' + F.version_header; META.node = F.version_node; META.files = []; META.directories = []; +META.ignore = () => true; exports.make = function(callback) { @@ -25,6 +26,10 @@ exports.make = function(callback) { console.time('Done'); } + try { + META.ignore = makeignore(Fs.readFileSync(Path.join(path, '.bundlesignore')).toString('utf8').split('\n')); + } catch (e) {} + blacklist[F.config['directory-temp']] = 1; blacklist[F.config['directory-bundles']] = 1; blacklist[F.config['directory-src']] = 1; @@ -150,6 +155,41 @@ exports.make = function(callback) { }; +function makeignore(arr) { + + var ext; + var code = ['var path=P.substring(0,P.lastIndexOf(\'/\') + 1);', 'var ext=U.getExtension(P);', 'var name=U.getName(P).replace(\'.\' + ext, \'\');']; + + for (var i = 0; i < arr.length; i++) { + var item = arr[i]; + var index = item.lastIndexOf('*.'); + + if (index !== -1) { + // only extensions on this path + ext = item.substring(index + 2); + item = item.substring(0, index); + code.push('tmp=\'{0}\';'.format(item)); + code.push('if((!tmp||path===tmp)&&ext===\'{0}\')return;'.format(ext)); + continue; + } + + ext = U.getExtension(item); + if (ext) { + // only filename + index = item.lastIndexOf('/'); + code.push('tmp=\'{0}\';'.format(item.substring(0, index + 1))); + code.push('if(path===tmp&&U.getName(\'{0}\').replace(\'.{1}\', \'\')===name&&ext===\'{1}\')return;'.format(item.substring(index + 1), ext)); + continue; + } + + // all nested path + code.push('if(path.startsWith(\'{0}\'))return;'.format(item.replace('*', ''))); + } + + code.push('return true'); + return new Function('P', code.join('')); +} + function normalize(path) { return isWindows ? path.replace(/\\/g, '/') : path; } @@ -218,6 +258,9 @@ function copyFiles(files, callback) { var path = F.path.root(F.config['directory-src']); files.wait(function(file, next) { + if (!META.ignore(file.name)) + return next(); + var filename = Path.join(path, file.name); var exists = false; var ext = U.getExtension(file.name); diff --git a/changes.txt b/changes.txt index ddf93ca71..6a09b2c79 100755 --- a/changes.txt +++ b/changes.txt @@ -4,6 +4,7 @@ - added: NoSQL storage `.find()` + `.count()` + '.scalar(type, field)' with multiple thread support - added: `U.reader()` - added: `bundles` supports merging files between bundle and project, project file must start with e.g. `--name.js` +- added: `.bundlesignore` support with similiar functionality like `.gitignore` - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` From 9960c86fde5b18c634927a15b234cfafe47bd6cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 29 Aug 2018 10:45:11 +0200 Subject: [PATCH 036/217] Added `tests` options to `F.http()`. --- index.js | 7 +++++++ test.js | 10 +++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 7ddf8c653..b8b9408c1 100755 --- a/index.js +++ b/index.js @@ -6486,6 +6486,13 @@ F.initialize = function(http, debug, options) { global.DEBUG = debug; global.RELEASE = !debug; global.I = global.isomorphic = F.isomorphic; + + if (options.tests) { + F.testlist = options.tests; + for (var i = 0; i < F.testlist.length; i++) + F.testlist[i] = F.testlist[i].replace(/\.js$/, ''); + } + F.$bundle(function() { F.$configure_configs(); diff --git a/test.js b/test.js index 58e61b814..d4abd2114 100644 --- a/test.js +++ b/test.js @@ -168,8 +168,16 @@ global.TESTUSER = function(user, flags) { }; exports.load = function() { - U.ls(F.path.tests(), function(files) { + var dir = F.path.tests(); + U.ls(dir, function(files) { files.waitFor(function(filename, next) { + + if (F.testlist) { + var tn = filename.replace(dir, '').replace(/\.js$/, ''); + if (F.testlist.indexOf(tn) === -1) + return next(); + } + T.current = { filename: filename, items: [] }; var m = require(filename); T.current.module = m; From 0862f448481751e560019a41472cfa9ba9b8f993 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 29 Aug 2018 19:33:43 +0200 Subject: [PATCH 037/217] Improved cloning values in Schemas because of ObjectID from MongoDB. --- builders.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/builders.js b/builders.js index 4b4556c72..38e6f49c5 100755 --- a/builders.js +++ b/builders.js @@ -2553,7 +2553,12 @@ function clone(obj) { continue; } - o[m] = clone(obj[m]); + // Because here can be a problem with MongoDB.ObjectID + // I assume plain/simple model + if (val.constructor === Object) + o[m] = clone(obj[m]); + else + o[m] = val; } return o; From 301792440433756d631d0acea0f1b361f13ebd95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 29 Aug 2018 20:14:45 +0200 Subject: [PATCH 038/217] Fixed nullable values. --- builders.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builders.js b/builders.js index 38e6f49c5..44c6c79b5 100755 --- a/builders.js +++ b/builders.js @@ -2555,7 +2555,7 @@ function clone(obj) { // Because here can be a problem with MongoDB.ObjectID // I assume plain/simple model - if (val.constructor === Object) + if (val && val.constructor === Object) o[m] = clone(obj[m]); else o[m] = val; From ee3f9fabc95b64216841bd34e48d5ecab2a6186c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 1 Sep 2018 20:58:05 +0200 Subject: [PATCH 039/217] Fixed merging files #663. --- debug.js | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/debug.js b/debug.js index 609821efa..455d62dbd 100644 --- a/debug.js +++ b/debug.js @@ -262,14 +262,19 @@ function runwatching() { if (files[filename]) { var tmp = isViewPublic(filename); if (tmp) { + var skip = true; if (isBUNDLE) { - copyFile(filename, Path.join(SRC, tmp)); - console.log(log); + if (filename.lastIndexOf('--') === -1) + copyFile(filename, Path.join(SRC, tmp)); + else + skip = false; + } + if (skip) { + files[filename] = ticks; + reload = true; + next(); + return; } - files[filename] = ticks; - reload = true; - next(); - return; } } From df7316074c706688f563c4d57efbbe25640a3210 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 1 Sep 2018 20:58:20 +0200 Subject: [PATCH 040/217] Update beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 145079f97..bce70aed9 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-6", + "version": "3.0.1-7", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 9754fc63b95753d1e2dd6eb0f21e50d796350945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 2 Sep 2018 11:11:16 +0200 Subject: [PATCH 041/217] Fixed repository. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 45efce0ac..d3a15ce0d 100755 --- a/nosql.js +++ b/nosql.js @@ -1669,7 +1669,7 @@ DP.$update_inmemory = function() { item.builder.$callback = null; } else { item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.filter.repository); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.repository); } } From 832ae5d8807adb651d126be6506f396c9d9bdf19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 2 Sep 2018 22:57:37 +0200 Subject: [PATCH 042/217] Extend `nosqlstorage.scalar()`. --- nosql.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index d3a15ce0d..d22e8c261 100755 --- a/nosql.js +++ b/nosql.js @@ -5140,8 +5140,8 @@ SP.count = function(beg, end, threads) { return builder; }; -SP.scalar = function(type, field) { - return this.find().scalar(type, field); +SP.scalar = function(beg, end, type, field, threads) { + return this.find(beg, end, threads).scalar(type, field); }; SP.scan = function(beg, end, mapreduce, callback, reverse) { From 719fd4a212eea695230b92c92716d1511c14eea8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 2 Sep 2018 22:58:08 +0200 Subject: [PATCH 043/217] Updated readme. --- changes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 6a09b2c79..b04c6dff3 100755 --- a/changes.txt +++ b/changes.txt @@ -1,7 +1,7 @@ ======= 3.0.1 - added: CSS variables support default values `border-radius: $radius || 10px` -- added: NoSQL storage `.find()` + `.count()` + '.scalar(type, field)' with multiple thread support +- added: NoSQL storage `.find(beg, end, [threads])` + `.count(beg, end, [threads])` + '.scalar(beg, end, type, field, [threads])' with multiple thread support - added: `U.reader()` - added: `bundles` supports merging files between bundle and project, project file must start with e.g. `--name.js` - added: `.bundlesignore` support with similiar functionality like `.gitignore` From f57fb4370ca4eb5869f96d0b6b4bd0bf98f1904c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 3 Sep 2018 09:02:15 +0200 Subject: [PATCH 044/217] Updated version. --- changes.txt | 3 ++- debug.js | 2 +- image.js | 2 +- index.js | 35 ++++++++++++++++++++++++++++------- nosql.js | 2 +- package.json | 2 +- utils.js | 2 +- websocketclient.js | 2 +- 8 files changed, 36 insertions(+), 14 deletions(-) diff --git a/changes.txt b/changes.txt index b04c6dff3..3309b9321 100755 --- a/changes.txt +++ b/changes.txt @@ -1,10 +1,11 @@ -======= 3.0.1 +======= 3.1.0 - added: CSS variables support default values `border-radius: $radius || 10px` - added: NoSQL storage `.find(beg, end, [threads])` + `.count(beg, end, [threads])` + '.scalar(beg, end, type, field, [threads])' with multiple thread support - added: `U.reader()` - added: `bundles` supports merging files between bundle and project, project file must start with e.g. `--name.js` - added: `.bundlesignore` support with similiar functionality like `.gitignore` +- added: support for `SameSite` cookie attribute - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` diff --git a/debug.js b/debug.js index 455d62dbd..1924a6728 100644 --- a/debug.js +++ b/debug.js @@ -21,7 +21,7 @@ /** * @module FrameworkDebug - * @version 3.0.1 + * @version 3.1.0 */ const Path = require('path'); diff --git a/image.js b/image.js index 0894df10b..04bcd26bc 100755 --- a/image.js +++ b/image.js @@ -21,7 +21,7 @@ /** * @module FrameworkImage - * @version 2.9.2 + * @version 3.0.0 */ 'use strict'; diff --git a/index.js b/index.js index b8b9408c1..0ec53c0f2 100755 --- a/index.js +++ b/index.js @@ -21,7 +21,7 @@ /** * @module Framework - * @version 3.0.1 + * @version 3.1.0 */ 'use strict'; @@ -626,8 +626,8 @@ var PERF = {}; function Framework() { this.$id = null; // F.id ==> property - this.version = 3001; - this.version_header = '3.0.1'; + this.version = 3100; + this.version_header = '3.1.0'; this.version_node = process.version.toString(); this.syshash = (Os.hostname() + '-' + Os.platform() + '-' + Os.arch() + '-' + Os.release() + '-' + Os.tmpdir()).md5(); @@ -14848,8 +14848,8 @@ function extend_response(PROTO) { if (self.headersSent || self.success) return; - var cookieHeaderStart = name + '='; - var builder = [cookieHeaderStart + value]; + var cookiename = name + '='; + var builder = [cookiename + value]; var type = typeof(expires); if (expires && !U.isDate(expires) && type === 'object') { @@ -14872,11 +14872,32 @@ function extend_response(PROTO) { if (options.httpOnly || options.httponly || options.HttpOnly) builder.push('HttpOnly'); + var same = options.security || options.samesite || options.sameSite; + if (same) { + switch (same) { + case 1: + same = 'lax'; + break; + case 2: + same = 'strict'; + break; + } + builder.push('SameSite=' + same); + } + var arr = self.getHeader('set-cookie') || []; // Cookie, already, can be in array, resulting in duplicate 'set-cookie' header - var idx = arr.findIndex(cookieStr => cookieStr.startsWith(cookieHeaderStart)); - idx !== -1 && arr.splice(idx, 1); + if (arr.length) { + var l = cookiename.length; + for (var i = 0; i < arr.length; i++) { + if (arr[i].substring(0, l) === cookiename) { + arr.splice(i, 1); + break; + } + } + } + arr.push(builder.join('; ')); self.setHeader('Set-Cookie', arr); return self; diff --git a/nosql.js b/nosql.js index d22e8c261..2ff84898c 100755 --- a/nosql.js +++ b/nosql.js @@ -21,7 +21,7 @@ /** * @module NoSQL - * @version 3.0.1 + * @version 3.1.0 */ 'use strict'; diff --git a/package.json b/package.json index bce70aed9..471541549 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-7", + "version": "3.0.1-8", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", diff --git a/utils.js b/utils.js index c067fe55e..640793530 100755 --- a/utils.js +++ b/utils.js @@ -21,7 +21,7 @@ /** * @module FrameworkUtils - * @version 3.0.1 + * @version 3.1.0 */ 'use strict'; diff --git a/websocketclient.js b/websocketclient.js index 5ecce29b6..b3aa7a69b 100644 --- a/websocketclient.js +++ b/websocketclient.js @@ -21,7 +21,7 @@ /** * @module WebSocketClient - * @version 3.0.0 + * @version 3.1.0 */ if (!global.framework_utils) From 1279b2dae3c829b6509fafb53e5bdc4fba378367 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 3 Sep 2018 14:47:32 +0200 Subject: [PATCH 045/217] Improved `U.reader()`. --- utils.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/utils.js b/utils.js index 640793530..bf6102b13 100755 --- a/utils.js +++ b/utils.js @@ -6121,11 +6121,21 @@ exports.Callback = function(count, callback) { function Reader() {} const RP = Reader.prototype; +RP.clear = function() { + var self = this; + var builder = self.reader.builders[0]; + builder.scalarcount = 0; + builder.count = 0; + builder.counter = 0; + builder.response = null; + return self; +}; + RP.push = function(data) { - if (data == null || !data.length) + if (data == null) this.reader.done(); else - this.reader.compare(data); + this.reader.compare(data instanceof Array ? data : [data]); return this; }; From f751cdf1943d65fba13dd10a9c635047d0dc6b0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 3 Sep 2018 19:11:27 +0200 Subject: [PATCH 046/217] Added support for multiple filter in `U.reader()`. --- utils.js | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/utils.js b/utils.js index bf6102b13..1530d9c5f 100755 --- a/utils.js +++ b/utils.js @@ -6122,12 +6122,18 @@ function Reader() {} const RP = Reader.prototype; RP.clear = function() { + var self = this; - var builder = self.reader.builders[0]; - builder.scalarcount = 0; - builder.count = 0; - builder.counter = 0; - builder.response = null; + var builders = self.reader.builders; + + for (var i = 0; i < builders.length; i++) { + var builder = builders[i]; + builder.scalarcount = 0; + builder.count = 0; + builder.counter = 0; + builder.response = null; + } + return self; }; @@ -6142,7 +6148,12 @@ RP.push = function(data) { RP.find = function() { var self = this; var builder = new framework_nosql.DatabaseBuilder(); - self.reader = new framework_nosql.NoSQLReader(builder); + + if (self.reader) + self.reader.add(builder); + else + self.reader = new framework_nosql.NoSQLReader(builder); + return builder; }; From 263e3326f23b01f08571b274a4a77bcf93559eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 4 Sep 2018 21:06:37 +0200 Subject: [PATCH 047/217] Fixed schema declaration with white characters. --- builders.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builders.js b/builders.js index 44c6c79b5..a6f8d09fb 100755 --- a/builders.js +++ b/builders.js @@ -31,7 +31,7 @@ const DEFAULT_SCHEMA = 'default'; const SKIP = { $$schema: true, $$async: true, $$repository: true, $$controller: true, $$workflow: true }; const REGEXP_CLEAN_EMAIL = /\s/g; const REGEXP_CLEAN_PHONE = /\s|\.|-|\(|\)/g; -const REGEXP_NEWOPERATION = /^(async\s)?function(\s)?([a-zA-Z$][a-zA-Z0-9$]+)?(\s)?\([a-zA-Z0-9$]+\)|^function anonymous\(\$/; +const REGEXP_NEWOPERATION = /^(async\s)?function(\s)?\([a-zA-Z$\s]+\)|^function anonymous\(\$|^\([a-zA-Z$\s]+\)/; const hasOwnProperty = Object.prototype.hasOwnProperty; const Qs = require('querystring'); From 189e3d58b492dd9bb80d3131cccf9c3a8f5ac114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 4 Sep 2018 21:17:43 +0200 Subject: [PATCH 048/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 471541549..c4b027ce5 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-8", + "version": "3.0.1-9", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 4ce5f8e2fcc50f5c1e1c9ea1b9529b05f2448508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 6 Sep 2018 10:19:30 +0200 Subject: [PATCH 049/217] Fixed `LOAD('release')`. --- index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/index.js b/index.js index 0ec53c0f2..f42863185 100755 --- a/index.js +++ b/index.js @@ -6351,6 +6351,7 @@ global.LOAD = F.load = function(debug, types, pwd) { switch (debug.toLowerCase().replace(/\.|\s/g, '-')) { case 'release': case 'production': + debug = false; break; case 'debug': From 8c49dd990f5215915d05772a90d023bb5ae018d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 6 Sep 2018 10:19:42 +0200 Subject: [PATCH 050/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c4b027ce5..b910f58ae 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-9", + "version": "3.0.1-10", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 22761e630c4f794b8a19943838bcb7250cd82a93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 6 Sep 2018 10:20:23 +0200 Subject: [PATCH 051/217] Updated changelog. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index 3309b9321..b23d0ad10 100755 --- a/changes.txt +++ b/changes.txt @@ -15,6 +15,7 @@ - fixed: `LOCALIZE()` for nested directories - fixed: sending of error handling when WebSocketClient is starting (for example: `unauthorized`) - fixed: `versions` and `auto` feature with enabled `F.wait()` +- fixed: `LOAD('release')` a release mode - improved: NoSQL reader From 6053437cdc4838b004f60f13c572a0d74cbd6052 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 9 Sep 2018 10:23:34 +0200 Subject: [PATCH 052/217] Fixed `$.model.$clean()` in Schemas. --- builders.js | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/builders.js b/builders.js index a6f8d09fb..4d37e1409 100755 --- a/builders.js +++ b/builders.js @@ -2152,9 +2152,9 @@ SchemaBuilderEntity.prototype.hook = function(name, model, options, callback, sk var $type = 'hook'; - if (skip === true) { - var builder = new ErrorBuilder(); + if (skip === true || model instanceof SchemaInstance) { + var builder = new ErrorBuilder(); self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); @@ -2286,7 +2286,7 @@ SchemaBuilderEntity.prototype.$execute = function(type, name, model, options, ca if (model && !controller && model.$$controller) controller = model.$$controller; - if (skip === true) { + if (skip === true || model instanceof SchemaInstance) { var builder = new ErrorBuilder(); self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); @@ -2526,7 +2526,10 @@ function clone(obj) { o[i] = obj[i]; continue; } - o[i] = clone(obj[i]); + if (obj[i] instanceof SchemaInstance) + o[i] = obj[i].$clean(); + else + o[i] = clone(obj[i]); } return o; @@ -2541,11 +2544,16 @@ function clone(obj) { var val = obj[m]; - if (val instanceof SchemaInstance) { + if (val instanceof Array) { o[m] = clone(val); continue; } + if (val instanceof SchemaInstance) { + o[m] = val.$clean(); + continue; + } + var type = typeof(val); if (type !== 'object' || val instanceof Date) { if (type !== 'function') From 1c27a43c31cee2768eaa3ea92521b577cb4b6da6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 9 Sep 2018 10:46:28 +0200 Subject: [PATCH 053/217] Added a new bug fix. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index b23d0ad10..0f37582f6 100755 --- a/changes.txt +++ b/changes.txt @@ -16,6 +16,7 @@ - fixed: sending of error handling when WebSocketClient is starting (for example: `unauthorized`) - fixed: `versions` and `auto` feature with enabled `F.wait()` - fixed: `LOAD('release')` a release mode +- fixed: `SchemaInstance.$clean()` for nested schemas - improved: NoSQL reader From 9893afcb766cafc540cbf8a544d772db83e9691f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 9 Sep 2018 12:04:13 +0200 Subject: [PATCH 054/217] Adde NOSQL/TABLE supports `!field` as a boolean toggle in modify/update method. --- changes.txt | 1 + nosql.js | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 0f37582f6..f6190f4d4 100755 --- a/changes.txt +++ b/changes.txt @@ -6,6 +6,7 @@ - added: `bundles` supports merging files between bundle and project, project file must start with e.g. `--name.js` - added: `.bundlesignore` support with similiar functionality like `.gitignore` - added: support for `SameSite` cookie attribute +- added: NOSQL/TABLE supports `!field` as a boolean toggle in modify/update method - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` diff --git a/nosql.js b/nosql.js index 2ff84898c..b9c84f1aa 100755 --- a/nosql.js +++ b/nosql.js @@ -892,6 +892,7 @@ DP.modify = function(doc, insert) { for (var i = 0; i < keys.length; i++) { var key = keys[i]; switch (key[0]) { + case '!': case '+': case '-': case '*': @@ -1500,6 +1501,9 @@ DP.$update = function() { doc[key] = val(doc[key], doc); else if (fil.inc && fil.inc[key]) { switch (fil.inc[key]) { + case '!': + doc[key] = !doc[key]; + break; case '+': doc[key] = (doc[key] || 0) + val; break; @@ -1624,6 +1628,9 @@ DP.$update_inmemory = function() { doc[key] = val(doc[key], doc); else if (fil.inc && fil.inc[key]) { switch (fil.inc[key]) { + case '!': + doc[key] = !doc[key]; + break; case '+': doc[key] = (doc[key] || 0) + val; break; @@ -5395,6 +5402,7 @@ TP.modify = function(doc, insert) { for (var i = 0; i < keys.length; i++) { var key = keys[i]; switch (key[0]) { + case '!': case '+': case '-': case '*': @@ -5874,6 +5882,9 @@ TP.$update = function() { doc[key] = val(doc[key], doc); else if (fil.inc && fil.inc[key]) { switch (fil.inc[key]) { + case '!': + doc[key] = doc[key] == null ? true : !doc[key]; + break; case '+': doc[key] = (doc[key] || 0) + val; break; @@ -5938,7 +5949,6 @@ TP.$update = function() { filters.compare2(arr, update, updateflush); }; - fs.$callback = function() { fs = null; @@ -6505,10 +6515,14 @@ NoSQLReader.prototype.add = function(builder, noTrimmer) { NoSQLReader.prototype.compare2 = function(docs, custom, done) { var self = this; + for (var i = 0; i < docs.length; i++) { var doc = docs[i]; + if (doc === EMPTYOBJECT) + continue; + if (self.builders.length === self.canceled) return false; From a6090ea357bcfa9ead139a6eeedd7bc95fa5fc06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 9 Sep 2018 23:28:00 +0200 Subject: [PATCH 055/217] Fixed critical bug with updating in NoSQL embedded. --- nosql.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nosql.js b/nosql.js index b9c84f1aa..d74df3711 100755 --- a/nosql.js +++ b/nosql.js @@ -1522,7 +1522,7 @@ DP.$update = function() { } } } else - docs[dindex] = typeof(f.doc) === 'function' ? fil.doc(doc) : fil.doc; + docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc) : fil.doc; self.$events[e] && self.emit(e, doc, old); f.builder.$options.backup && f.builder.$backupdoc(rec.doc); @@ -1649,7 +1649,7 @@ DP.$update_inmemory = function() { } } } else - docs[dindex] = typeof(f.doc) === 'function' ? fil.doc(doc) : fil.doc; + docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc) : fil.doc; self.$events[e] && self.emit(e, doc, old); f.builder.$options.backup && f.builder.$backupdoc(old); @@ -5903,7 +5903,7 @@ TP.$update = function() { } } } else - docs[dindex] = typeof(f.doc) === 'function' ? fil.doc(doc) : fil.doc; + docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc) : fil.doc; self.$events[e] && self.emit(e, doc, old); f.builder.$options.backup && f.builder.$backupdoc(rec.doc); From 020cf47b048c46fe6c1ebcdd4565e7bd698d232e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 9 Sep 2018 23:43:10 +0200 Subject: [PATCH 056/217] Fixed a problem with `repository` when updating. --- nosql.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nosql.js b/nosql.js index d74df3711..177293b08 100755 --- a/nosql.js +++ b/nosql.js @@ -1522,7 +1522,7 @@ DP.$update = function() { } } } else - docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc) : fil.doc; + docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc, f.filter.repository) : fil.doc; self.$events[e] && self.emit(e, doc, old); f.builder.$options.backup && f.builder.$backupdoc(rec.doc); @@ -1649,7 +1649,7 @@ DP.$update_inmemory = function() { } } } else - docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc) : fil.doc; + docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc, f.filter.repository) : fil.doc; self.$events[e] && self.emit(e, doc, old); f.builder.$options.backup && f.builder.$backupdoc(old); @@ -5903,7 +5903,7 @@ TP.$update = function() { } } } else - docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc) : fil.doc; + docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc, f.filter.repository) : fil.doc; self.$events[e] && self.emit(e, doc, old); f.builder.$options.backup && f.builder.$backupdoc(rec.doc); From 3bc2833f04d0702fc124dc93ff93f26f3f55c00c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 10 Sep 2018 08:08:10 +0200 Subject: [PATCH 057/217] Improved quicksort. --- utils.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/utils.js b/utils.js index 1530d9c5f..bbe2be5c8 100755 --- a/utils.js +++ b/utils.js @@ -4700,10 +4700,20 @@ AP.quicksort = AP.orderBy = function(name, asc) { if (typeof(name) === 'boolean') { asc = name; name = undefined; - } - - if (asc === undefined) + } else if (asc === undefined) asc = true; + else { + switch (asc) { + case 'asc': + case 'ASC': + asc = true; + break; + case 'desc': + case 'DESC': + asc = false; + break; + } + } var self = this; var type = 0; From c0e6378c8aabd3f7e33d214836b1feb27353b5bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 10 Sep 2018 10:41:07 +0200 Subject: [PATCH 058/217] Improved OPERATIONS. --- builders.js | 125 +++++++++++++++++++++++++++++++++++++++++++++------- changes.txt | 2 + 2 files changed, 111 insertions(+), 16 deletions(-) diff --git a/builders.js b/builders.js index 4d37e1409..040fac05a 100755 --- a/builders.js +++ b/builders.js @@ -3046,6 +3046,9 @@ function ErrorBuilder(onResource) { // Hidden: when the .push() contains a classic Error instance // this.unexpected; + // A default path for .push() + // this.path; + !onResource && this._resource(); } @@ -3391,6 +3394,9 @@ ErrorBuilder.prototype.push = function(name, error, path, index, prefix) { path = undefined; } + if (this.path && !path) + path = this.path; + if (!error && typeof(name) === 'string') { var m = name.length; if (m > 15) @@ -4589,7 +4595,10 @@ function $decodeURIComponent(value) { } } -global.NEWOPERATION = function(name, fn) { +global.NEWOPERATION = function(name, fn, repeat, stop) { + + // @repeat {Number} How many times will be the operation repeated after error? + // @stop {Boolean} Stop when the error is thrown // Remove operation if (fn == null) { @@ -4600,6 +4609,9 @@ global.NEWOPERATION = function(name, fn) { operations[name] = fn; operations[name].$owner = F.$owner(); operations[name].$newversion = REGEXP_NEWOPERATION.test(fn.toString()); + operations[name].$repeat = repeat; + operations[name].$stop = stop; + return this; }; @@ -4618,25 +4630,32 @@ global.OPERATION = function(name, value, callback, param, controller) { if (fn) { if (fn.$newversion) { var self = new OperationOptions(error, value, param, controller); - if (callback && callback !== NOOP) { - self.callback = function(value) { - if (arguments.length > 1) { - if (value instanceof Error || (value instanceof ErrorBuilder && value.hasError())) { - self.error.push(value); - value = EMPTYOBJECT; - } else - value = arguments[1]; - } else if (value instanceof Error || (value instanceof ErrorBuilder && value.hasError())) { + self.$repeat = fn.$repeat; + self.callback = function(value) { + + if (arguments.length > 1) { + if (value instanceof Error || (value instanceof ErrorBuilder && value.hasError())) { self.error.push(value); value = EMPTYOBJECT; - } + } else + value = arguments[1]; + } else if (value instanceof Error || (value instanceof ErrorBuilder && value.hasError())) { + self.error.push(value); + value = EMPTYOBJECT; + } + + if (self.error.items.length && self.$repeat) { + self.error.clear(); + self.$repeat--; + fn(self); + } else + callback && callback(self.error.hasError() ? self.error : null, value, self.options); + + return self; + }; - callback(self.error.hasError() ? self.error : null, value, self.options); - return self; - }; - } else - self.callback = NOOP; fn(self); + } else fn(error, value, function(value) { if (callback) { @@ -4653,6 +4672,80 @@ global.OPERATION = function(name, value, callback, param, controller) { } }; +global.RUN = function(name, value, callback, param, controller) { + + if (typeof(value) === 'function') { + controller = param; + param = callback; + callback = value; + value = EMPTYOBJECT; + } + + if (typeof(name) === 'string') + name = name.split(',').trim(); + + var error = new ErrorBuilder(); + var opt = new OperationOptions(error, value, param, controller); + + opt.meta = {}; + opt.meta.items = name; + opt.response = {}; + + opt.callback = function(value) { + + if (arguments.length > 1) { + if (value instanceof Error || (value instanceof ErrorBuilder && value.hasError())) { + opt.error.push(value); + value = EMPTYOBJECT; + } else + value = arguments[1]; + } else if (value instanceof Error || (value instanceof ErrorBuilder && value.hasError())) { + opt.error.push(value); + value = EMPTYOBJECT; + } + + if (opt.error.items.length && opt.$repeat > 0) { + opt.error.clear(); + opt.$repeat--; + opt.repeated = true; + setImmediate(opt => opt.$current(opt), opt); + } else { + opt.error.items.length && error.push(opt.error); + if (opt.error.items.length && opt.$current.$stop) { + name = null; + opt.next = null; + callback(error, opt.response, opt); + } else { + opt.response[opt.meta.current] = value; + opt.meta.prev = opt.meta.current; + opt.$next(); + } + } + }; + + name.wait(function(key, next, index) { + + var fn = operations[key]; + if (!fn) { + // What now? + // F.error('Operation "{0}" not found'.format(key), 'RUN()'); + return next(); + } + + opt.repeated = false; + opt.error = new ErrorBuilder(); + opt.error.path = 'operation: ' + key; + opt.meta.index = index; + opt.meta.current = key; + opt.$repeat = fn.$repeat; + opt.$current = fn; + opt.$next = next; + opt.meta.next = name[index]; + fn(opt); + + }, () => callback(error.items.length ? error : null, opt.response, opt)); +}; + function OperationOptions(error, value, options, controller) { if (!controller && options instanceof global.Controller) { diff --git a/changes.txt b/changes.txt index f6190f4d4..39d46fae8 100755 --- a/changes.txt +++ b/changes.txt @@ -7,8 +7,10 @@ - added: `.bundlesignore` support with similiar functionality like `.gitignore` - added: support for `SameSite` cookie attribute - added: NOSQL/TABLE supports `!field` as a boolean toggle in modify/update method +- added: `RUN()` for executing multiple Total.js operations - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` +- updated: `NEWOPERATION()` supports `repeat` and `stop` argument - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function From ce5907332454a106216f714f75d722250d1afec6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 10 Sep 2018 13:21:05 +0200 Subject: [PATCH 059/217] Improved `RUN()`. --- builders.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/builders.js b/builders.js index 040fac05a..54168ca23 100755 --- a/builders.js +++ b/builders.js @@ -4681,6 +4681,12 @@ global.RUN = function(name, value, callback, param, controller) { value = EMPTYOBJECT; } + if (param instanceof global.Controller || (param && param.isWebSocket)) { + controller = param; + param = EMPTYOBJECT; + } else if (param instanceof OperationOptions) + controller = param.controller; + if (typeof(name) === 'string') name = name.split(',').trim(); @@ -4690,6 +4696,7 @@ global.RUN = function(name, value, callback, param, controller) { opt.meta = {}; opt.meta.items = name; opt.response = {}; + opt.errors = error; opt.callback = function(value) { From 66ac4b0c6f9b64f35e3d3bedaecc6572bbd30ba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 11 Sep 2018 09:52:05 +0200 Subject: [PATCH 060/217] Fixed extracting bundles. --- bundles.js | 26 +++--- changes.txt | 1 + test/test-tmp.js | 211 ++++------------------------------------------- 3 files changed, 31 insertions(+), 207 deletions(-) diff --git a/bundles.js b/bundles.js index a8fc63ed5..7eaa944cc 100755 --- a/bundles.js +++ b/bundles.js @@ -33,10 +33,9 @@ exports.make = function(callback) { blacklist[F.config['directory-temp']] = 1; blacklist[F.config['directory-bundles']] = 1; blacklist[F.config['directory-src']] = 1; + blacklist[F.config['directory-logs']] = 1; blacklist['/node_modules/'] = 1; - // blacklist['/debug.js'] = 1; blacklist['/debug.pid'] = 1; - //blacklist['/package.json'] = 1; blacklist['/package-lock.json'] = 1; var Files = []; @@ -118,7 +117,8 @@ exports.make = function(callback) { for (var i = 0, length = files.length; i < length; i++) { var file = files[i].substring(Length); var type = 0; - if (file.startsWith(F.config['directory-databases'])) + + if (file.startsWith(F.config['directory-databases']) || file.startsWith('/flow/') || file.startsWith('/dashboard/')) type = 1; else if (file.startsWith(F.config['directory-public'])) type = 2; @@ -158,7 +158,7 @@ exports.make = function(callback) { function makeignore(arr) { var ext; - var code = ['var path=P.substring(0,P.lastIndexOf(\'/\') + 1);', 'var ext=U.getExtension(P);', 'var name=U.getName(P).replace(\'.\' + ext, \'\');']; + var code = ['var path=P.substring(0,P.lastIndexOf(\'/\')+1);', 'var ext=U.getExtension(P);', 'var name=U.getName(P).replace(\'.\'+ ext,\'\');']; for (var i = 0; i < arr.length; i++) { var item = arr[i]; @@ -215,10 +215,13 @@ function cleanFiles(callback) { if (meta.files && meta.files.length) { for (var i = 0, length = meta.files.length; i < length; i++) { var filename = meta.files[i]; - try { - F.consoledebug('Remove', filename); - Fs.unlinkSync(Path.join(path, filename)); - } catch (e) {} + var dir = filename.substring(0, filename.indexOf('/', 1) + 1); + if (!blacklist[dir]) { + try { + F.consoledebug('Remove', filename); + Fs.unlinkSync(Path.join(path, filename)); + } catch (e) {} + } } } @@ -226,7 +229,8 @@ function cleanFiles(callback) { meta.directories.quicksort('length', false); for (var i = 0, length = meta.directories.length; i < length; i++) { try { - Fs.rmdirSync(Path.join(path, meta.directories[i])); + if (!blacklist[meta.directories[i]]) + Fs.rmdirSync(Path.join(path, meta.directories[i])); } catch (e) {} } } @@ -285,9 +289,9 @@ function copyFiles(files, callback) { append = true; if (CONSOLE && exists) { - F.config['allow-debug'] && F.consoledebug(append ? 'EXT: ' : 'REW:', p); + F.config['allow-debug'] && F.consoledebug(append ? 'EXT:' : 'REW:', p); } else - F.consoledebug(append ? 'EXT:' : 'COP:', p); + F.consoledebug(append ? 'EXT:' : 'COP:', p); if (append) { Fs.appendFile(filename, '\n' + Fs.readFileSync(file.filename).toString('utf8'), next); diff --git a/changes.txt b/changes.txt index 39d46fae8..b375a2380 100755 --- a/changes.txt +++ b/changes.txt @@ -20,6 +20,7 @@ - fixed: `versions` and `auto` feature with enabled `F.wait()` - fixed: `LOAD('release')` a release mode - fixed: `SchemaInstance.$clean()` for nested schemas +- fixed: extracting `bundles` (added `/flow/` and `/dashboard/`) - improved: NoSQL reader diff --git a/test/test-tmp.js b/test/test-tmp.js index b03fe94bc..b6aceaf82 100644 --- a/test/test-tmp.js +++ b/test/test-tmp.js @@ -1,201 +1,20 @@ require('../index'); +// NOSQLMEMORY('test'); +// NOSQL('test').find2().take(10).callback(console.log); +// NOSQL('test').count().callback(console.log); +// NOSQL('test').remove().between('index', 0, 10).callback(console.log).backup('fet'); +// NOSQL('test').remove().between('index', 1, 3).callback(console.log); +// NOSQL('test').on('modify', console.log); +// NOSQL('test').update({ name: GUID(5) }).between('index', 1, 3).callback(console.log); -function SQL(query) { +F.config['table.test'] = 'index:number | name:string'; - var self = this; - self.options = {}; - self.builder = new framework_nosql.DatabaseBuilder(null); - self.query = query; +TABLE('test').find().take(10).skip(10).callback(console.log); - var type = self.parseType(); +// TABLE('test').modify({ name: GUID(5) }).between('index', 3, 5).callback(console.log); +// TABLE('test').remove().between('index', 0, 2).callback(console.log); - switch (type) { - case 'select': - self.parseLimit(); - self.parseOrder(); - self.parseWhere(); - self.parseJoins(); - self.parseTable(); - self.parseNames(); - break; - case 'update': - self.parseWhere(); - self.parseTable(); - self.parseUpdate(); - break; - case 'insert': - self.parseWhere(); - self.parseInsert(); - break; - case 'delete': - self.parseWhere(); - self.parseTable(); - break; - } - - console.log(self.options); -} - -var SQLP = SQL.prototype; - -SQLP.parseLimit = function() { - var self = this; - var tmp = self.query.match(/take \d+ skip \d+$/i); - !tmp && (tmp = self.query.match(/skip \d+ take \d+$/i)); - !tmp && (tmp = self.query.match(/take \d+$/i)); - !tmp && (tmp = self.query.match(/skip \d+$/i)); - if (!tmp) - return self; - self.query = self.query.replace(tmp, '').trim(); - var arr = tmp[0].toString().toLowerCase().split(' '); - if (arr[0] === 'take') - self.options.take = +arr[1]; - else if (arr[2] === 'take') - self.options.take = +arr[3]; - if (arr[0] === 'skip') - self.options.skip = +arr[1]; - else if (arr[2] === 'skip') - self.options.skip = +arr[3]; - return self; -}; - -SQLP.parseOrder = function() { - var self = this; - var tmp = self.query.match(/order by .*?$/i); - - if (!tmp) - return self; - - self.query = self.query.replace(tmp, '').trim(); - var arr = tmp[0].toString().substring(9).split(','); - - self.options.sort = []; - - for (var i = 0; i < arr.length; i++) { - tmp = arr[i].trim().split(' '); - self.options.sort.push({ name: tmp[0], desc: (tmp[1] || '').toLowerCase() === 'desc' }); - } - - return self; -}; - -SQLP.parseWhere = function() { - var self = this; - var tmp = self.query.match(/where .*?$/i); - if (!tmp) - return self; - self.query = self.query.replace(tmp, ''); - - tmp = tmp[0].toString().substring(6).replace(/\sAND\s/gi, ' && ').replace(/\sOR\s/gi, ' || ').replace(/[a-z0-9]=/gi, function(text) { - return text + '='; - }); - - self.options.where = tmp; - return self; -}; - -SQLP.parseJoins = function() { - var self = this; - var tmp = self.query.match(/left join.*?$/i); - if (!tmp) { - tmp = self.query.match(/join.*?$/i); - if (!tmp) - return self; - } - self.query = self.query.replace(tmp, ''); - tmp = tmp[0].toString().trim(); - - self.options.joins = []; - - tmp = tmp.substring(5).split(/\s?JOIN\s/i); - for (var i = 0; i < tmp.length; i++) { - var join = tmp[i].split(/\son\s/i); - self.options.joins.push({ table: join[0], condition: join[1] }); - } - - return self; -}; - -SQLP.parseTable = function() { - var self = this; - var tmp = self.query.match(/from\s.*?$/i); - if (!tmp) - return self; - self.query = self.query.replace(tmp, ''); - tmp = tmp[0].toString().substring(5).trim(); - - var arr = tmp.split(' '); - // console.log(arr); - - return self; -}; - -SQLP.parseNames = function() { - var self = this; - var tmp = self.query.match(/select\s.*?$/i); - if (!tmp) - return self; - - self.query = self.query.replace(tmp, ''); - tmp = tmp[0].toString().substring(6).trim().split(','); - - self.options.fields = []; - - for (var i = 0; i < tmp.length; i++) { - var field = tmp[i].trim(); - var alias = field.match(/as\s.*?$/); - var name = ''; - var type = 0; - - if (alias) { - field = field.replace(alias, ''); - alias = alias.toString().substring(3); - } - - var index = field.indexOf('('); - if (index !== -1) { - switch (field.substring(0, index).toLowerCase()) { - case 'count': - type = 1; - break; - case 'min': - type = 2; - break; - case 'max': - type = 3; - break; - case 'avg': - type = 4; - break; - case 'sum': - type = 5; - break; - case 'distinct': - type = 6; - break; - } - name = field.substring(index + 1, field.lastIndexOf(')')); - } else - name = field; - self.options.fields.push({ alias: alias || name, name: name, type: type }); - } - - return self; -}; - -SQLP.parseUpdate = function() { - var self = this; - return self; -}; - -SQLP.parseInsert = function() { - var self = this; - return self; -}; - -SQLP.parseType = function() { - var self = this; - return self.query.substring(0, self.query.indexOf(' ')).toLowerCase(); -}; - -var sql = new SQL('SELECT COUNT(*) as count, id FROM table a JOIN users ON users.id=a.id JOIN orders ON orders.id=a.id WHERE a.name="Peter" AND a.age=30 ORDER BY a.name, a.age ASC TAKE 20 SKIP 10'); \ No newline at end of file +/* +for (var i = 0; i < 100; i++) + TABLE('test').insert({ index: i }); +*/ \ No newline at end of file From a6a2a6a8d3ca58a66e44f8366826a8cc5c7c2593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 11 Sep 2018 13:51:24 +0200 Subject: [PATCH 061/217] New improvements. --- changes.txt | 2 + index.js | 171 +++++++++++++++++++++++++++------------------------- internal.js | 6 +- 3 files changed, 94 insertions(+), 85 deletions(-) diff --git a/changes.txt b/changes.txt index b375a2380..5af3b299f 100755 --- a/changes.txt +++ b/changes.txt @@ -8,6 +8,7 @@ - added: support for `SameSite` cookie attribute - added: NOSQL/TABLE supports `!field` as a boolean toggle in modify/update method - added: `RUN()` for executing multiple Total.js operations +- added: a new global alias `CONF` (it's a reference to config) for `F.config` - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat` and `stop` argument @@ -21,6 +22,7 @@ - fixed: `LOAD('release')` a release mode - fixed: `SchemaInstance.$clean()` for nested schemas - fixed: extracting `bundles` (added `/flow/` and `/dashboard/`) +- fixed: subdomain routing for `localhost` - improved: NoSQL reader diff --git a/index.js b/index.js index f42863185..e46acdb7c 100755 --- a/index.js +++ b/index.js @@ -62,6 +62,7 @@ const REG_WINDOWSPATH = /\\/g; const REG_SCRIPTCONTENT = /<|>|;/; const REG_HTTPHTTPS = /^(\/)?(http|https):\/\//i; const REG_NOCOMPRESS = /[.|-]+min(@[a-z0-9]*)?\.(css|js)$/i; +const REG_WWW = /^www\./i; const REG_TEXTAPPLICATION = /text|application/; const REG_ENCODINGCLEANER = /[;\s]charset=utf-8/g; const REG_SKIPERROR = /epipe|invalid\sdistance/i; @@ -625,13 +626,15 @@ var PERF = {}; function Framework() { - this.$id = null; // F.id ==> property - this.version = 3100; - this.version_header = '3.1.0'; - this.version_node = process.version.toString(); - this.syshash = (Os.hostname() + '-' + Os.platform() + '-' + Os.arch() + '-' + Os.release() + '-' + Os.tmpdir()).md5(); + var self = this; + + self.$id = null; // F.id ==> property + self.version = 3100; + self.version_header = '3.1.0'; + self.version_node = process.version.toString(); + self.syshash = (Os.hostname() + '-' + Os.platform() + '-' + Os.arch() + '-' + Os.release() + '-' + Os.tmpdir()).md5(); - this.config = { + global.CONF = self.config = { debug: true, trace: true, @@ -640,8 +643,8 @@ function Framework() { name: 'Total.js', version: '1.0.0', author: '', - secret: this.syshash, - 'secret-uid': this.syshash.substring(10), + secret: self.syshash, + 'secret-uid': self.syshash.substring(10), 'security.txt': 'Contact: mailto:support@totaljs.com\nContact: https://www.totaljs.com/contact/', 'etag-version': '', @@ -746,24 +749,24 @@ function Framework() { 'default-interval-uptodate': 5 }; - global.G = this.global = {}; - this.$bundling = true; - this.resources = {}; - this.connections = {}; - this.functions = {}; - this.themes = {}; - this.versions = null; - this.workflows = {}; - this.uptodates = null; - this.schedules = []; - - this.isDebug = true; - this.isTest = false; - this.isLoaded = false; - this.isWorker = true; - this.isCluster = process.env.PASSENGER_APP_ENV ? false : require('cluster').isWorker; - - this.routes = { + global.G = self.global = {}; + self.$bundling = true; + self.resources = {}; + self.connections = {}; + self.functions = {}; + self.themes = {}; + self.versions = null; + self.workflows = {}; + self.uptodates = null; + self.schedules = []; + + self.isDebug = true; + self.isTest = false; + self.isLoaded = false; + self.isWorker = true; + self.isCluster = process.env.PASSENGER_APP_ENV ? false : require('cluster').isWorker; + + self.routes = { sitemap: null, web: [], system: {}, @@ -783,27 +786,27 @@ function Framework() { resources: {} }; - this.owners = []; - this.modificators = null; - this.helpers = {}; - this.modules = {}; - this.models = {}; - this.sources = {}; - this.controllers = {}; - this.dependencies = {}; - this.isomorphic = {}; - this.components = { has: false, css: false, js: false, views: {}, instances: {}, version: null, links: '', groups: {}, files: {} }; - this.convertors = []; - this.convertors2 = null; - this.tests = []; - this.errors = []; - this.problems = []; - this.changes = []; - this.server = null; - this.port = 0; - this.ip = ''; - - this.validators = { + self.owners = []; + self.modificators = null; + self.helpers = {}; + self.modules = {}; + self.models = {}; + self.sources = {}; + self.controllers = {}; + self.dependencies = {}; + self.isomorphic = {}; + self.components = { has: false, css: false, js: false, views: {}, instances: {}, version: null, links: '', groups: {}, files: {} }; + self.convertors = []; + self.convertors2 = null; + self.tests = []; + self.errors = []; + self.problems = []; + self.changes = []; + self.server = null; + self.port = 0; + self.ip = ''; + + self.validators = { email: new RegExp('^[a-zA-Z0-9-_.+]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$'), url: /^(https?:\/\/(?:www\.|(?!www))[^\s.#!:?+=&@!$'~*,;/()[\]]+\.[^\s#!?+=&@!$'~*,;()[\]\\]{2,}\/?|www\.[^\s#!:.?+=&@!$'~*,;/()[\]]+\.[^\s#!?+=&@!$'~*,;()[\]\\]{2,}\/?)/i, phone: /^[+]?[(]?[0-9]{3}[)]?[-\s.]?[0-9]{3}[-\s.]?[0-9]{4,6}$/im, @@ -811,17 +814,17 @@ function Framework() { uid: /^\d{14,}[a-z]{3}[01]{1}$/ }; - this.workers = {}; - this.databases = {}; - this.databasescleaner = {}; - this.directory = HEADERS.workers.cwd = directory; - this.isLE = Os.endianness ? Os.endianness() === 'LE' : true; - this.isHTTPS = false; + self.workers = {}; + self.databases = {}; + self.databasescleaner = {}; + self.directory = HEADERS.workers.cwd = directory; + self.isLE = Os.endianness ? Os.endianness() === 'LE' : true; + self.isHTTPS = false; // It's hidden - // this.waits = {}; + // self.waits = {}; - this.temporary = { + self.temporary = { path: {}, notfound: {}, processing: {}, @@ -836,7 +839,7 @@ function Framework() { service: { redirect: 0, request: 0, file: 0 } }; - this.stats = { + self.stats = { performance: { request: 0, @@ -904,29 +907,29 @@ function Framework() { }; // intialize cache - this.cache = new FrameworkCache(); - this.path = new FrameworkPath(); - - this._request_check_redirect = false; - this._request_check_referer = false; - this._request_check_POST = false; - this._request_check_robot = false; - this._request_check_mobile = false; - this._length_middleware = 0; - this._length_request_middleware = 0; - this._length_files = 0; - this._length_wait = 0; - this._length_themes = 0; - this._length_cors = 0; - this._length_subdomain_web = 0; - this._length_subdomain_websocket = 0; - this._length_convertors = 0; - - this.isVirtualDirectory = false; - this.isTheme = false; - this.isWindows = Os.platform().substring(0, 3).toLowerCase() === 'win'; - - this.$events = {}; + self.cache = new FrameworkCache(); + self.path = new FrameworkPath(); + + self._request_check_redirect = false; + self._request_check_referer = false; + self._request_check_POST = false; + self._request_check_robot = false; + self._request_check_mobile = false; + self._length_middleware = 0; + self._length_request_middleware = 0; + self._length_files = 0; + self._length_wait = 0; + self._length_themes = 0; + self._length_cors = 0; + self._length_subdomain_web = 0; + self._length_subdomain_websocket = 0; + self._length_convertors = 0; + + self.isVirtualDirectory = false; + self.isTheme = false; + self.isWindows = Os.platform().substring(0, 3).toLowerCase() === 'win'; + + self.$events = {}; } // ====================================================== @@ -6465,7 +6468,7 @@ F.initialize = function(http, debug, options) { var ip = options.ip; var listenpath = options.listenpath; - options.config && U.extend(F.config, options.config, true); + options.config && U.extend_headers2(F.config, options.config); if (options.debug || options['allow-debug']) F.config['allow-debug'] = true; @@ -14189,9 +14192,11 @@ function extend_request(PROTO) { get: function() { if (this._subdomain) return this._subdomain; - var subdomain = this.uri.host.toLowerCase().replace(/^www\./i, '').split('.'); - if (subdomain.length > 2) - this._subdomain = subdomain.slice(0, subdomain.length - 2); // example: [subdomain].domain.com + var subdomain = this.uri.hostname.toLowerCase().replace(REG_WWW, '').split('.'); + if (subdomain.length > 2) // example: [subdomain].domain.com + this._subdomain = subdomain.slice(0, subdomain.length - 2); + else if (subdomain.length > 1 && subdomain[subdomain.length - 1] === 'localhost') // example: [subdomain].localhost + this._subdomain = subdomain.slice(0, subdomain.length - 1); else this._subdomain = null; return this._subdomain; diff --git a/internal.js b/internal.js index 4762f30f8..462cdbb0c 100755 --- a/internal.js +++ b/internal.js @@ -59,7 +59,7 @@ const REG_HEAD = /<\/head>/i; const REG_COMPONENTS = /@{(\s)?(component|components)(\s)?\(/i; const REG_COMPONENTS_GROUP = /('|")[a-z0-9]+('|")/i; const HTTPVERBS = { 'get': true, 'post': true, 'options': true, 'put': true, 'delete': true, 'patch': true, 'upload': true, 'head': true, 'trace': true, 'propfind': true }; -const RENDERNOW = ['self.$import(', 'self.route', 'self.$js(', 'self.$css(', 'self.$favicon(', 'self.$script(', '$STRING(self.resource(', '$STRING(RESOURCE(', 'self.translate(', 'language', 'self.sitemap_url(', 'self.sitemap_name(', '$STRING(CONFIG(', '$STRING(config.', '$STRING(config[', '$STRING(config(']; +const RENDERNOW = ['self.$import(', 'self.route', 'self.$js(', 'self.$css(', 'self.$favicon(', 'self.$script(', '$STRING(self.resource(', '$STRING(RESOURCE(', 'self.translate(', 'language', 'self.sitemap_url(', 'self.sitemap_name(', '$STRING(CONFIG(', '$STRING(config.', '$STRING(config[', '$STRING(CONF.', '$STRING(CONF[', '$STRING(config(']; const REG_NOTRANSLATE = /@\{notranslate\}/gi; const REG_NOCOMPRESS = /@\{nocompress\s\w+}/gi; const REG_TAGREMOVE = /[^>](\r)\n\s{1,}$/; @@ -80,7 +80,7 @@ const REG_CSS_11 = /\$.*?(;|\})/gi; const REG_CSS_12 = /(margin|padding):.*?(;|})/g; const AUTOVENDOR = ['filter', 'appearance', 'column-count', 'column-gap', 'column-rule', 'display', 'transform', 'transform-style', 'transform-origin', 'transition', 'user-select', 'animation', 'perspective', 'animation-name', 'animation-duration', 'animation-timing-function', 'animation-delay', 'animation-iteration-count', 'animation-direction', 'animation-play-state', 'opacity', 'background', 'background-image', 'font-smoothing', 'text-size-adjust', 'backface-visibility', 'box-sizing', 'overflow-scrolling']; const WRITESTREAM = { flags: 'w' }; -const ALLOWEDMARKUP = { G: 1, M: 1, R: 1, repository: 1, model: 1, config: 1, global: 1, resource: 1, RESOURCE: 1, CONFIG: 1, author: 1, root: 1, functions: 1, NOW: 1, F: 1 }; +const ALLOWEDMARKUP = { G: 1, M: 1, R: 1, repository: 1, model: 1, CONF: 1, config: 1, global: 1, resource: 1, RESOURCE: 1, CONFIG: 1, author: 1, root: 1, functions: 1, NOW: 1, F: 1 }; var INDEXFILE = 0; @@ -2094,6 +2094,7 @@ function view_prepare(command, dynamicCommand, functions, controller) { case 'session': case 'user': case 'config': + case 'CONF': case 'controller': return view_is_assign(command) ? ('self.$set(' + command + ')') : ('$STRING(' + command + ').encode()'); @@ -2133,6 +2134,7 @@ function view_prepare(command, dynamicCommand, functions, controller) { case '!session': case '!user': case '!config': + case '!CONF': case '!functions': case '!model': case '!CONFIG': From 349f7ad87adc2528a4cea3099e85fb18a92eec32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 11 Sep 2018 14:32:44 +0200 Subject: [PATCH 062/217] Fixed `GET` priority in `ROUTE()`. --- index.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/index.js b/index.js index e46acdb7c..5f115f324 100755 --- a/index.js +++ b/index.js @@ -2146,6 +2146,10 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu tmp.push(flag); break; } + + if (flag === 'get') + priority -= 2; + } if (isROLE && !membertype) { From 5f7f942eb6922ea3b53dec7f5c3f224c504e587a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 13 Sep 2018 12:49:17 +0200 Subject: [PATCH 063/217] Improved `ROUTE()` by adding operations. --- builders.js | 4 ++-- changes.txt | 1 + index.js | 43 +++++++++++++++++++++++++++++++------------ 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/builders.js b/builders.js index 54168ca23..c1ddb611a 100755 --- a/builders.js +++ b/builders.js @@ -4672,7 +4672,7 @@ global.OPERATION = function(name, value, callback, param, controller) { } }; -global.RUN = function(name, value, callback, param, controller) { +global.RUN = function(name, value, callback, param, controller, result) { if (typeof(value) === 'function') { controller = param; @@ -4750,7 +4750,7 @@ global.RUN = function(name, value, callback, param, controller) { opt.meta.next = name[index]; fn(opt); - }, () => callback(error.items.length ? error : null, opt.response, opt)); + }, () => callback(error.items.length ? error : null, result ? opt.response[result] : opt.response, opt)); }; function OperationOptions(error, value, options, controller) { diff --git a/changes.txt b/changes.txt index 5af3b299f..e57ec0ac9 100755 --- a/changes.txt +++ b/changes.txt @@ -12,6 +12,7 @@ - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat` and `stop` argument +- updated: routing, now it supports operations in the form `ROUTE('.. * --> @save_operation @load_operation (response)')` - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function diff --git a/index.js b/index.js index 5f115f324..bd0c44b3b 100755 --- a/index.js +++ b/index.js @@ -1851,7 +1851,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu } - url = url.replace(/(^|\s?)\*[a-z0-9].*?$/i, function(text) { + url = url.replace(/(^|\s?)\*([a-z0-9]|\s).*?$/i, function(text) { !flags && (flags = []); flags.push(text.trim()); return ''; @@ -2024,20 +2024,23 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu workflow = null; } - schema = schema.replace(/\\/g, '/').split('/'); + schema = schema.replace(/\\/g, '/').split('/').trim(); - if (schema.length === 1) { - schema[1] = schema[0]; - schema[0] = 'default'; - } + if (schema.length) { - index = schema[1].indexOf('#'); - if (index !== -1) { - schema[2] = schema[1].substring(index + 1).trim(); - schema[1] = schema[1].substring(0, index).trim(); - (schema[2] && schema[2][0] !== '*') && (schema[2] = '*' + schema[2]); - } + if (schema.length === 1) { + schema[1] = schema[0]; + schema[0] = 'default'; + } + index = schema[1].indexOf('#'); + if (index !== -1) { + schema[2] = schema[1].substring(index + 1).trim(); + schema[1] = schema[1].substring(0, index).trim(); + (schema[2] && schema[2][0] !== '*') && (schema[2] = '*' + schema[2]); + } + + } // else it's an operation continue; } @@ -16618,8 +16621,17 @@ function controller_json_workflow(id) { var self = this; self.id = id; var w = self.route.workflow; + if (w instanceof Object) { if (!w.type) { + + + // IS IT AN OPERATION? + if (!self.route.schema.length) { + OPERATION(w.id, EMPTYOBJECT, w.view ? self.callback(w.view) : self.callback(), self); + return; + } + var schema = GETSCHEMA(self.route.schema[0], self.route.schema[1]); if (!schema) { @@ -16662,6 +16674,13 @@ function controller_json_workflow_multiple(id) { var w = self.route.workflow; if (w instanceof Object) { if (!w.type) { + + // IS IT AN OPERATION? + if (!self.route.schema.length) { + RUN(w.id, EMPTYOBJECT, w.view ? self.callback(w.view) : self.callback(), null, self, w.index ? w.id[w.index] : null); + return; + } + var schema = GETSCHEMA(self.route.schema[0], self.route.schema[1]); if (!schema) { From ae9c62fb580f6128633b27f5ee1560ffd472d4a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 13 Sep 2018 16:31:20 +0200 Subject: [PATCH 064/217] Added `nosql builder cache` (uncompleted). --- nosql.js | 305 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 190 insertions(+), 115 deletions(-) diff --git a/nosql.js b/nosql.js index 177293b08..64183e42a 100755 --- a/nosql.js +++ b/nosql.js @@ -2408,6 +2408,13 @@ DatabaseBuilder.prototype.join = function(field, name) { return join; }; +DatabaseBuilder.prototype.statement = function(id, fn) { + this.id = id; + this.$iscache = CACHE[self.db.name + '_' + id] == null; + fn.call(this, this); + return this; +}; + DatabaseBuilder.prototype.first = function() { this.$options.first = true; return this.take(1); @@ -2425,13 +2432,15 @@ DatabaseBuilder.prototype.filter = function(fn) { self.$functions = []; var index = self.$functions.push(fn) - 1; - var code = '$is=!!fn[{0}].call($F,doc,index,repository);'.format(index); - if (self.$scope) - code = 'if(!$is){' + code + '}'; + if (!self.$iscache) { + var code = '$is=!!fn[{0}].call($F,doc,index,repository);'.format(index); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + } - self.$code.push(code); - !self.$scope && self.$code.push('if(!$is)return;'); self.$counter++; return self; }; @@ -2446,33 +2455,43 @@ DatabaseBuilder.prototype.scalar = function(type, name) { DatabaseBuilder.prototype.contains = function(name) { var self = this; - var code = '$is=doc.{0} instanceof Array?!!doc.{0}.length:!!doc.{0};'.format(name); - if (self.$scope) - code = 'if(!$is){' + code + '}'; - self.$code.push(code); - !self.$scope && self.$code.push('if(!$is)return;'); + + if (!self.$iscache) { + var code = '$is=doc.{0} instanceof Array?!!doc.{0}.length:!!doc.{0};'.format(name); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + } + self.$counter++; return self; }; DatabaseBuilder.prototype.empty = function(name) { var self = this; - var code = '$is=doc.{0} instanceof Array?!doc.{0}.length:!doc.{0};'.format(name); - if (self.$scope) - code = 'if(!$is){' + code + '}'; - self.$code.push(code); - !self.$scope && self.$code.push('if(!$is)return;'); + + if (!self.$iscache) { + var code = '$is=doc.{0} instanceof Array?!doc.{0}.length:!doc.{0};'.format(name); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + } + self.$counter++; return self; }; DatabaseBuilder.prototype.map = function(name, code) { var self = this; - var data = { name: name, code: code }; - if (self.$options.mappers) - self.$options.mappers.push(data); - else - self.$options.mappers = [data]; + if (!self.$iscache) { + var data = { name: name, code: code }; + if (self.$options.mappers) + self.$options.mappers.push(data); + else + self.$options.mappers = [data]; + } return self; }; @@ -2503,39 +2522,43 @@ DatabaseBuilder.prototype.where = function(name, operator, value) { var date = framework_utils.isDate(value); self.$params[key] = date ? value.getTime() : value; - switch (operator) { - case '=': - operator = '=='; - break; - case '<>': - operator = '!='; - break; - } - - code = (date ? '$is=(doc.{0} instanceof Date?doc.{0}:new Date(doc.{0})).getTime(){2}arg.{1};' : '$is=doc.{0}{2}arg.{1};'); + if (!self.$iscache) { + switch (operator) { + case '=': + operator = '=='; + break; + case '<>': + operator = '!='; + break; + } - if (self.$scope) - code = 'if(!$is){' + code + '}'; + code = (date ? '$is=(doc.{0} instanceof Date?doc.{0}:new Date(doc.{0})).getTime(){2}arg.{1};' : '$is=doc.{0}{2}arg.{1};'); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + self.$code.push(code.format(name, key, operator)); + !self.$scope && self.$code.push('if(!$is)return;'); + self.$keys && self.$keys.push(name); + } - self.$keys && self.$keys.push(name); - self.$code.push(code.format(name, key, operator)); - !self.$scope && self.$code.push('if(!$is)return;'); return self; }; DatabaseBuilder.prototype.query = function(code) { var self = this; - code = '$is=(' + code + ');'; + if (!self.$iscache) { + code = '$is=(' + code + ');'; - if (self.$scope) - code = 'if(!$is){' + code + '}'; + if (self.$scope) + code = 'if(!$is){' + code + '}'; - if (self.$keys) - self.$keys = null; + if (self.$keys) + self.$keys = null; + + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + } - self.$code.push(code); - !self.$scope && self.$code.push('if(!$is)return;'); self.$counter++; return self; }; @@ -2551,13 +2574,14 @@ DatabaseBuilder.prototype.month = function(name, operator, value) { self.$params[key] = value; - var code = compare_datetype('month', name, key, operator); - if (self.$scope) - code = 'if(!$is){' + code + '}'; - - self.$keys && self.$keys.push(name); - self.$code.push(code); - !self.$scope && self.$code.push('if(!$is)return;'); + if (!self.$iscache) { + var code = compare_datetype('month', name, key, operator); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + self.$keys && self.$keys.push(name); + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + } return self; }; @@ -2572,13 +2596,15 @@ DatabaseBuilder.prototype.day = function(name, operator, value) { self.$params[key] = value; - var code = compare_datetype('day', name, key, operator); - if (self.$scope) - code = 'if(!$is){' + code + '}'; + if (!self.$iscache) { + var code = compare_datetype('day', name, key, operator); + if (self.$scope) + code = 'if(!$is){' + code + '}'; - self.$keys && self.$keys.push(name); - self.$code.push(code); - !self.$scope && self.$code.push('if(!$is)return;'); + self.$keys && self.$keys.push(name); + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + } return self; }; @@ -2593,14 +2619,17 @@ DatabaseBuilder.prototype.year = function(name, operator, value) { self.$params[key] = value; - var code = compare_datetype('year', name, key, operator); - if (self.$scope) - code = 'if(!$is){' + code + '}'; + if (!self.$iscache) { + var code = compare_datetype('year', name, key, operator); + if (self.$scope) + code = 'if(!$is){' + code + '}'; - self.$keys && self.$keys.push(name); + self.$keys && self.$keys.push(name); + + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + } - self.$code.push(code); - !self.$scope && self.$code.push('if(!$is)return;'); return self; }; @@ -2615,13 +2644,16 @@ DatabaseBuilder.prototype.hour = function(name, operator, value) { self.$params[key] = value; - var code = compare_datetype('hour', name, key, operator); - if (self.$scope) - code = 'if(!$is){' + code + '}'; + if (!self.$iscache) { + var code = compare_datetype('hour', name, key, operator); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + + self.$keys && self.$keys.push(name); + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + } - self.$keys && self.$keys.push(name); - self.$code.push(code); - !self.$scope && self.$code.push('if(!$is)return;'); return self; }; @@ -2636,13 +2668,16 @@ DatabaseBuilder.prototype.minute = function(name, operator, value) { self.$params[key] = value; - var code = compare_datetype('minute', name, key, operator); - if (self.$scope) - code = 'if(!$is){' + code + '}'; + if (!self.$iscache) { + var code = compare_datetype('minute', name, key, operator); + if (self.$scope) + code = 'if(!$is){' + code + '}'; + + self.$keys && self.$keys.push(name); + self.$code.push(code); + !self.$scope && self.$code.push('if(!$is)return;'); + } - self.$keys && self.$keys.push(name); - self.$code.push(code); - !self.$scope && self.$code.push('if(!$is)return;'); return self; }; @@ -2652,44 +2687,55 @@ DatabaseBuilder.prototype.like = DatabaseBuilder.prototype.search = function(nam var code; var key = 'l' + (self.$counter++); - if (!where) - where = '*'; + if (!self.$iscache) { + if (!where) + where = '*'; - switch (where) { - case 'beg': - code = '$is=doc.{0}?doc.{0}.startsWith(arg.{1}):false;'; - break; - case 'end': - code = '$is=doc.{0}?doc.{0}.endsWith(arg.{1}):false;'; - break; - case '*': - code = '$is=false;if(doc.{0}){if(doc.{0} instanceof Array){for(var $i=0;$i> 0; - - var code = '$is=false;if(doc.{0}&&doc.{0}.toLowerCase){var $a={2},$b=doc.{0}.toLowerCase();for(var $i=0;$i> 0; + var code = '$is=false;if(doc.{0}&&doc.{0}.toLowerCase){var $a={2},$b=doc.{0}.toLowerCase();for(var $i=0;$i Date: Thu, 13 Sep 2018 19:10:47 +0200 Subject: [PATCH 065/217] Improved code. --- nosql.js | 185 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 107 insertions(+), 78 deletions(-) diff --git a/nosql.js b/nosql.js index 64183e42a..34b4421c0 100755 --- a/nosql.js +++ b/nosql.js @@ -2408,19 +2408,18 @@ DatabaseBuilder.prototype.join = function(field, name) { return join; }; -DatabaseBuilder.prototype.statement = function(id, fn) { - this.id = id; - this.$iscache = CACHE[self.db.name + '_' + id] == null; - fn.call(this, this); - return this; -}; - DatabaseBuilder.prototype.first = function() { this.$options.first = true; return this.take(1); }; -DatabaseBuilder.prototype.make = function(fn) { +DatabaseBuilder.prototype.make = function(id, fn) { + if (fn == null) { + fn = id; + } else { + this.$options.id = id; + this.$iscache = !!CACHE[this.db.name + '_' + id]; + } fn.call(this, this); return this; }; @@ -2743,6 +2742,7 @@ DatabaseBuilder.prototype.fulltext = function(name, value, weight) { var self = this; var key = 'l' + (self.$counter++); + var key2 = 'l' + (self.$counter++); if (value instanceof Array) { for (var i = 0; i < value.length; i++) @@ -2756,17 +2756,22 @@ DatabaseBuilder.prototype.fulltext = function(name, value, weight) { self.$params[key] = value; + var count = 1; + + if (weight) + count = ((value.length / 100) * weight) >> 0; + + self.$params[key2] = count || 1; + if (!self.$iscache) { self.$keys && self.$keys.push(name); - var count = 1; - if (weight) - count = ((value.length / 100) * weight) >> 0; - var code = '$is=false;if(doc.{0}&&doc.{0}.toLowerCase){var $a={2},$b=doc.{0}.toLowerCase();for(var $i=0;$i Date: Thu, 13 Sep 2018 19:17:51 +0200 Subject: [PATCH 066/217] Improved code. --- nosql.js | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/nosql.js b/nosql.js index 34b4421c0..c773820cb 100755 --- a/nosql.js +++ b/nosql.js @@ -2777,21 +2777,22 @@ DatabaseBuilder.prototype.fulltext = function(name, value, weight) { DatabaseBuilder2.prototype.stringify = DatabaseBuilder.prototype.stringify = function() { + var self = this; var obj = {}; - obj.options = this.$options; - obj.code = this.$code; - obj.params = this.$params; - obj.insert = this.$insertcallback ? this.$insertcallback.toString() : null; + obj.options = self.$options; + obj.code = self.$code; + obj.params = self.$params; + obj.insert = self.$insertcallback ? self.$insertcallback.toString() : null; - if (this.$functions) { + if (self.$functions) { obj.functions = []; - for (var i = 0; i < this.$functions.length; i++) - obj.functions.push(this.$functions[i].toString()); + for (var i = 0; i < self.$functions.length; i++) + obj.functions.push(self.$functions[i].toString()); } - if (this.$repository) - obj.repository = this.$repository; + if (self.$repository) + obj.repository = self.$repository; return JSON.stringify(obj); }; From 38caa139b52aafd96b7c2e2c67c1358dc94a72eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 13 Sep 2018 19:53:58 +0200 Subject: [PATCH 067/217] Improved code again. --- nosql.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/nosql.js b/nosql.js index c773820cb..7f3230fbf 100755 --- a/nosql.js +++ b/nosql.js @@ -2908,7 +2908,6 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { self.$mappersexec = cache.mexec; self.$keys = cache.keys; self.$options.sort = cache.sort; - self.$functions = cache.functions; return cache.filter; } @@ -2916,14 +2915,13 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { var code = 'var repository=$F.repository,options=$F.options,arg=$F.arg,fn=$F.fn,$is=false,$tmp;var R=repository;' + raw + (self.$code.length && raw.substring(raw.length - 7) !== 'return;' ? 'if(!$is)return;' : '') + (noTrimmer ? 'return doc' : 'if(options.fields){var $doc={};for(var $i=0;$i Date: Sat, 15 Sep 2018 09:44:56 +0200 Subject: [PATCH 068/217] Improved OPERATIONS. --- builders.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builders.js b/builders.js index c1ddb611a..150b4840a 100755 --- a/builders.js +++ b/builders.js @@ -4714,7 +4714,7 @@ global.RUN = function(name, value, callback, param, controller, result) { if (opt.error.items.length && opt.$repeat > 0) { opt.error.clear(); opt.$repeat--; - opt.repeated = true; + opt.repeated++; setImmediate(opt => opt.$current(opt), opt); } else { opt.error.items.length && error.push(opt.error); @@ -4739,7 +4739,7 @@ global.RUN = function(name, value, callback, param, controller, result) { return next(); } - opt.repeated = false; + opt.repeated = 0; opt.error = new ErrorBuilder(); opt.error.path = 'operation: ' + key; opt.meta.index = index; From 73af913bd8cce068433c94f730ab71f8cdeac6df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 15 Sep 2018 23:31:37 +0200 Subject: [PATCH 069/217] Fixed NoSQL cleaner + improved code. --- changes.txt | 1 + nosql.js | 67 ++++++++++++++++++++--------------------------------- 2 files changed, 26 insertions(+), 42 deletions(-) diff --git a/changes.txt b/changes.txt index e57ec0ac9..103cdb3d8 100755 --- a/changes.txt +++ b/changes.txt @@ -24,6 +24,7 @@ - fixed: `SchemaInstance.$clean()` for nested schemas - fixed: extracting `bundles` (added `/flow/` and `/dashboard/`) - fixed: subdomain routing for `localhost` +- fixed: service for database cleaner - improved: NoSQL reader diff --git a/nosql.js b/nosql.js index 7f3230fbf..b626935de 100755 --- a/nosql.js +++ b/nosql.js @@ -1580,7 +1580,10 @@ DP.$update = function() { } } - change && self.$events.change && self.emit('change', 'update'); + if (change) { + self.$events.change && self.emit('change', 'update'); + !F.databasescleaner[self.name] && (F.databasescleaner[self.name] = 1); + } }; fs.openupdate(); @@ -1952,7 +1955,10 @@ DP.$remove = function() { fs = null; self.$writting = false; self.next(0); - change && self.$events.change && self.emit('change', 'remove'); + if (change) { + self.$events.change && self.emit('change', 'remove'); + !F.databasescleaner[self.name] && (F.databasescleaner[self.name] = 1); + } }; fs.openupdate(); @@ -2413,10 +2419,8 @@ DatabaseBuilder.prototype.first = function() { return this.take(1); }; -DatabaseBuilder.prototype.make = function(id, fn) { - if (fn == null) { - fn = id; - } else { +DatabaseBuilder.prototype.make = function(fn, id) { + if (id) { this.$options.id = id; this.$iscache = !!CACHE[this.db.name + '_' + id]; } @@ -2530,13 +2534,11 @@ DatabaseBuilder.prototype.where = function(name, operator, value) { operator = '!='; break; } - code = (date ? '$is=(doc.{0} instanceof Date?doc.{0}:new Date(doc.{0})).getTime(){2}arg.{1};' : '$is=doc.{0}{2}arg.{1};'); if (self.$scope) code = 'if(!$is){' + code + '}'; self.$code.push(code.format(name, key, operator)); !self.$scope && self.$code.push('if(!$is)return;'); - self.$keys && self.$keys.push(name); } return self; @@ -2547,13 +2549,8 @@ DatabaseBuilder.prototype.query = function(code) { if (!self.$iscache) { code = '$is=(' + code + ');'; - if (self.$scope) code = 'if(!$is){' + code + '}'; - - if (self.$keys) - self.$keys = null; - self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); } @@ -2562,6 +2559,11 @@ DatabaseBuilder.prototype.query = function(code) { return self; }; +DatabaseBuilder.prototype.param = function(key, value) { + this.$params[key] = value; + return this; +}; + DatabaseBuilder.prototype.month = function(name, operator, value) { var self = this; var key = 'dm' + (self.$counter++); @@ -2577,7 +2579,6 @@ DatabaseBuilder.prototype.month = function(name, operator, value) { var code = compare_datetype('month', name, key, operator); if (self.$scope) code = 'if(!$is){' + code + '}'; - self.$keys && self.$keys.push(name); self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); } @@ -2599,8 +2600,6 @@ DatabaseBuilder.prototype.day = function(name, operator, value) { var code = compare_datetype('day', name, key, operator); if (self.$scope) code = 'if(!$is){' + code + '}'; - - self.$keys && self.$keys.push(name); self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); } @@ -2622,9 +2621,6 @@ DatabaseBuilder.prototype.year = function(name, operator, value) { var code = compare_datetype('year', name, key, operator); if (self.$scope) code = 'if(!$is){' + code + '}'; - - self.$keys && self.$keys.push(name); - self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); } @@ -2647,8 +2643,6 @@ DatabaseBuilder.prototype.hour = function(name, operator, value) { var code = compare_datetype('hour', name, key, operator); if (self.$scope) code = 'if(!$is){' + code + '}'; - - self.$keys && self.$keys.push(name); self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); } @@ -2671,8 +2665,6 @@ DatabaseBuilder.prototype.minute = function(name, operator, value) { var code = compare_datetype('minute', name, key, operator); if (self.$scope) code = 'if(!$is){' + code + '}'; - - self.$keys && self.$keys.push(name); self.$code.push(code); !self.$scope && self.$code.push('if(!$is)return;'); } @@ -2710,7 +2702,6 @@ DatabaseBuilder.prototype.like = DatabaseBuilder.prototype.search = function(nam if (self.$scope) code = 'if(!$is){' + code + '}'; - self.$keys && self.$keys.push(name); self.$code.push(code.format(name, key)); !self.$scope && self.$code.push('if(!$is)return;'); } else { @@ -2731,7 +2722,6 @@ DatabaseBuilder.prototype.regexp = function(name, value) { var code = '$is=false;if(doc.{0}&&doc.{0}.toLowerCase){$is=({1}).test(doc.{0})}'; if (self.$scope) code = 'if(!$is){' + code + '}'; - self.$keys && self.$keys.push(name); self.$code.push(code.format(name, value.toString())); !self.$scope && self.$code.push('if(!$is)return;'); } @@ -2764,7 +2754,6 @@ DatabaseBuilder.prototype.fulltext = function(name, value, weight) { self.$params[key2] = count || 1; if (!self.$iscache) { - self.$keys && self.$keys.push(name); var code = '$is=false;if(doc.{0}&&doc.{0}.toLowerCase){var $a=arg.{2},$b=doc.{0}.toLowerCase();for(var $i=0;$i Date: Sat, 15 Sep 2018 23:38:27 +0200 Subject: [PATCH 070/217] Added a missing `repository`. --- nosql.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nosql.js b/nosql.js index b626935de..77b855743 100755 --- a/nosql.js +++ b/nosql.js @@ -1498,7 +1498,7 @@ DP.$update = function() { var val = fil.doc[key]; if (val !== undefined) { if (typeof(val) === 'function') - doc[key] = val(doc[key], doc); + doc[key] = val(doc[key], doc, f.filter.repository); else if (fil.inc && fil.inc[key]) { switch (fil.inc[key]) { case '!': @@ -5956,7 +5956,7 @@ TP.$update = function() { var val = fil.doc[key]; if (val !== undefined) { if (typeof(val) === 'function') - doc[key] = val(doc[key], doc); + doc[key] = val(doc[key], doc, f.filter.repository); else if (fil.inc && fil.inc[key]) { switch (fil.inc[key]) { case '!': @@ -5992,7 +5992,6 @@ TP.$update = function() { var rec = fs.docsbuffer[dindex]; var upd = self.stringify(doc, null, rec.length); - if (upd === rec.doc) return; From 8798ef2a2fd81efed29606194d7aeeaa3ada392d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 16 Sep 2018 09:33:53 +0200 Subject: [PATCH 071/217] Improved NoSQL. --- changes.txt | 1 + nosql.js | 42 +++++++++++++++++++++--------------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/changes.txt b/changes.txt index 103cdb3d8..8fe27b629 100755 --- a/changes.txt +++ b/changes.txt @@ -9,6 +9,7 @@ - added: NOSQL/TABLE supports `!field` as a boolean toggle in modify/update method - added: `RUN()` for executing multiple Total.js operations - added: a new global alias `CONF` (it's a reference to config) for `F.config` +- added: `DatabaseBuilder.arg(key, value)` for adding an dynamic argument - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat` and `stop` argument diff --git a/nosql.js b/nosql.js index 77b855743..80e7e03ad 100755 --- a/nosql.js +++ b/nosql.js @@ -2180,7 +2180,7 @@ function DatabaseBuilder(db) { this.$scope = 0; this.$callback = NOOP; this.$code = []; - this.$params = {}; + this.$args = {}; this.$options = {}; this.$repository = {}; this.$counter = 0; @@ -2190,7 +2190,7 @@ function DatabaseBuilder(db) { DatabaseBuilder.prototype.promise = promise; DatabaseBuilder.prototype.makefilter = function() { - return { repository: this.$repository, options: this.$options, arg: this.$params, fn: this.$functions }; + return { repository: this.$repository, options: this.$options, arg: this.$args, fn: this.$functions }; }; DatabaseBuilder.prototype.id = function(id) { @@ -2258,7 +2258,7 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { builder.$code = join.builder.$code.slice(0); U.extend_headers2(builder.$options, join.builder.$options); builder.$repository = join.builder.$repository; - builder.$params = CLONE(join.builder.$params); + builder.$args = CLONE(join.builder.$args); } builder.$take = join.builder.$take; @@ -2523,7 +2523,7 @@ DatabaseBuilder.prototype.where = function(name, operator, value) { } var date = framework_utils.isDate(value); - self.$params[key] = date ? value.getTime() : value; + self.$args[key] = date ? value.getTime() : value; if (!self.$iscache) { switch (operator) { @@ -2559,8 +2559,8 @@ DatabaseBuilder.prototype.query = function(code) { return self; }; -DatabaseBuilder.prototype.param = function(key, value) { - this.$params[key] = value; +DatabaseBuilder.prototype.arg = function(key, value) { + this.$args[key] = value; return this; }; @@ -2573,7 +2573,7 @@ DatabaseBuilder.prototype.month = function(name, operator, value) { operator = '='; } - self.$params[key] = value; + self.$args[key] = value; if (!self.$iscache) { var code = compare_datetype('month', name, key, operator); @@ -2594,7 +2594,7 @@ DatabaseBuilder.prototype.day = function(name, operator, value) { operator = '='; } - self.$params[key] = value; + self.$args[key] = value; if (!self.$iscache) { var code = compare_datetype('day', name, key, operator); @@ -2615,7 +2615,7 @@ DatabaseBuilder.prototype.year = function(name, operator, value) { operator = '='; } - self.$params[key] = value; + self.$args[key] = value; if (!self.$iscache) { var code = compare_datetype('year', name, key, operator); @@ -2637,7 +2637,7 @@ DatabaseBuilder.prototype.hour = function(name, operator, value) { operator = '='; } - self.$params[key] = value; + self.$args[key] = value; if (!self.$iscache) { var code = compare_datetype('hour', name, key, operator); @@ -2659,7 +2659,7 @@ DatabaseBuilder.prototype.minute = function(name, operator, value) { operator = '='; } - self.$params[key] = value; + self.$args[key] = value; if (!self.$iscache) { var code = compare_datetype('minute', name, key, operator); @@ -2697,7 +2697,7 @@ DatabaseBuilder.prototype.like = DatabaseBuilder.prototype.search = function(nam break; } - self.$params[key] = value; + self.$args[key] = value; if (self.$scope) code = 'if(!$is){' + code + '}'; @@ -2710,7 +2710,7 @@ DatabaseBuilder.prototype.like = DatabaseBuilder.prototype.search = function(nam value = value.join(' '); value = value.toLowerCase(); } - self.$params[key] = value; + self.$args[key] = value; } return self; @@ -2744,14 +2744,14 @@ DatabaseBuilder.prototype.fulltext = function(name, value, weight) { value = value.toLowerCase().split(' '); } - self.$params[key] = value; + self.$args[key] = value; var count = 1; if (weight) count = ((value.length / 100) * weight) >> 0; - self.$params[key2] = count || 1; + self.$args[key2] = count || 1; if (!self.$iscache) { var code = '$is=false;if(doc.{0}&&doc.{0}.toLowerCase){var $a=arg.{2},$b=doc.{0}.toLowerCase();for(var $i=0;$i Date: Sun, 16 Sep 2018 10:12:30 +0200 Subject: [PATCH 072/217] Improved NoSQL modify/update. --- changes.txt | 2 ++ nosql.js | 51 +++++++++++++++++++++++---------------------------- 2 files changed, 25 insertions(+), 28 deletions(-) diff --git a/changes.txt b/changes.txt index 8fe27b629..f074e107b 100755 --- a/changes.txt +++ b/changes.txt @@ -10,6 +10,8 @@ - added: `RUN()` for executing multiple Total.js operations - added: a new global alias `CONF` (it's a reference to config) for `F.config` - added: `DatabaseBuilder.arg(key, value)` for adding an dynamic argument +- added: NOSQL/TABLE modify supports a new type `$age: 'js_code'` with some JS code +- added: NOSQL/TABLE update supports a new type `'js_code'` with some JS code - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat` and `stop` argument diff --git a/nosql.js b/nosql.js index 80e7e03ad..f72c4bc3c 100755 --- a/nosql.js +++ b/nosql.js @@ -873,6 +873,8 @@ DP.update = function(doc, insert) { var builder = new DatabaseBuilder(self); var data = framework_builders.isSchema(doc) ? doc.$clean() : doc; builder.$options.readertype = 1; + if (typeof(data) === 'string') + data = new Function('doc', 'repository', 'arg', data.indexOf('return ') === -1 ? ('return (' + data + ')') : data); self.pending_update.push({ builder: builder, doc: data, insert: insert === true ? data : insert }); setImmediate(next_operation, self, 2); return builder; @@ -889,6 +891,7 @@ DP.modify = function(doc, insert) { builder.$options.readertype = 1; if (keys.length) { + var tmp; for (var i = 0; i < keys.length; i++) { var key = keys[i]; switch (key[0]) { @@ -898,12 +901,18 @@ DP.modify = function(doc, insert) { case '*': case '/': !inc && (inc = {}); - var tmp = key.substring(1); + tmp = key.substring(1); inc[tmp] = key[0]; doc[tmp] = doc[key]; doc[key] = undefined; keys[i] = tmp; break; + case '$': + tmp = key.substring(1); + doc[tmp] = new Function('val', 'doc', 'repository', 'arg', doc[key].indexOf('return ') === -1 ? ('return (' + doc[key] + ')') : doc[key]); + doc[key] = undefined; + keys[i] = tmp; + break; } } self.pending_update.push({ builder: builder, doc: data, keys: keys, inc: inc, insert: insert === true ? data : insert }); @@ -1498,7 +1507,7 @@ DP.$update = function() { var val = fil.doc[key]; if (val !== undefined) { if (typeof(val) === 'function') - doc[key] = val(doc[key], doc, f.filter.repository); + doc[key] = val(doc[key], doc, f.filter.repository, f.filter.arg); else if (fil.inc && fil.inc[key]) { switch (fil.inc[key]) { case '!': @@ -1522,7 +1531,7 @@ DP.$update = function() { } } } else - docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc, f.filter.repository) : fil.doc; + docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc, f.filter.repository, f.filter.arg) : fil.doc; self.$events[e] && self.emit(e, doc, old); f.builder.$options.backup && f.builder.$backupdoc(rec.doc); @@ -2546,7 +2555,6 @@ DatabaseBuilder.prototype.where = function(name, operator, value) { DatabaseBuilder.prototype.query = function(code) { var self = this; - if (!self.$iscache) { code = '$is=(' + code + ');'; if (self.$scope) @@ -2900,7 +2908,7 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { } var raw = self.$code.join(''); - var code = 'var repository=$F.repository,options=$F.options,arg=$F.arg,fn=$F.fn,$is=false,$tmp;var R=repository;' + raw + (self.$code.length && raw.substring(raw.length - 7) !== 'return;' ? 'if(!$is)return;' : '') + (noTrimmer ? 'return doc' : 'if(options.fields){var $doc={};for(var $i=0;$i Date: Sun, 16 Sep 2018 11:23:45 +0200 Subject: [PATCH 073/217] Updated change log. --- changes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index f074e107b..7421e3892 100755 --- a/changes.txt +++ b/changes.txt @@ -6,10 +6,10 @@ - added: `bundles` supports merging files between bundle and project, project file must start with e.g. `--name.js` - added: `.bundlesignore` support with similiar functionality like `.gitignore` - added: support for `SameSite` cookie attribute -- added: NOSQL/TABLE supports `!field` as a boolean toggle in modify/update method - added: `RUN()` for executing multiple Total.js operations - added: a new global alias `CONF` (it's a reference to config) for `F.config` - added: `DatabaseBuilder.arg(key, value)` for adding an dynamic argument +- added: NOSQL/TABLE modify supports `!field` as boolean toggle - added: NOSQL/TABLE modify supports a new type `$age: 'js_code'` with some JS code - added: NOSQL/TABLE update supports a new type `'js_code'` with some JS code From 350cba956bdcf2b25f2f7583acc993acab4bfc83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 16 Sep 2018 18:43:31 +0200 Subject: [PATCH 074/217] Improved NoSQL updating. --- nosql.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index f72c4bc3c..09eda7b43 100755 --- a/nosql.js +++ b/nosql.js @@ -1531,7 +1531,7 @@ DP.$update = function() { } } } else - docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc, f.filter.repository, f.filter.arg) : fil.doc; + docs[dindex] = typeof(fil.doc) === 'function' ? (fil.doc(doc, f.filter.repository, f.filter.arg) || doc) : fil.doc; self.$events[e] && self.emit(e, doc, old); f.builder.$options.backup && f.builder.$backupdoc(rec.doc); @@ -5975,7 +5975,7 @@ TP.$update = function() { } } } else - docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc, f.filter.repository, f.filter.arg) : fil.doc; + docs[dindex] = typeof(fil.doc) === 'function' ? (fil.doc(doc, f.filter.repository, f.filter.arg) || doc) : fil.doc; self.$events[e] && self.emit(e, doc, old); f.builder.$options.backup && f.builder.$backupdoc(rec.doc); From 979cfd58244fdbac138cbe12ea739a7e335cffb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 16 Sep 2018 21:36:44 +0200 Subject: [PATCH 075/217] Improved code. --- 503.html | 1 - 1 file changed, 1 deletion(-) diff --git a/503.html b/503.html index 45ffc1e30..9b82dfe69 100644 --- a/503.html +++ b/503.html @@ -23,7 +23,6 @@ 50%{ transform:scale(1.5) rotate(180deg); color:gray } 100%{ transform:scale(1); } } - From 61120ad9bd1abc9ac4fc4517466e6cfc69239d6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 16 Sep 2018 22:35:29 +0200 Subject: [PATCH 076/217] Updated version. --- builders.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builders.js b/builders.js index 150b4840a..a4a3b79f6 100755 --- a/builders.js +++ b/builders.js @@ -21,7 +21,7 @@ /** * @module FrameworkBuilders - * @version 3.0.0 + * @version 3.1.0 */ 'use strict'; From 751523d441a7ecec32f0bb2df3a709839dfdbff5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 17 Sep 2018 09:51:30 +0200 Subject: [PATCH 077/217] Improved operations. --- builders.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/builders.js b/builders.js index a4a3b79f6..0896483ac 100755 --- a/builders.js +++ b/builders.js @@ -4723,7 +4723,11 @@ global.RUN = function(name, value, callback, param, controller, result) { opt.next = null; callback(error, opt.response, opt); } else { - opt.response[opt.meta.current] = value; + + if (result && (result === opt.meta.current || result === opt.name)) + opt.output = value; + + opt.response[opt.name] = value; opt.meta.prev = opt.meta.current; opt.$next(); } @@ -4742,6 +4746,7 @@ global.RUN = function(name, value, callback, param, controller, result) { opt.repeated = 0; opt.error = new ErrorBuilder(); opt.error.path = 'operation: ' + key; + opt.name = opt.meta.current; opt.meta.index = index; opt.meta.current = key; opt.$repeat = fn.$repeat; @@ -4750,7 +4755,7 @@ global.RUN = function(name, value, callback, param, controller, result) { opt.meta.next = name[index]; fn(opt); - }, () => callback(error.items.length ? error : null, result ? opt.response[result] : opt.response, opt)); + }, () => callback(error.items.length ? error : null, result ? opt.output : opt.response, opt)); }; function OperationOptions(error, value, options, controller) { From 59dc01721cf8796685aa943f754eadb415ccfe81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 17 Sep 2018 09:58:54 +0200 Subject: [PATCH 078/217] Improved operation again. --- builders.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/builders.js b/builders.js index 0896483ac..b8f409d20 100755 --- a/builders.js +++ b/builders.js @@ -4675,6 +4675,7 @@ global.OPERATION = function(name, value, callback, param, controller) { global.RUN = function(name, value, callback, param, controller, result) { if (typeof(value) === 'function') { + result = controller; controller = param; param = callback; callback = value; @@ -4682,10 +4683,13 @@ global.RUN = function(name, value, callback, param, controller, result) { } if (param instanceof global.Controller || (param && param.isWebSocket)) { + result = controller; controller = param; param = EMPTYOBJECT; - } else if (param instanceof OperationOptions) + } else if (param instanceof OperationOptions) { + result = controller; controller = param.controller; + } if (typeof(name) === 'string') name = name.split(',').trim(); From 630f1566b4092e42974886700cd3d1baf0c034f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 17 Sep 2018 10:10:08 +0200 Subject: [PATCH 079/217] Improved OPERATIONS. --- builders.js | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/builders.js b/builders.js index b8f409d20..02993f563 100755 --- a/builders.js +++ b/builders.js @@ -4691,6 +4691,16 @@ global.RUN = function(name, value, callback, param, controller, result) { controller = param.controller; } + if (!result) { + if (typeof(param) === 'string') { + result = param; + param = EMPTYOBJECT; + } else if (typeof(controller) === 'string') { + result = controller; + controller = null; + } + } + if (typeof(name) === 'string') name = name.split(',').trim(); @@ -4750,9 +4760,8 @@ global.RUN = function(name, value, callback, param, controller, result) { opt.repeated = 0; opt.error = new ErrorBuilder(); opt.error.path = 'operation: ' + key; - opt.name = opt.meta.current; opt.meta.index = index; - opt.meta.current = key; + opt.name = opt.meta.current = key; opt.$repeat = fn.$repeat; opt.$current = fn; opt.$next = next; From 893c668a8d7ec043e9bef0c90ed4b18ff4d939c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 17 Sep 2018 11:24:32 +0200 Subject: [PATCH 080/217] Improved error handling. --- nosql.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nosql.js b/nosql.js index 09eda7b43..5addc443a 100755 --- a/nosql.js +++ b/nosql.js @@ -584,7 +584,7 @@ function Table(name, filename) { if (schema && t.stringifySchema() !== schema) t.extend(schema); - }).on('error', function() { + }).on('error', function(e) { if (schema) { t.parseSchema(schema.replace(/;|,/g, '|').trim().split('|')); Fs.writeFileSync(t.filename, t.stringifySchema() + NEWLINE, 'utf8'); @@ -601,7 +601,7 @@ function Table(name, filename) { t.pending_locks.length && (t.pending_locks = []); t.pending_clean.length && (t.pending_clean = []); t.pending_clear.length && (t.pending_clear = []); - t.throwReadonly(); + t.throwReadonly(e); } }); } @@ -1168,8 +1168,8 @@ DP.stream = function(fn, repository, callback) { return self; }; -DP.throwReadonly = function() { - throw new Error('Database "{0}" is readonly.'.format(this.name)); +DP.throwReadonly = function(e) { + throw new Error('Database "{0}" is readonly.'.format(this.name) + (e ? '\n' + e.toString() : ''); }; DP.scalar = function(type, field) { From 7f8a571c8280a9ca77ce38b1ab9c98e262d41094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 17 Sep 2018 11:25:54 +0200 Subject: [PATCH 081/217] Fixed update. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 5addc443a..2355e51b7 100755 --- a/nosql.js +++ b/nosql.js @@ -1169,7 +1169,7 @@ DP.stream = function(fn, repository, callback) { }; DP.throwReadonly = function(e) { - throw new Error('Database "{0}" is readonly.'.format(this.name) + (e ? '\n' + e.toString() : ''); + throw new Error('Database "{0}" is readonly.'.format(this.name) + (e ? '\n' + e.toString() : '')); }; DP.scalar = function(type, field) { From ff45097acbc618596265e45aec096f8f0410dcb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 17 Sep 2018 16:42:19 +0200 Subject: [PATCH 082/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b910f58ae..f3dca4dfd 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-10", + "version": "3.0.1-11", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 77cd81c3ada358d4954aa62b4c1cec3628962fae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 19 Sep 2018 15:42:39 +0200 Subject: [PATCH 083/217] Fixed cleaning of `TABLE()`. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index bd0c44b3b..579da41a8 100755 --- a/index.js +++ b/index.js @@ -7001,7 +7001,7 @@ F.service = function(count) { keys = Object.keys(F.databasescleaner); keys.wait(function(item, next) { if (item[0] === '$') - TABLE(item).clean(next); + TABLE(item.substring(1)).clean(next); else NOSQL(item).clean(next); }); From 1174354272e2305891ecbfd9389dbcfa1c33784e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 20 Sep 2018 21:13:42 +0200 Subject: [PATCH 084/217] Fixed NoSQL cache for compiler. --- nosql.js | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/nosql.js b/nosql.js index 2355e51b7..c420e564c 100755 --- a/nosql.js +++ b/nosql.js @@ -2875,8 +2875,7 @@ DatabaseBuilder.prototype.random = function() { DatabaseBuilder.prototype.sort = function(name, desc) { var self = this; - if (!self.$iscache) - self.$options.sort = { name: name, asc: desc ? false : true }; + self.$options.sort = { name: name, asc: desc ? false : true }; return self; }; @@ -2903,7 +2902,6 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { if (key && cache) { self.$mappers = cache.mitems; self.$mappersexec = cache.mexec; - self.$options.sort = cache.sort; return cache.filter; } @@ -2916,7 +2914,6 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { if (cache) { self.$mappers = cache.mitems; self.$mappersexec = cache.mexec; - self.$options.sort = cache.sort; return cache.filter; } } @@ -2937,7 +2934,6 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { cache.filter = new Function('doc', '$F', 'index', code); cache.mexec = self.$mappersexec; cache.mitems = self.$mappers; - cache.sort = self.$options.sort; CACHE[key] = cache; return cache.filter; }; @@ -3023,17 +3019,15 @@ DatabaseBuilder.prototype.done = function() { DatabaseBuilder.prototype.fields = function() { var self = this; - if (!self.$iscache) { - var opt = self.$options; - for (var i = 0, length = arguments.length; i < length; i++) { - var name = arguments[i]; - if (name[0] === '-') { - !opt.fields2 && (opt.fields2 = {}); - opt.fields2[name.substring(1)] = 1; - } else { - !opt.fields && (opt.fields = []); - opt.fields.push(name); - } + var opt = self.$options; + for (var i = 0, length = arguments.length; i < length; i++) { + var name = arguments[i]; + if (name[0] === '-') { + !opt.fields2 && (opt.fields2 = {}); + opt.fields2[name.substring(1)] = 1; + } else { + !opt.fields && (opt.fields = []); + opt.fields.push(name); } } return self; From 58591a5c0c1e22427cfc021a1c5cd83ed240846c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 20 Sep 2018 21:14:26 +0200 Subject: [PATCH 085/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f3dca4dfd..6e2bdc377 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-11", + "version": "3.0.1-12", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 033cd55696dede1e44455112e5217f59dd749f31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 22 Sep 2018 21:33:33 +0200 Subject: [PATCH 086/217] Changed `Date.add()` and `Date.extend()` to UTC. --- utils.js | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/utils.js b/utils.js index bbe2be5c8..5723111c9 100755 --- a/utils.js +++ b/utils.js @@ -2682,44 +2682,44 @@ DP.add = function(type, value) { case 'sec': case 'second': case 'seconds': - dt.setSeconds(dt.getSeconds() + value); + dt.setUTCSeconds(dt.getUTCSeconds() + value); return dt; case 'm': case 'mm': case 'minute': case 'min': case 'minutes': - dt.setMinutes(dt.getMinutes() + value); + dt.setUTCMinutes(dt.getUTCMinutes() + value); return dt; case 'h': case 'hh': case 'hour': case 'hours': - dt.setHours(dt.getHours() + value); + dt.setUTCHours(dt.getUTCHours() + value); return dt; case 'd': case 'dd': case 'day': case 'days': - dt.setDate(dt.getDate() + value); + dt.setUTCDate(dt.getUTCDate() + value); return dt; case 'w': case 'ww': case 'week': case 'weeks': - dt.setDate(dt.getDate() + (value * 7)); + dt.setUTCDate(dt.getUTCDate() + (value * 7)); return dt; case 'M': case 'MM': case 'month': case 'months': - dt.setMonth(dt.getMonth() + value); + dt.setUTCMonth(dt.getUTCMonth() + value); return dt; case 'y': case 'yyyy': case 'year': case 'years': - dt.setFullYear(dt.getFullYear() + value); + dt.setUTCFullYear(dt.getUTCFullYear() + value); return dt; } return dt; @@ -2800,16 +2800,16 @@ DP.extend = function(date) { arr = m.split(':'); tmp = +arr[0]; - tmp >= 0 && dt.setHours(tmp); + tmp >= 0 && dt.setUTCHours(tmp); if (arr[1]) { tmp = +arr[1]; - tmp >= 0 && dt.setMinutes(tmp); + tmp >= 0 && dt.setUTCMinutes(tmp); } if (arr[2]) { tmp = +arr[2]; - tmp >= 0 && dt.setSeconds(tmp); + tmp >= 0 && dt.setUTCSeconds(tmp); } continue; @@ -2819,16 +2819,16 @@ DP.extend = function(date) { arr = m.split('-'); tmp = +arr[0]; - tmp && dt.setFullYear(tmp); + tmp && dt.setUTCFullYear(tmp); if (arr[1]) { tmp = +arr[1]; - tmp >= 0 && dt.setMonth(tmp - 1); + tmp >= 0 && dt.setUTCMonth(tmp - 1); } if (arr[2]) { tmp = +arr[2]; - tmp >= 0 && dt.setDate(tmp); + tmp >= 0 && dt.setUTCDate(tmp); } continue; @@ -2839,16 +2839,16 @@ DP.extend = function(date) { if (arr[2]) { tmp = +arr[2]; - !isNaN(tmp) && dt.setFullYear(tmp); + !isNaN(tmp) && dt.setUTCFullYear(tmp); } if (arr[1]) { tmp = +arr[1]; - !isNaN(tmp) && dt.setMonth(tmp - 1); + !isNaN(tmp) && dt.setUTCMonth(tmp - 1); } tmp = +arr[0]; - !isNaN(tmp) && dt.setDate(tmp); + !isNaN(tmp) && dt.setUTCDate(tmp); continue; } From 92c1e98404cfb62dad25e7685229d3ce72ffe579 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 24 Sep 2018 09:55:09 +0200 Subject: [PATCH 087/217] Fixed routing with operations. --- index.js | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/index.js b/index.js index 579da41a8..01bc6bcf8 100755 --- a/index.js +++ b/index.js @@ -868,6 +868,7 @@ function Framework() { path: 0, upload: 0, schema: 0, + operation: 0, blocked: 0, 'delete': 0, mobile: 0, @@ -14600,13 +14601,18 @@ function extend_request(PROTO) { PROTO.$total_validate = function(route, next, code) { - this.$total_schema = false; + var self = this; + self.$total_schema = false; - if (!this.$total_route.schema || this.method === 'DELETE') - return next(this, code); + if (!self.$total_route.schema || self.method === 'DELETE') + return next(self, code); - var self = this; - F.onSchema(this, this.$total_route.schema[0], this.$total_route.schema[1], function(err, body) { + if (!self.$total_route.schema[1]) { + F.stats.request.operation++; + return next(self, code); + } + + F.onSchema(self, self.$total_route.schema[0], self.$total_route.schema[1], function(err, body) { if (err) { self.$total_400(err); From 59a0a2f07190921d6707d1869e86cd729096c6e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 24 Sep 2018 15:04:14 +0200 Subject: [PATCH 088/217] Fixed parsing CSV. --- bin/totaljs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/totaljs b/bin/totaljs index f17802003..82360f69b 100755 --- a/bin/totaljs +++ b/bin/totaljs @@ -316,7 +316,7 @@ function parse_csv(content) { return; var key = arr[0].trim().replace(/(^\")|(\"$)/g, ''); - var value = arr[1].trim().replace(/(^\")|(\"$)/g, ''); + var value = arr[2].trim().replace(/(^\")|(\"$)/g, ''); if (!key || !value) return; From c5eb0f2104b76f394c457ad0cdbec457dd23b0cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 24 Sep 2018 15:04:38 +0200 Subject: [PATCH 089/217] Improved code. --- bin/totaljs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/totaljs b/bin/totaljs index 82360f69b..ac82242e7 100755 --- a/bin/totaljs +++ b/bin/totaljs @@ -315,8 +315,8 @@ function parse_csv(content) { if (arr.length <= 1) return; - var key = arr[0].trim().replace(/(^\")|(\"$)/g, ''); - var value = arr[2].trim().replace(/(^\")|(\"$)/g, ''); + var key = arr[0].trim().replace(/(^")|("$)/g, ''); + var value = arr[2].trim().replace(/(^")|("$)/g, ''); if (!key || !value) return; @@ -810,7 +810,7 @@ function git(dir, type) { } F.path.mkdir(dir); - exec('git clone https://github.com/totaljs/{0}.git {1}'.format(type, dir), function(err) { + exec('git clone https://github.com/totaljs/{0}.git {1}'.format(type, dir), function() { F.path.mkdir(path.join(dir, '/node_modules/')); F.rmdir(path.join(dir, '.git'), function() { F.unlink(path.join(dir, '.gitignore'), function() { From 3148b630c2ccc3a44dc25f397fea72275f5da41d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 25 Sep 2018 14:50:00 +0200 Subject: [PATCH 090/217] Improved binary. --- bin/totaljs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/totaljs b/bin/totaljs index ac82242e7..04d7be51d 100755 --- a/bin/totaljs +++ b/bin/totaljs @@ -307,10 +307,16 @@ function parse_csv(content) { var output = {}; var max = 0; - content.split('\n').forEach(function(line) { + content.split('\n').forEach(function(line, index) { + + if (!index) + return; + line = line.trim(); + if (!line) return; + var arr = line.split(';'); if (arr.length <= 1) return; From 9d837902a3cda8a2ed70015e143799804d8f6fc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 25 Sep 2018 14:50:14 +0200 Subject: [PATCH 091/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6e2bdc377..68f13a2f5 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-12", + "version": "3.0.1-13", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From eb1295c3a0f7bdc81c57121462ee0cc374993a48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 26 Sep 2018 19:43:01 +0200 Subject: [PATCH 092/217] Updated `stop` to `true` (by default) in `NEWOPERATION()`. --- builders.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builders.js b/builders.js index 02993f563..f012f0828 100755 --- a/builders.js +++ b/builders.js @@ -4610,7 +4610,7 @@ global.NEWOPERATION = function(name, fn, repeat, stop) { operations[name].$owner = F.$owner(); operations[name].$newversion = REGEXP_NEWOPERATION.test(fn.toString()); operations[name].$repeat = repeat; - operations[name].$stop = stop; + operations[name].$stop = stop !== false; return this; }; From 42744e6480c690600255b66f258e035df878f49c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 26 Sep 2018 19:52:54 +0200 Subject: [PATCH 093/217] Added `default-restbuilder-timeout`. --- changes.txt | 1 + index.js | 2 ++ utils.js | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 7421e3892..f737af71a 100755 --- a/changes.txt +++ b/changes.txt @@ -12,6 +12,7 @@ - added: NOSQL/TABLE modify supports `!field` as boolean toggle - added: NOSQL/TABLE modify supports a new type `$age: 'js_code'` with some JS code - added: NOSQL/TABLE update supports a new type `'js_code'` with some JS code +- added: a new config item `default-restbuilder-timeout : 10000` - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat` and `stop` argument diff --git a/index.js b/index.js index 01bc6bcf8..e65b9a415 100755 --- a/index.js +++ b/index.js @@ -709,6 +709,7 @@ function Framework() { // in milliseconds 'default-request-timeout': 3000, 'default-dependency-timeout': 1500, + 'default-restbuilder-timeout': 10000, // otherwise is used ImageMagick (Heroku supports ImageMagick) // gm = graphicsmagick or im = imagemagick @@ -8625,6 +8626,7 @@ F.$configure_configs = function(arr, rewrite) { case 'default-interval-websocket-ping': case 'default-interval-clear-dnscache': case 'default-dependency-timeout': + case 'default-restbuilder-timeout': case 'nosql-cleaner': obj[name] = U.parseInt(value); break; diff --git a/utils.js b/utils.js index 5723111c9..67204fb17 100755 --- a/utils.js +++ b/utils.js @@ -485,7 +485,7 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, if (callback === NOOP) callback = null; - var options = { length: 0, timeout: timeout || 10000, evt: new EventEmitter2(), encoding: typeof(encoding) !== 'string' ? ENCODING : encoding, callback: callback, post: false, redirect: 0 }; + var options = { length: 0, timeout: timeout || CONF['default-restbuilder-timeout'], evt: new EventEmitter2(), encoding: typeof(encoding) !== 'string' ? ENCODING : encoding, callback: callback, post: false, redirect: 0 }; var method; var type = 0; var isCookies = false; From b8d954f0125f6fb7a854f940b2b47451908d0c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 27 Sep 2018 23:28:03 +0200 Subject: [PATCH 094/217] Added URI validation in `REQUEST`. --- utils.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/utils.js b/utils.js index 67204fb17..4ad5fe530 100755 --- a/utils.js +++ b/utils.js @@ -625,6 +625,12 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, } var uri = Url.parse(url); + + if (!uri.hostname || !uri.host) { + callback && callback(new Error('URL doesn\'t contain a hostname'), '', 0); + return; + } + uri.method = method; uri.headers = headers; options.uri = uri; From aa4700a97be07b0b272f146a72b1b71b8f139f4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 28 Sep 2018 19:37:45 +0200 Subject: [PATCH 095/217] Added `logger`. --- builders.js | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++-- changes.txt | 1 + index.js | 77 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 165 insertions(+), 6 deletions(-) diff --git a/builders.js b/builders.js index f012f0828..837d7542c 100755 --- a/builders.js +++ b/builders.js @@ -1161,6 +1161,10 @@ SchemaBuilderEntity.prototype.execute = function(TYPE, model, options, callback, controller = model.$$controller; var builder = new ErrorBuilder(); + var $now; + + if (CONF.logger) + $now = Date.now(); self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); @@ -1168,10 +1172,12 @@ SchemaBuilderEntity.prototype.execute = function(TYPE, model, options, callback, if (!isGenerator(self, $type, self[TYPE])) { if (self[TYPE].$newversion) self[TYPE](new SchemaOptions(builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type), controller, $now); self.$process(arguments, model, $type, undefined, builder, res, callback, controller); }, controller)); else self[TYPE](builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type), controller, $now); self.$process(arguments, model, $type, undefined, builder, res, callback, controller); }, controller, skip !== true); return self; @@ -1189,6 +1195,9 @@ SchemaBuilderEntity.prototype.execute = function(TYPE, model, options, callback, }; var onCallback = function(res) { + + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); + if (callback.success) return; @@ -1236,6 +1245,7 @@ SchemaBuilderEntity.prototype.get = SchemaBuilderEntity.prototype.read = functio var self = this; var builder = new ErrorBuilder(); + var $now; self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); @@ -1243,16 +1253,21 @@ SchemaBuilderEntity.prototype.get = SchemaBuilderEntity.prototype.read = functio if (controller instanceof SchemaOptions) controller = controller.controller; + if (CONF.logger) + $now = Date.now(); + var output = self.default(); var $type = 'get'; if (!isGenerator(self, $type, self.onGet)) { if (self.onGet.$newversion) self.onGet(new SchemaOptions(builder, output, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, 'get'), controller, $now); self.$process(arguments, output, $type, undefined, builder, res, callback, controller); }, controller)); else self.onGet(builder, output, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, 'get'), controller, $now); self.$process(arguments, output, $type, undefined, builder, res, callback, controller); }, controller); return self; @@ -1270,6 +1285,9 @@ SchemaBuilderEntity.prototype.get = SchemaBuilderEntity.prototype.read = functio }; var onCallback = function(res) { + + CONF.logger && F.ilogger(self.getLoggerName($type, 'get'), controller, $now); + if (callback.success) return; @@ -1309,6 +1327,7 @@ SchemaBuilderEntity.prototype.remove = function(options, callback, controller) { var self = this; var builder = new ErrorBuilder(); var $type = 'remove'; + var $now; if (!self.onRemove) return callback(new Error('Operation "{0}/{1}" not found'.format(self.name, $type))); @@ -1316,16 +1335,21 @@ SchemaBuilderEntity.prototype.remove = function(options, callback, controller) { if (controller instanceof SchemaOptions) controller = controller.controller; + if (CONF.logger) + $now = Date.now(); + self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); if (!isGenerator(self, $type, self.onRemove)) { if (self.onRemove.$newversion) self.onRemove(new SchemaOptions(builder, undefined, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); self.$process(arguments, undefined, $type, undefined, builder, res, callback, controller); }, controller)); else self.onRemove(builder, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); self.$process(arguments, undefined, $type, undefined, builder, res, callback, controller); }, controller); return self; @@ -1344,6 +1368,8 @@ SchemaBuilderEntity.prototype.remove = function(options, callback, controller) { var onCallback = function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); + if (callback.success) return; @@ -1386,17 +1412,23 @@ SchemaBuilderEntity.prototype.query = function(options, callback, controller) { var self = this; var builder = new ErrorBuilder(); var $type = 'query'; + var $now; self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); + if (CONF.logger) + $now = Date.now(); + if (!isGenerator(self, $type, self.onQuery)) { if (self.onQuery.$newversion) self.onQuery(new SchemaOptions(builder, undefined, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type), controller, $now); self.$process(arguments, undefined, $type, undefined, builder, res, callback, controller); }, controller)); else self.onQuery(builder, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type), controller, $now); self.$process(arguments, undefined, $type, undefined, builder, res, callback, controller); }, controller); return self; @@ -1415,6 +1447,8 @@ SchemaBuilderEntity.prototype.query = function(options, callback, controller) { var onCallback = function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type), controller, $now); + if (callback.success) return; @@ -2159,6 +2193,10 @@ SchemaBuilderEntity.prototype.hook = function(name, model, options, callback, sk self.resourcePrefix && builder.setPrefix(self.resourcePrefix); var output = []; + var $now; + + if (CONF.logger) + $now = Date.now(); async_wait(hook, function(item, next) { if (item.fn.$newversion) @@ -2172,6 +2210,7 @@ SchemaBuilderEntity.prototype.hook = function(name, model, options, callback, sk next(); }, controller, skip !== true); }, function() { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); self.$process_hook(model, $type, name, builder, output, callback); }, 0); @@ -2187,19 +2226,25 @@ SchemaBuilderEntity.prototype.hook = function(name, model, options, callback, sk var builder = new ErrorBuilder(); var output = []; + var $now; self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); + if (CONF.logger) + $now = Date.now(); + async_wait(hook, function(item, next, index) { if (!isGenerator(self, 'hook.' + name + '.' + index, item.fn)) { if (item.fn.$newversion) { item.fn.call(self, new SchemaOptions(builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); output.push(res === undefined ? model : res); next(); }, controller)); } else { item.fn.call(self, builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); output.push(res === undefined ? model : res); next(); }, controller, skip !== true); @@ -2216,6 +2261,7 @@ SchemaBuilderEntity.prototype.hook = function(name, model, options, callback, sk builder.push(err); next(); }, new SchemaOptions(builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); output.push(res == undefined ? model : res); next(); }, controller)); @@ -2226,6 +2272,7 @@ SchemaBuilderEntity.prototype.hook = function(name, model, options, callback, sk builder.push(err); next(); }, builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); output.push(res == undefined ? model : res); next(); }, controller, skip !== true); @@ -2274,12 +2321,16 @@ SchemaBuilderEntity.prototype.$execute = function(type, name, model, options, ca var ref = self[type + 's']; var item = ref ? ref[name] : undefined; + var $now; if (!item) { callback(new ErrorBuilder().push('', type.capitalize() + ' "{0}" not found.'.format(name))); return self; } + if (CONF.logger) + $now = Date.now(); + if (controller instanceof SchemaOptions) controller = controller.controller; @@ -2292,10 +2343,12 @@ SchemaBuilderEntity.prototype.$execute = function(type, name, model, options, ca self.resourcePrefix && builder.setPrefix(self.resourcePrefix); if (item.$newversion) item.call(self, new SchemaOptions(builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName(type, name), controller, $now); self.$process(arguments, model, type, name, builder, res, callback, controller); }, controller)); else item.call(self, builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName(type, name), controller, $now); self.$process(arguments, model, type, name, builder, res, callback, controller); }, controller, skip !== true); return self; @@ -2319,10 +2372,12 @@ SchemaBuilderEntity.prototype.$execute = function(type, name, model, options, ca if (!isGenerator(self, type + '.' + name, item)) { if (item.$newversion) item.call(self, new SchemaOptions(builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName(type, name), controller, $now); self.$process(arguments, model, type, name, builder, res, callback, controller); }, controller)); else item.call(self, builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName(type, name), controller, $now); self.$process(arguments, model, type, name, builder, res, callback, controller); }, controller); return; @@ -2341,6 +2396,8 @@ SchemaBuilderEntity.prototype.$execute = function(type, name, model, options, ca var onCallback = function(res) { + CONF.logger && F.ilogger(self.getLoggerName(type, name), controller, $now); + if (callback.success) return; @@ -2365,6 +2422,10 @@ SchemaBuilderEntity.prototype.$execute = function(type, name, model, options, ca return self; }; +SchemaBuilderEntity.prototype.getLoggerName = function(type, name) { + return this.name + '.' + type + (name ? ('(\'' + name + '\')') : '()'); +}; + /** * Run a workflow * @param {String} name @@ -2417,6 +2478,7 @@ SchemaBuilderEntity.prototype.operation = function(name, model, options, callbac var builder = new ErrorBuilder(); var $type = 'operation'; + var $now; self.resourceName && builder.setResource(self.resourceName); self.resourcePrefix && builder.setPrefix(self.resourcePrefix); @@ -2427,13 +2489,18 @@ SchemaBuilderEntity.prototype.operation = function(name, model, options, callbac if (model && !controller && model.$$controller) controller = model.$$controller; + if (CONF.logger) + $now = Date.now(); + if (!isGenerator(self, 'operation.' + name, operation)) { if (operation.$newversion) { operation.call(self, new SchemaOptions(builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); self.$process(arguments, model, $type, name, builder, res, callback, controller); }, controller)); } else operation.call(self, builder, model, options, function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); self.$process(arguments, model, $type, name, builder, res, callback, controller); }, controller, skip !== true); return self; @@ -2452,6 +2519,8 @@ SchemaBuilderEntity.prototype.operation = function(name, model, options, callbac var onCallback = function(res) { + CONF.logger && F.ilogger(self.getLoggerName($type, name), controller, $now); + if (callback.success) return; @@ -4615,6 +4684,10 @@ global.NEWOPERATION = function(name, fn, repeat, stop) { return this; }; +function getLoggerNameOperation(name) { + return 'OPERATION(\'' + name + '\')'; +} + global.OPERATION = function(name, value, callback, param, controller) { if (typeof(value) === 'function') { @@ -4624,15 +4697,24 @@ global.OPERATION = function(name, value, callback, param, controller) { value = EMPTYOBJECT; } + if (param instanceof Controller) { + controller = param; + param = undefined; + } + var fn = operations[name]; var error = new ErrorBuilder(); + var $now; + + if (CONF.logger) + $now = Date.now(); if (fn) { if (fn.$newversion) { var self = new OperationOptions(error, value, param, controller); self.$repeat = fn.$repeat; self.callback = function(value) { - + CONF.logger && F.ilogger(getLoggerNameOperation(name), controller, $now); if (arguments.length > 1) { if (value instanceof Error || (value instanceof ErrorBuilder && value.hasError())) { self.error.push(value); @@ -4653,11 +4735,10 @@ global.OPERATION = function(name, value, callback, param, controller) { return self; }; - fn(self); - } else fn(error, value, function(value) { + CONF.logger && F.ilogger(getLoggerNameOperation(name), controller, $now); if (callback) { if (value instanceof Error) { error.push(value); @@ -4714,6 +4795,8 @@ global.RUN = function(name, value, callback, param, controller, result) { opt.callback = function(value) { + CONF.logger && F.ilogger(getLoggerNameOperation(opt.name), controller, opt.duration); + if (arguments.length > 1) { if (value instanceof Error || (value instanceof ErrorBuilder && value.hasError())) { opt.error.push(value); @@ -4766,6 +4849,10 @@ global.RUN = function(name, value, callback, param, controller, result) { opt.$current = fn; opt.$next = next; opt.meta.next = name[index]; + + if (CONF.logger) + opt.duration = Date.now(); + fn(opt); }, () => callback(error.items.length ? error : null, result ? opt.output : opt.response, opt)); diff --git a/changes.txt b/changes.txt index f737af71a..99f20ccf8 100755 --- a/changes.txt +++ b/changes.txt @@ -13,6 +13,7 @@ - added: NOSQL/TABLE modify supports a new type `$age: 'js_code'` with some JS code - added: NOSQL/TABLE update supports a new type `'js_code'` with some JS code - added: a new config item `default-restbuilder-timeout : 10000` +- added: a new config item `logger : false` which enables logging for Middleware, Schemas and Operations - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat` and `stop` argument diff --git a/index.js b/index.js index e65b9a415..4dfb12433 100755 --- a/index.js +++ b/index.js @@ -739,6 +739,7 @@ function Framework() { 'nosql-inmemory': null, // String Array 'nosql-cleaner': 1440, 'nosql-logger': true, + logger: false, // Used in F.service() // All values are in minutes @@ -12004,6 +12005,8 @@ Controller.prototype.callback = function(view) { var self = this; return function(err, data) { + CONF.logger && self.req.$logger && F.ilogger(null, self.req); + if (self.res && self.res.success) return; @@ -16425,6 +16428,10 @@ function existsSync(filename, file) { } } +function getLoggerMiddleware(name) { + return 'MIDDLEWARE("' + name + '")'; +} + function async_middleware(index, req, res, middleware, callback, options, controller) { if (res.success || res.headersSent || res.finished) { @@ -16444,10 +16451,13 @@ function async_middleware(index, req, res, middleware, callback, options, contro } var output; + var $now; + + if (CONF.logger) + $now = Date.now(); if (item.$newversion) { var opt = req.$total_middleware; - if (!index || !opt) { opt = req.$total_middleware = new MiddlewareOptions(); opt.req = req; @@ -16457,6 +16467,7 @@ function async_middleware(index, req, res, middleware, callback, options, contro opt.controller = controller; opt.callback2 = callback; opt.next = function(err) { + CONF.logger && F.ilogger(getLoggerMiddleware(name), req, $now); var mid = req.$total_middleware; if (err === false) { req.$total_route && req.$total_success(); @@ -16476,6 +16487,7 @@ function async_middleware(index, req, res, middleware, callback, options, contro } else { output = item.call(framework, req, res, function(err) { + CONF.logger && F.ilogger(getLoggerMiddleware(name), req, $now); if (err === false) { req.$total_route && req.$total_success(); callback = null; @@ -16630,10 +16642,11 @@ function controller_json_workflow(id) { self.id = id; var w = self.route.workflow; + CONF.logger && (self.req.$logger = []); + if (w instanceof Object) { if (!w.type) { - // IS IT AN OPERATION? if (!self.route.schema.length) { OPERATION(w.id, EMPTYOBJECT, w.view ? self.callback(w.view) : self.callback(), self); @@ -16680,6 +16693,9 @@ function controller_json_workflow_multiple(id) { var self = this; self.id = id; var w = self.route.workflow; + + CONF.logger && (self.req.$logger = []); + if (w instanceof Object) { if (!w.type) { @@ -16743,6 +16759,61 @@ function parseSchema(name) { return schema; } +function ilogger(body) { + F.path.verify('logs'); + U.queue('F.ilogger', 5, (next) => Fs.appendFile(U.combine(F.config['directory-logs'], 'logger.log'), body, next)); +} + +F.ilogger = function(name, req, ts) { + + if (req && req instanceof Controller) + req = req.req; + + var isc = CONF.logger === 'console'; + var divider = ''; + + for (var i = 0; i < (isc ? 60 : 220); i++) + divider += '-'; + + var msg; + + if (req && !name && req.$logger && req.$logger.length) { + + msg = req.method + ' ' + req.url; + + req.$logger.unshift(msg); + req.$logger.push(divider + '\n'); + + if (isc) + console.log(req.$logger.join('\n')); + else + ilogger(req.$logger.join('\n')); + + req.$logger = null; + return; + } + + if (!name) + return; + + var dt = new Date(); + + msg = dt.format('yyyy-MM-dd HH:mm:ss') + ' | ' + name.padRight(40, ' ') + ' | ' + (((dt.getTime() - ts) / 1000).format(3) + ' sec.').padRight(12) + ' | ' + (req ? (req.method + ' ' + req.url).max(70) : '').padRight(70); + + if (isc) { + if (req && req.$logger) + req.$logger.push(msg); + else + console.log(msg + '\n' + divider); + } else { + msg = msg + ' | ' + (req ? (req.ip || '') : '').padRight(20) + ' | ' + (req && req.headers ? (req.headers['user-agent'] || '') : ''); + if (req && req.$logger) + req.$logger.push(msg); + else + ilogger(msg + '\n' + divider + '\n'); + } +}; + // Because of controller prototypes // It's used in F.view() and F.viewCompile() const EMPTYCONTROLLER = new Controller('', null, null, ''); @@ -16753,4 +16824,4 @@ EMPTYCONTROLLER.req.uri = EMPTYOBJECT; EMPTYCONTROLLER.req.query = EMPTYOBJECT; EMPTYCONTROLLER.req.body = EMPTYOBJECT; EMPTYCONTROLLER.req.files = EMPTYARRAY; -global.EMPTYCONTROLLER = EMPTYCONTROLLER; +global.EMPTYCONTROLLER = EMPTYCONTROLLER; \ No newline at end of file From 70f8cc81403fdc410ef7defcad867af4f47e35f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 28 Sep 2018 19:43:04 +0200 Subject: [PATCH 096/217] Improved logging. --- index.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 4dfb12433..eb9ab2b9f 100755 --- a/index.js +++ b/index.js @@ -16772,7 +16772,7 @@ F.ilogger = function(name, req, ts) { var isc = CONF.logger === 'console'; var divider = ''; - for (var i = 0; i < (isc ? 60 : 220); i++) + for (var i = 0; i < (isc ? 64 : 220); i++) divider += '-'; var msg; @@ -16782,12 +16782,14 @@ F.ilogger = function(name, req, ts) { msg = req.method + ' ' + req.url; req.$logger.unshift(msg); - req.$logger.push(divider + '\n'); + req.$logger.push(divider); if (isc) console.log(req.$logger.join('\n')); - else + else { + req.$logger.push(''); ilogger(req.$logger.join('\n')); + } req.$logger = null; return; From 72b798bccf0e506f289ba750badc3ca2416ae6f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 29 Sep 2018 22:22:47 +0200 Subject: [PATCH 097/217] Fixed operation routing. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index eb9ab2b9f..7f6a8ac6b 100755 --- a/index.js +++ b/index.js @@ -16649,7 +16649,7 @@ function controller_json_workflow(id) { // IS IT AN OPERATION? if (!self.route.schema.length) { - OPERATION(w.id, EMPTYOBJECT, w.view ? self.callback(w.view) : self.callback(), self); + OPERATION(w.id, self.body, w.view ? self.callback(w.view) : self.callback(), self); return; } From 2832d3aec15574bb1574fc4ec9621c4d2e6f1fac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 30 Sep 2018 11:18:19 +0200 Subject: [PATCH 098/217] Fixed rendering groups of components. --- changes.txt | 1 + index.js | 19 +++++++++++++++---- internal.js | 50 ++++++++------------------------------------------ 3 files changed, 24 insertions(+), 46 deletions(-) diff --git a/changes.txt b/changes.txt index 99f20ccf8..40f1039c9 100755 --- a/changes.txt +++ b/changes.txt @@ -30,6 +30,7 @@ - fixed: extracting `bundles` (added `/flow/` and `/dashboard/`) - fixed: subdomain routing for `localhost` - fixed: service for database cleaner +- fixed: rendering group of components - improved: NoSQL reader diff --git a/index.js b/index.js index 7f6a8ac6b..2f7ea35d7 100755 --- a/index.js +++ b/index.js @@ -11312,10 +11312,19 @@ Controller.prototype.head = function() { var self = this; if (!arguments.length) { - // OBSOLETE: this is useless - // F.emit('controller-render-head', self); var author = self.repository[REPOSITORY_META_AUTHOR] || self.config.author; - return (author ? '' : '') + (self.repository[REPOSITORY_HEAD] || ''); + var plus = ''; + if (self.$hasComponents) { + if (self.$hasComponents instanceof Array) { + for (var i = 0; i < self.$hasComponents.length; i++) { + var group = F.components.groups[self.$hasComponents[i]]; + if (group) + plus += group.links; + } + } else + plus = F.components.links; + } + return (author ? '' : '') + (self.repository[REPOSITORY_HEAD] || '') + plus; } var header = (self.repository[REPOSITORY_HEAD] || ''); @@ -11532,7 +11541,9 @@ Controller.prototype.$import = function() { } if (filename === 'components' && F.components.has) { - builder += F.components.links; + // Generated in controller.head() + // self.$hasComponents = true; + // builder += F.components.links; continue; } diff --git a/internal.js b/internal.js index 462cdbb0c..af11d69ca 100755 --- a/internal.js +++ b/internal.js @@ -55,8 +55,6 @@ const REG_BLOCK_BEG = /@\{block.*?\}/i; const REG_BLOCK_END = /@\{end\}/i; const REG_SKIP_1 = /\('|"/; const REG_SKIP_2 = /,(\s)?\w+/; -const REG_HEAD = /<\/head>/i; -const REG_COMPONENTS = /@{(\s)?(component|components)(\s)?\(/i; const REG_COMPONENTS_GROUP = /('|")[a-z0-9]+('|")/i; const HTTPVERBS = { 'get': true, 'post': true, 'options': true, 'put': true, 'delete': true, 'patch': true, 'upload': true, 'head': true, 'trace': true, 'propfind': true }; const RENDERNOW = ['self.$import(', 'self.route', 'self.$js(', 'self.$css(', 'self.$favicon(', 'self.$script(', '$STRING(self.resource(', '$STRING(RESOURCE(', 'self.translate(', 'language', 'self.sitemap_url(', 'self.sitemap_name(', '$STRING(CONFIG(', '$STRING(config.', '$STRING(config[', '$STRING(CONF.', '$STRING(CONF[', '$STRING(config(']; @@ -1724,39 +1722,6 @@ function view_parse(content, minify, filename, controller) { var index = 0; var isCookie = false; - if (!controller.$hasComponents) - controller.$hasComponents = REG_COMPONENTS.test(content) && REG_HEAD.test(content); - - if (controller.$hasComponents) { - - index = content.indexOf('@{import('); - - var add = true; - while (index !== -1) { - var str = content.substring(index, content.indexOf(')', index)); - if (str.indexOf('components') !== -1) { - add = false; - break; - } else - index = content.indexOf('@{import(', index + str.length); - } - - if (add && controller.$hasComponents) { - if (controller.$hasComponents instanceof Array) { - content = content.replace(REG_HEAD, function(text) { - var str = ''; - for (var i = 0; i < controller.$hasComponents.length; i++) { - var group = F.components.groups[controller.$hasComponents[i]]; - if (group) - str += group.links; - } - return str + text; - }); - } else - content = content.replace(REG_HEAD, text => F.components.links + text); - } - } - function escaper(value) { var is = REG_TAGREMOVE.test(value); @@ -1938,24 +1903,24 @@ function view_parse(content, minify, filename, controller) { if (!a) { var isMeta = tmp.indexOf('\'meta\'') !== -1; var isHead = tmp.indexOf('\'head\'') !== -1; - var isComponent = tmp.indexOf('\'components\'') !== -1; - tmp = tmp.replace(/'(meta|head|components)',/g, '').replace(/(,,|,\)|\s{1,})/g, ''); - if (isMeta || isHead || isComponent) { + tmp = tmp.replace(/'(meta|head)',/g, '').replace(/(,,|,\)|\s{1,})/g, ''); + if (isMeta || isHead) { var tmpimp = ''; if (isMeta) tmpimp += (isMeta ? '\'meta\'' : ''); if (isHead) tmpimp += (tmpimp ? ',' : '') + (isHead ? '\'head\'' : ''); - if (isComponent) - tmpimp += (tmpimp ? ',' : '') + (isComponent ? '\'components\'' : ''); builder += '+self.$import(' + tmpimp + ')'; } } + if (tmp.indexOf('components') !== -1) + controller.$hasComponents = true; can = true; break; } } - } + } else if (!controller.$hasComponents && tmp.indexOf('components') !== -1) + controller.$hasComponents = true; if (can && !counter) { try { @@ -2000,7 +1965,8 @@ function view_parse(content, minify, filename, controller) { if (RELEASE) builder = builder.replace(/(\+\$EMPTY\+)/g, '+').replace(/(\$output=\$EMPTY\+)/g, '$output=').replace(/(\$output\+=\$EMPTY\+)/g, '$output+=').replace(/(\}\$output\+=\$EMPTY)/g, '}').replace(/(\{\$output\+=\$EMPTY;)/g, '{').replace(/(\+\$EMPTY\+)/g, '+').replace(/(>'\+'<)/g, '><').replace(/'\+'/g, ''); - var fn = '(function(self,repository,model,session,query,body,url,global,helpers,user,config,functions,index,output,files,mobile,settings){var get=query;var post=body;var G=F.global;var R=this.repository;var M=model;var theme=this.themeName;var language=this.language;var sitemap=this.repository.$sitemap;' + (isCookie ? 'var cookie=function(name){return self.req.cookie(name)};' : '') + (functions.length ? functions.join('') + ';' : '') + 'var controller=self;' + builder + ';return $output;})'; + var fn = ('(function(self,repository,model,session,query,body,url,global,helpers,user,config,functions,index,output,files,mobile,settings){self.$hasComponents=' + (controller.$hasComponents instanceof Array ? JSON.stringify(controller.$hasComponents).replace(/"/g, '\'') : controller.$hasComponents === true ? 'true' : 'null') + ';var get=query;var post=body;var G=F.global;var R=this.repository;var M=model;var theme=this.themeName;var language=this.language;var sitemap=this.repository.$sitemap;' + (isCookie ? 'var cookie=function(name){return self.req.cookie(name)};' : '') + (functions.length ? functions.join('') + ';' : '') + 'var controller=self;' + builder + ';return $output;})'); + try { fn = eval(fn); } catch (e) { From 3a98f227f4def53c0f6455b4165e596ae0a90460 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 30 Sep 2018 13:56:00 +0200 Subject: [PATCH 099/217] Added a new config option `default-request-maxkeys : 33` --- changes.txt | 1 + index.js | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/changes.txt b/changes.txt index 40f1039c9..e54f67f86 100755 --- a/changes.txt +++ b/changes.txt @@ -13,6 +13,7 @@ - added: NOSQL/TABLE modify supports a new type `$age: 'js_code'` with some JS code - added: NOSQL/TABLE update supports a new type `'js_code'` with some JS code - added: a new config item `default-restbuilder-timeout : 10000` +- added: a new config item `default-request-maxkeys : 33` for restricting query max. keys - added: a new config item `logger : false` which enables logging for Middleware, Schemas and Operations - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` diff --git a/index.js b/index.js index 2f7ea35d7..bb88e4db2 100755 --- a/index.js +++ b/index.js @@ -691,6 +691,7 @@ function Framework() { 'default-layout': 'layout', 'default-theme': '', 'default-proxy': '', + 'default-request-maxkeys': 33, // default maximum request size / length // default 10 kB @@ -8619,6 +8620,7 @@ F.$configure_configs = function(arr, rewrite) { case 'default-cors-maxage': case 'default-request-timeout': case 'default-request-maxlength': + case 'default-request-maxkeys': case 'default-websocket-maxlength': case 'default-interval-clear-cache': case 'default-interval-clear-resources': @@ -8778,6 +8780,8 @@ F.$configure_configs = function(arr, rewrite) { if (F.config['allow-performance']) http.globalAgent.maxSockets = 9999; + QUERYPARSEROPTIONS.maxKeys = F.config['default-request-maxkeys'] || 33; + var xpowered = F.config['default-xpoweredby']; Object.keys(HEADERS).forEach(function(key) { From 9bc76f35502b8783f268f7e5006f5670f88190a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 30 Sep 2018 13:56:29 +0200 Subject: [PATCH 100/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 68f13a2f5..ff6b89ea7 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-13", + "version": "3.0.1-14", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 26b4f5c493be339db5f917f86ba612479f631f80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 30 Sep 2018 22:59:08 +0200 Subject: [PATCH 101/217] Added a multiple method declaration. --- changes.txt | 1 + index.js | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index e54f67f86..975140b0b 100755 --- a/changes.txt +++ b/changes.txt @@ -19,6 +19,7 @@ - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat` and `stop` argument - updated: routing, now it supports operations in the form `ROUTE('.. * --> @save_operation @load_operation (response)')` +- updated: `ROUTE()` supports multiple HTTP method declaration `ROUTE('GET,POST,PUT /something/', action)` - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function diff --git a/index.js b/index.js index bb88e4db2..865590644 100755 --- a/index.js +++ b/index.js @@ -1854,12 +1854,17 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu url = url.substring(index + 1).trim(); } - url = url.replace(/(^|\s?)\*([a-z0-9]|\s).*?$/i, function(text) { !flags && (flags = []); flags.push(text.trim()); return ''; }); + + if (method.indexOf(',') !== -1) { + !flags && (flags = []); + method.split(',').forEach(m => flags.push(m.trim())); + method = ''; + } } if (url[0] === '#') { From 738b04d9cb1a9672e157724329de1980f58ae8c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 1 Oct 2018 08:37:01 +0200 Subject: [PATCH 102/217] Fixed sending data. --- changes.txt | 1 + utils.js | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/changes.txt b/changes.txt index 975140b0b..8b8e5a01d 100755 --- a/changes.txt +++ b/changes.txt @@ -33,6 +33,7 @@ - fixed: subdomain routing for `localhost` - fixed: service for database cleaner - fixed: rendering group of components +- fixed: RESTBuilder - JSON request without param sends an empty object - improved: NoSQL reader diff --git a/utils.js b/utils.js index 4ad5fe530..0d7ec33c2 100755 --- a/utils.js +++ b/utils.js @@ -606,6 +606,9 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, data.length && url.indexOf('?') === -1 && (url += '?' + data); data = ''; } + + if (type === 1 && (data === EMPTYOBJECT || !data)) + data = BUFEMPTYJSON; } if (data && type !== 4) { @@ -6187,5 +6190,7 @@ exports.reader = function() { return new Reader(); }; +const BUFEMPTYJSON = exports.createBuffer('{}'); + global.WAIT = exports.wait; !global.F && require('./index'); \ No newline at end of file From b66beab6fd1b12254f63190646ea02baaa2f36a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 1 Oct 2018 08:38:37 +0200 Subject: [PATCH 103/217] Fixed data serialization. --- utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.js b/utils.js index 0d7ec33c2..d392150bf 100755 --- a/utils.js +++ b/utils.js @@ -607,7 +607,7 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, data = ''; } - if (type === 1 && (data === EMPTYOBJECT || !data)) + if (type === 1 && (data === EMPTYOBJECT || data === '' || data === undefined)) data = BUFEMPTYJSON; } From 182798a62bf9be95be3c204f3287ea28ae6edc90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 1 Oct 2018 08:39:44 +0200 Subject: [PATCH 104/217] Improved condition. --- utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.js b/utils.js index d392150bf..3b572616c 100755 --- a/utils.js +++ b/utils.js @@ -607,7 +607,7 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, data = ''; } - if (type === 1 && (data === EMPTYOBJECT || data === '' || data === undefined)) + if (type === 1 && !data && (data === EMPTYOBJECT || data === '' || data === undefined)) data = BUFEMPTYJSON; } From 959344db068b08fcdfe7a0df57859ede19b8fb66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 1 Oct 2018 10:52:58 +0200 Subject: [PATCH 105/217] Fixed `$MAKE()` with `callback`. --- changes.txt | 1 + index.js | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/changes.txt b/changes.txt index 8b8e5a01d..13fb451b4 100755 --- a/changes.txt +++ b/changes.txt @@ -34,6 +34,7 @@ - fixed: service for database cleaner - fixed: rendering group of components - fixed: RESTBuilder - JSON request without param sends an empty object +- fixed: `$MAKE()` with `callback` - improved: NoSQL reader diff --git a/index.js b/index.js index 865590644..5b4b4d421 100755 --- a/index.js +++ b/index.js @@ -358,6 +358,12 @@ global.$MAKE = function(schema, model, filter, callback, novalidate, argument) { var o = framework_builders.getschema(schema[0], schema[1]); var w = null; + if (typeof(filter) === 'function') { + var tmp = callback; + callback = filter; + filter = tmp; + } + if (filter instanceof Array) { w = {}; for (var i = 0; i < filter.length; i++) From 145d873aa5cff661d63348aaa04fcfbcf5333c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 2 Oct 2018 19:23:35 +0200 Subject: [PATCH 106/217] Fixed `String.slug()`. --- changes.txt | 1 + utils.js | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/changes.txt b/changes.txt index 13fb451b4..ed5b82cca 100755 --- a/changes.txt +++ b/changes.txt @@ -35,6 +35,7 @@ - fixed: rendering group of components - fixed: RESTBuilder - JSON request without param sends an empty object - fixed: `$MAKE()` with `callback` +- fixed: `String.slug()` for UTF - Chinese/Japan/Arabic/etc. chars - improved: NoSQL reader diff --git a/utils.js b/utils.js index 3b572616c..22671ee81 100755 --- a/utils.js +++ b/utils.js @@ -275,7 +275,6 @@ exports.wait = function(fnValid, fnCallback, timeout, interval) { clearInterval(id_interval); clearTimeout(id_timeout); fnCallback && fnCallback(null, true); - return; } }, interval || 500); @@ -4048,6 +4047,11 @@ SP.slug = SP.toSlug = SP.toLinker = SP.linker = function(max) { var c = self[i]; var code = self.charCodeAt(i); + if (code > 540){ + builder = ''; + break; + } + if (builder.length >= max) break; @@ -4060,8 +4064,17 @@ SP.slug = SP.toSlug = SP.toLinker = SP.linker = function(max) { if ((code > 47 && code < 58) || (code > 94 && code < 123)) builder += c; } - var l = builder.length - 1; - return builder[l] === '-' ? builder.substring(0, l) : builder; + + if (builder.length > 1) { + length = builder.length - 1; + return builder[length] === '-' ? builder.substring(0, length) : builder; + } else if (!length) + return ''; + + length = self.length; + self = self.replace(/\s/g, ''); + builder = self.crc32(true).toString(36) + ''; + return self[0].charCodeAt(0).toString(32) + builder + self[self.length - 1].charCodeAt(0).toString(32) + length; }; SP.pluralize = function(zero, one, few, other) { From 191c5072ba53745748c753c976d02ce75e214804 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 2 Oct 2018 19:39:08 +0200 Subject: [PATCH 107/217] Fixed component rendering. --- index.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/index.js b/index.js index 5b4b4d421..497d9ba6d 100755 --- a/index.js +++ b/index.js @@ -12825,10 +12825,8 @@ Controller.prototype.$viewrender = function(filename, generator, model, headers, obj.options = obj.settings = item.settings; obj.next = obj.callback = function(model) { - if (arguments.length > 1) model = arguments[1]; - item.value = self.component(item.name, item.settings, model); value = value.replace(item.replace, item.value); if (isLayout && self.precache) @@ -12892,7 +12890,7 @@ Controller.prototype.$viewrender = function(filename, generator, model, headers, done.callback && done.callback(null, value); } - }, 3); + }); return cachekey ? value : (partial ? (fn => done.callback = fn) : self); } From ffc852cbb955285de79bc49f2b460aa084449ac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 2 Oct 2018 19:39:44 +0200 Subject: [PATCH 108/217] Added new change. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index ed5b82cca..2bd9f4eaa 100755 --- a/changes.txt +++ b/changes.txt @@ -36,6 +36,7 @@ - fixed: RESTBuilder - JSON request without param sends an empty object - fixed: `$MAKE()` with `callback` - fixed: `String.slug()` for UTF - Chinese/Japan/Arabic/etc. chars +- fixed: async rendering of `components` - improved: NoSQL reader From d20978d4eb85d0a7ef84739ecd3f51cca834eacd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 2 Oct 2018 19:41:01 +0200 Subject: [PATCH 109/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ff6b89ea7..4ae2bf8ea 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-14", + "version": "3.0.1-15", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From c96aa515bbe2d5b2e3b37dcba2be4363698b6d57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 6 Oct 2018 00:17:14 +0200 Subject: [PATCH 110/217] Improved `UID()`. --- changes.txt | 1 + index.js | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 2bd9f4eaa..acb2538a3 100755 --- a/changes.txt +++ b/changes.txt @@ -39,6 +39,7 @@ - fixed: async rendering of `components` - improved: NoSQL reader +- improved: `UID()` -> now it changes a random hash each minute ======= 3.0.0 diff --git a/index.js b/index.js index 497d9ba6d..1387bb017 100755 --- a/index.js +++ b/index.js @@ -605,9 +605,14 @@ var directory = U.$normalize(require.main ? Path.dirname(require.main.filename) // F.service() changes the values below: var DATE_EXPIRES = new Date().add('y', 1).toUTCString(); +const _randomstring = 'abcdefghijklmnoprstuwxy'.split(''); +function random3string() { + return _randomstring[(Math.random() * _randomstring.length) >> 0] + _randomstring[(Math.random() * _randomstring.length) >> 0] + _randomstring[(Math.random() * _randomstring.length) >> 0]; +} + const WEBSOCKET_COMPRESS = U.createBuffer([0x00, 0x00, 0xFF, 0xFF]); const WEBSOCKET_COMPRESS_OPTIONS = { windowBits: Zlib.Z_DEFAULT_WINDOWBITS }; -const UIDGENERATOR = { date: new Date().format('yyMMddHHmm'), instance: 'abcdefghijklmnoprstuwxy'.split('').random().join('').substring(0, 3), index: 1, types: {} }; +const UIDGENERATOR = { date: new Date().format('yyMMddHHmm'), instance: random3string(), index: 1, types: {} }; const EMPTYBUFFER = U.createBufferSize(0); global.EMPTYBUFFER = EMPTYBUFFER; @@ -6880,6 +6885,7 @@ F.service = function(count) { UIDGENERATOR.date = NOW.format('yyMMddHHmm'); UIDGENERATOR.index = 1; + UIDGENERATOR.instance = random3string(); var keys; From a4063ca809a30157702f06fe9d5f2ba6db28f09e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 7 Oct 2018 15:52:40 +0200 Subject: [PATCH 111/217] Fixed `cache` in RESTBuilder. --- builders.js | 4 ++-- changes.txt | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/builders.js b/builders.js index 837d7542c..298466d97 100755 --- a/builders.js +++ b/builders.js @@ -4610,13 +4610,13 @@ RESTBuilder.prototype.exec = function(callback) { return callback(err, EMPTYOBJECT, output); self.$schema.make(self.maketransform(output.value, output), function(err, model) { - !err && key && F.cache.add(key, output, self.$cache_expire); + !err && key && output.status === 200 && F.cache.add(key, output, self.$cache_expire); callback(err, err ? EMPTYOBJECT : model, output); output.cache = true; }); } else { - !err && key && F.cache.add(key, output, self.$cache_expire); + !err && key && output.status === 200 && F.cache.add(key, output, self.$cache_expire); callback(err, self.maketransform(output.value, output), output); output.cache = true; } diff --git a/changes.txt b/changes.txt index acb2538a3..567c89324 100755 --- a/changes.txt +++ b/changes.txt @@ -37,6 +37,7 @@ - fixed: `$MAKE()` with `callback` - fixed: `String.slug()` for UTF - Chinese/Japan/Arabic/etc. chars - fixed: async rendering of `components` +- fixed: RESTBuilder cache works only if the response status is `200` - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute From f599ef5d3767348d5f2fd5448b6b74fb23c63e36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 7 Oct 2018 17:38:48 +0200 Subject: [PATCH 112/217] Clean code. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 1387bb017..face150d2 100755 --- a/index.js +++ b/index.js @@ -14918,7 +14918,7 @@ function extend_response(PROTO) { options = {}; options.path = options.path || '/'; - expires && builder.push('Expires=' + expires.toUTCString()); + expires && builder.push('Expires=' + expires.toUTCString()); options.domain && builder.push('Domain=' + options.domain); options.path && builder.push('Path=' + options.path); options.secure && builder.push('Secure'); From 256a620f1d0c108d870b28a599d78e8c0621f0f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 7 Oct 2018 18:37:50 +0200 Subject: [PATCH 113/217] Added a new global alias `FUNC`. --- changes.txt | 1 + index.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 567c89324..31bc59294 100755 --- a/changes.txt +++ b/changes.txt @@ -8,6 +8,7 @@ - added: support for `SameSite` cookie attribute - added: `RUN()` for executing multiple Total.js operations - added: a new global alias `CONF` (it's a reference to config) for `F.config` +- added: a new global alias `FUNC` (it's a reference to functions) for `F.functions` - added: `DatabaseBuilder.arg(key, value)` for adding an dynamic argument - added: NOSQL/TABLE modify supports `!field` as boolean toggle - added: NOSQL/TABLE modify supports a new type `$age: 'js_code'` with some JS code diff --git a/index.js b/index.js index face150d2..9a089ee5f 100755 --- a/index.js +++ b/index.js @@ -767,7 +767,7 @@ function Framework() { self.$bundling = true; self.resources = {}; self.connections = {}; - self.functions = {}; + global.FUNC = self.functions = {}; self.themes = {}; self.versions = null; self.workflows = {}; From 446dbb5ca0cdf3f45fbad88639c93a3e0a769fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 7 Oct 2018 22:51:56 +0200 Subject: [PATCH 114/217] Fixed `versions` with `auto` (added cleaning cache) --- index.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/index.js b/index.js index 9a089ee5f..fd833c7b1 100755 --- a/index.js +++ b/index.js @@ -8486,6 +8486,10 @@ F.$configure_versions = function(arr, clean) { var index = key.lastIndexOf('.'); filename = key.substring(0, index) + '-' + hash + key.substring(index); F.versions[key] = filename; + var keys = Object.keys(F.temporary.other); + for (var i = 0; i < keys[i]; i++) + if (keys[i].substring(0, 4) === 'view') + delete F.temporary.other[keys[i]]; } }); }); From c3ab193724b29559386dc193fcc4b3c6a945f04c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 8 Oct 2018 00:01:02 +0200 Subject: [PATCH 115/217] Fixed compressing CSS. --- changes.txt | 1 + internal.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 31bc59294..457bc7702 100755 --- a/changes.txt +++ b/changes.txt @@ -39,6 +39,7 @@ - fixed: `String.slug()` for UTF - Chinese/Japan/Arabic/etc. chars - fixed: async rendering of `components` - fixed: RESTBuilder cache works only if the response status is `200` +- fixed: compressing CSS with `\t` tabs - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute diff --git a/internal.js b/internal.js index af11d69ca..e9bfb093a 100755 --- a/internal.js +++ b/internal.js @@ -63,7 +63,7 @@ const REG_NOCOMPRESS = /@\{nocompress\s\w+}/gi; const REG_TAGREMOVE = /[^>](\r)\n\s{1,}$/; const REG_HELPERS = /helpers\.[a-z0-9A-Z_$]+\(.*?\)+/g; const REG_SITEMAP = /\s+(sitemap_navigation\(|sitemap\()+/g; -const REG_CSS_0 = /\s{2,}/g; +const REG_CSS_0 = /\s{2,}|\t/g; const REG_CSS_1 = /\n/g; const REG_CSS_2 = /\s?\{\s{1,}/g; const REG_CSS_3 = /\s?\}\s{1,}/g; From 8988b6ec7145b602f05a165314e3da84123130ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 8 Oct 2018 00:10:46 +0200 Subject: [PATCH 116/217] Fixed cache for `versions` and `auto` value. --- index.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/index.js b/index.js index fd833c7b1..993932f1a 100755 --- a/index.js +++ b/index.js @@ -2557,10 +2557,12 @@ global.MERGE = F.merge = function(url) { if (url[0] !== '/') url = '/' + url; - var filename = F.path.temp((F.id ? 'i-' + F.id + '_' : '') + 'merged_' + createTemporaryKey(url)); + var key = createTemporaryKey(url); + var filename = F.path.temp((F.id ? 'i-' + F.id + '_' : '') + 'merged_' + key); F.routes.merge[url] = { filename: filename.replace(/\.(js|css)$/g, ext => '.min' + ext), files: arr }; Fs.unlink(F.routes.merge[url].filename, NOOP); F.owners.push({ type: 'merge', owner: _owner, id: url }); + delete F.temporary.notfound[key]; return F; }; @@ -8486,10 +8488,9 @@ F.$configure_versions = function(arr, clean) { var index = key.lastIndexOf('.'); filename = key.substring(0, index) + '-' + hash + key.substring(index); F.versions[key] = filename; - var keys = Object.keys(F.temporary.other); - for (var i = 0; i < keys[i]; i++) - if (keys[i].substring(0, 4) === 'view') - delete F.temporary.other[keys[i]]; + F.temporary.views = {}; + F.temporary.other = {}; + global.$VIEWCACHE = []; } }); }); From b8280bb45c94d0fa68f6ef7665feef6ee5f98c6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 8 Oct 2018 13:35:58 +0200 Subject: [PATCH 117/217] Fixed `controller.autoclear()`. --- changes.txt | 1 + index.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 457bc7702..979e8d213 100755 --- a/changes.txt +++ b/changes.txt @@ -40,6 +40,7 @@ - fixed: async rendering of `components` - fixed: RESTBuilder cache works only if the response status is `200` - fixed: compressing CSS with `\t` tabs +- fixed: `controller.autoclear()` - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute diff --git a/index.js b/index.js index 993932f1a..4f2f5c040 100755 --- a/index.js +++ b/index.js @@ -12089,7 +12089,7 @@ Controller.prototype.noClear = function(enable) { }; Controller.prototype.autoclear = function(enable) { - this.req._manual = enable === true; + this.req._manual = enable === false; return this; }; From 3596979f6e4d1644cc0b5909ecb4549aae59da07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 8 Oct 2018 18:21:22 +0200 Subject: [PATCH 118/217] Updated test. --- test/test-css.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-css.js b/test/test-css.js index eff390911..c3e534726 100755 --- a/test/test-css.js +++ b/test/test-css.js @@ -13,7 +13,7 @@ buffer.push('div{background:linear-gradient(90deg, #000000, #FFFFFF);}'); var css = buffer.join('\n'); assert.ok(internal.compile_css(css) === 'b{border-radius:1px}a{border-radius:1px 2px 3px 4px}a{text-overflow:ellipsis}span{opacity:0;filter:alpha(opacity=0)}@keyframes test{border-radius:5px}@-webkit-keyframes test{border-radius:5px}@-moz-keyframes test{border-radius:5px}@-o-keyframes test{border-radius:5px}div{background:-webkit-linear-gradient(90deg,#000000,#FFFFFF);background:-moz-linear-gradient(90deg,#000000,#FFFFFF);background:-ms-linear-gradient(90deg,#000000,#FFFFFF);background:linear-gradient(90deg,#000000,#FFFFFF)}', 'automated CSS vendor prefixes'); -css = '.input{ }, .input:disabled, .input:hover { background-color: red; } .required{content:"This, field is required"}'; +css = '.input{ }, .input:disabled, .input:hover { background-color: red; } .required{content:"This, field is required"}'; assert.ok(internal.compile_css(css) === '.input{},.input:disabled,.input:hover{background-color:red}.required{content:"This, field is required"}', 'Problem with content.'); buffer = []; From 3eea20693bcf98b87845af75bc8f0313a6f6b49a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 8 Oct 2018 18:32:38 +0200 Subject: [PATCH 119/217] Updated `NEWOPERATION()`. --- builders.js | 11 +++++++++-- changes.txt | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/builders.js b/builders.js index 298466d97..dbd46b4b7 100755 --- a/builders.js +++ b/builders.js @@ -4664,10 +4664,11 @@ function $decodeURIComponent(value) { } } -global.NEWOPERATION = function(name, fn, repeat, stop) { +global.NEWOPERATION = function(name, fn, repeat, stop, binderror) { // @repeat {Number} How many times will be the operation repeated after error? // @stop {Boolean} Stop when the error is thrown + // @binderror {Boolean} Binds error when chaining of operations // Remove operation if (fn == null) { @@ -4680,6 +4681,7 @@ global.NEWOPERATION = function(name, fn, repeat, stop) { operations[name].$newversion = REGEXP_NEWOPERATION.test(fn.toString()); operations[name].$repeat = repeat; operations[name].$stop = stop !== false; + operations[name].$binderror = binderror === true; return this; }; @@ -4814,7 +4816,12 @@ global.RUN = function(name, value, callback, param, controller, result) { opt.repeated++; setImmediate(opt => opt.$current(opt), opt); } else { - opt.error.items.length && error.push(opt.error); + if (opt.error.items.length) { + error.push(opt.error); + if (opt.$current.$binderror) + value = opt.error.output(false); + } + if (opt.error.items.length && opt.$current.$stop) { name = null; opt.next = null; diff --git a/changes.txt b/changes.txt index 979e8d213..7970aea75 100755 --- a/changes.txt +++ b/changes.txt @@ -18,7 +18,7 @@ - added: a new config item `logger : false` which enables logging for Middleware, Schemas and Operations - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` -- updated: `NEWOPERATION()` supports `repeat` and `stop` argument +- updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) - updated: routing, now it supports operations in the form `ROUTE('.. * --> @save_operation @load_operation (response)')` - updated: `ROUTE()` supports multiple HTTP method declaration `ROUTE('GET,POST,PUT /something/', action)` From 66071d4acf8a79aff5c2448e66171663ac3f35ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 9 Oct 2018 09:06:01 +0200 Subject: [PATCH 120/217] Fixed JavaScript minificator. --- changes.txt | 1 + internal.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 7970aea75..d669bba59 100755 --- a/changes.txt +++ b/changes.txt @@ -23,6 +23,7 @@ - updated: `ROUTE()` supports multiple HTTP method declaration `ROUTE('GET,POST,PUT /something/', action)` - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` +- fixed: a critical bug with JavaScript minificator - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function - fixed: `F.wait()` in the test mode - fixed: `LOCALIZE()` for nested directories diff --git a/internal.js b/internal.js index e9bfb093a..7d25cd636 100755 --- a/internal.js +++ b/internal.js @@ -1202,7 +1202,7 @@ function minify_javascript(data) { } } - if (c === '+' || c === '-' && next === ' ') { + if ((c === '+' || c === '-') && next === ' ') { if (data[index + 1] === c) { index += 2; output.push(c); From 2fed4603fc5e0a3f179fa6df204df32354b21ba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 9 Oct 2018 09:29:41 +0200 Subject: [PATCH 121/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4ae2bf8ea..92287866c 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-15", + "version": "3.0.1-16", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 95114107d864a86aef2e3f9b6f511a7da7f6db32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 11 Oct 2018 13:22:31 +0200 Subject: [PATCH 122/217] Added `$.cancel()` for Schemas/Operations. --- builders.js | 20 ++++++++++++++++++++ changes.txt | 1 + 2 files changed, 21 insertions(+) diff --git a/builders.js b/builders.js index dbd46b4b7..57d880cfa 100755 --- a/builders.js +++ b/builders.js @@ -91,6 +91,16 @@ SchemaOptions.prototype = { } }; +SchemaOptions.prototype.cancel = function() { + var self = this; + self.callback = self.next = null; + self.error = null; + self.controller = null; + self.model = null; + self.options = null; + return self; +}; + SchemaOptions.prototype.clean = function() { return this.model.$clean(); }; @@ -4918,6 +4928,16 @@ OperationOptions.prototype = { } }; +OperationOptions.prototype.cancel = function() { + var self = this; + self.callback = null; + self.error = null; + self.controller = null; + self.options = null; + self.model = self.value = null; + return self; +}; + OperationOptions.prototype.DB = function() { return F.database(this.error); }; diff --git a/changes.txt b/changes.txt index d669bba59..5eba13522 100755 --- a/changes.txt +++ b/changes.txt @@ -16,6 +16,7 @@ - added: a new config item `default-restbuilder-timeout : 10000` - added: a new config item `default-request-maxkeys : 33` for restricting query max. keys - added: a new config item `logger : false` which enables logging for Middleware, Schemas and Operations +- added: `SchemaOptions` and `OperationOptions` supports `$.cancel()` method - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) From eb0af55c6e2dfbaa1a9a22e231d9459d3ea875a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 12 Oct 2018 11:11:26 +0200 Subject: [PATCH 123/217] Improved `CORS` + added `default-cors` to `config`. --- changes.txt | 2 + index.js | 72 +++++++++++++++++++++++++++------- test/test-framework-debug.js | 2 +- test/test-framework-release.js | 2 +- 4 files changed, 61 insertions(+), 17 deletions(-) diff --git a/changes.txt b/changes.txt index 5eba13522..5c4ccdb18 100755 --- a/changes.txt +++ b/changes.txt @@ -14,6 +14,7 @@ - added: NOSQL/TABLE modify supports a new type `$age: 'js_code'` with some JS code - added: NOSQL/TABLE update supports a new type `'js_code'` with some JS code - added: a new config item `default-restbuilder-timeout : 10000` +- added: a new config item `default-cors : https://www.totaljs.com, https://www.componentator.com` which allows originators for `CORS()` method - added: a new config item `default-request-maxkeys : 33` for restricting query max. keys - added: a new config item `logger : false` which enables logging for Middleware, Schemas and Operations - added: `SchemaOptions` and `OperationOptions` supports `$.cancel()` method @@ -46,6 +47,7 @@ - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute +- improved: CORS ======= 3.0.0 diff --git a/index.js b/index.js index 4f2f5c040..088b4b220 100755 --- a/index.js +++ b/index.js @@ -715,6 +715,9 @@ function Framework() { 'default-response-maxage': '11111111', 'default-errorbuilder-status': 200, + // Default originators + 'default-cors': null, + // Seconds (2 minutes) 'default-cors-maxage': 120, @@ -1660,7 +1663,7 @@ global.CORS = F.cors = function(url, flags, credentials) { } var route = {}; - var origins = []; + var origin = []; var methods = []; var headers = []; var age; @@ -1683,8 +1686,14 @@ global.CORS = F.cors = function(url, flags, credentials) { continue; } + if (flag.substring(0, 2) === '//') { + origin.push('http:' + flag); + origin.push('https:' + flag); + continue; + } + if (flag.startsWith('http://') || flag.startsWith('https://')) { - origins.push(flag); + origin.push(flag); continue; } @@ -1713,6 +1722,9 @@ global.CORS = F.cors = function(url, flags, credentials) { if (!methods.length) methods = 'POST,PUT,GET,DELETE,PATCH,GET,HEAD'.split(','); + if (!origin.length && CONF['default-cors']) + origin = CONF['default-cors']; + route.isWILDCARD = url.lastIndexOf('*') !== -1; var index = url.indexOf('{'); @@ -1731,7 +1743,7 @@ global.CORS = F.cors = function(url, flags, credentials) { route.hash = url.hash(); route.owner = _owner; route.url = framework_internal.routeSplitCreate(url); - route.origins = origins.length ? origins : null; + route.origin = origin.length ? origin : null; route.methods = methods.length ? methods : null; route.headers = headers.length ? headers : null; route.credentials = credentials; @@ -1744,10 +1756,10 @@ global.CORS = F.cors = function(url, flags, credentials) { if (e) { // Extends existing - if (route.origins && e.origins) - corsextend(route.origins, e.origins); - else if (e.origins && !route.origins) - e.origins = null; + if (route.origin && e.origin) + corsextend(route.origin, e.origin); + else if (e.origin && !route.origin) + e.origin = null; if (route.methods && e.methods) corsextend(route.methods, e.methods); @@ -2987,7 +2999,7 @@ global.WEBSOCKET = F.websocket = function(url, funcInitialize, flags, length) { flag = flag.toString().toLowerCase(); - // Origins + // Origin if (flag.startsWith('http://') || flag.startsWith('https://')) { !allow && (allow = []); allow.push(flag); @@ -7258,7 +7270,7 @@ F.$requestcontinue = function(req, res, headers) { req.flags = flags; F.$events['request-begin'] && F.emit('request-begin', req, res); - var isCORS = (F._length_cors || F.routes.corsall) && req.headers['origin'] != null; + var isCORS = (F._length_cors || F.routes.corsall) && req.headers.origin != null; switch (first) { case 'G': @@ -7337,7 +7349,9 @@ F.$cors = function(req, res, fn, arg) { var isAllowed = F.routes.corsall; var cors, origin; var headers = req.headers; + if (!isAllowed) { + for (var i = 0; i < F._length_cors; i++) { cors = F.routes.cors[i]; if (framework_internal.routeCompare(req.path, cors.url, false, cors.isWILDCARD)) { @@ -7383,11 +7397,11 @@ F.$cors = function(req, res, fn, arg) { } } - origin = headers['origin'].toLowerCase(); - if (!stop && cors.origins) { + if (!stop && cors.origin) { + origin = headers.origin.toLowerCase(); isAllowed = false; - for (var i = 0, length = cors.origins.length; i < length; i++) { - if (cors.origins[i].indexOf(origin) !== -1) { + for (var i = 0, length = cors.origin.length; i < length; i++) { + if (cors.origin[i].indexOf(origin) !== -1) { isAllowed = true; break; } @@ -7395,9 +7409,20 @@ F.$cors = function(req, res, fn, arg) { if (!isAllowed) stop = true; } + } else if (CONF['default-cors']) { + origin = headers.origin.toLowerCase(); + if (CONF['default-cors'].indexOf(origin) === -1) { + stop = true; + isAllowed = false; + } } - res.setHeader('Access-Control-Allow-Origin', F.routes.corsall ? headers['origin'] : (cors.origins ? cors.origins : cors.credentials ? isAllowed ? origin : cors.origins ? cors.origins : origin : headers['origin'])); + if (stop) + origin = 'null'; + else + origin = headers.origin; + + res.setHeader('Access-Control-Allow-Origin', origin); if (!cors || cors.credentials) res.setHeader('Access-Control-Allow-Credentials', 'true'); @@ -8692,6 +8717,23 @@ F.$configure_configs = function(arr, rewrite) { obj[tmp] = value; break; + case 'default-cors': + value = value.replace(/,/g, ' ').split(' '); + tmp = []; + for (var j = 0; j < value.length; j++) { + var co = (value[j] || '').trim(); + if (co) { + co = co.toLowerCase(); + if (co.substring(0, 2) === '//') { + tmp.push('http:' + co); + tmp.push('https:' + co); + } else + tmp.push(co); + } + } + obj[name] = tmp.length ? tmp : null; + break; + case 'allow-handle-static-files': OBSOLETE('config["allow-handle-static-files"]', 'The key has been renamed to "allow-static-files"'); obj['allow-static-files'] = true; @@ -13680,7 +13722,7 @@ WebSocketClient.prototype.prepare = function(flags, protocols, allow, length) { self.length = length; - var origin = self.req.headers['origin'] || ''; + var origin = self.req.headers.origin || ''; var length = allow.length; if (length && allow.indexOf('*') === -1) { diff --git a/test/test-framework-debug.js b/test/test-framework-debug.js index cbcade096..5f723711b 100755 --- a/test/test-framework-debug.js +++ b/test/test-framework-debug.js @@ -140,7 +140,7 @@ function test_routing(next) { // "access-control-allow-origin" doesn't support * (wildcard) when "access-control-allow-credentials" is set to true // node.js doesn't support duplicates headers - assert.ok(headers['access-control-allow-origin'] === 'http://www.petersirka.eu', 'CORS, headers problem 1'); + assert.ok(headers['access-control-allow-origin'] === 'null', 'CORS, headers problem 1'); assert.ok(headers['access-control-allow-credentials'] === 'true', 'CORS, headers problem 2'); assert.ok(headers['access-control-allow-methods'] === 'POST, PUT, DELETE, OPTIONS', 'CORS, headers problem 3'); assert.ok(headers['access-control-allow-headers'] === 'x-ping', 'CORS, headers problem 4'); diff --git a/test/test-framework-release.js b/test/test-framework-release.js index e47ac4ad0..3cfccb3ac 100755 --- a/test/test-framework-release.js +++ b/test/test-framework-release.js @@ -140,7 +140,7 @@ function test_routing(next) { // "access-control-allow-origin" doesn't support * (wildcard) when "access-control-allow-credentials" is set to true // node.js doesn't support duplicates headers - assert.ok(headers['access-control-allow-origin'] === 'http://www.petersirka.eu', 'CORS, headers problem 1'); + assert.ok(headers['access-control-allow-origin'] === 'null', 'CORS, headers problem 1'); assert.ok(headers['access-control-allow-credentials'] === 'true', 'CORS, headers problem 2'); assert.ok(headers['access-control-allow-methods'] === 'POST, PUT, DELETE, OPTIONS', 'CORS, headers problem 3'); assert.ok(headers['access-control-allow-headers'] === 'x-ping', 'CORS, headers problem 4'); From 3c4f49053a58afe3096098c0782021e2139fbd91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 13 Oct 2018 00:28:02 +0200 Subject: [PATCH 124/217] Fixed CORS. --- index.js | 83 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 47 insertions(+), 36 deletions(-) diff --git a/index.js b/index.js index 088b4b220..b74dcbeef 100755 --- a/index.js +++ b/index.js @@ -1687,13 +1687,12 @@ global.CORS = F.cors = function(url, flags, credentials) { } if (flag.substring(0, 2) === '//') { - origin.push('http:' + flag); - origin.push('https:' + flag); + origin.push(flag.substring(2)); continue; } if (flag.startsWith('http://') || flag.startsWith('https://')) { - origin.push(flag); + origin.push(flag.substring(flag.indexOf('/') + 2)); continue; } @@ -7349,6 +7348,7 @@ F.$cors = function(req, res, fn, arg) { var isAllowed = F.routes.corsall; var cors, origin; var headers = req.headers; + var key; if (!isAllowed) { @@ -7365,29 +7365,18 @@ F.$cors = function(req, res, fn, arg) { var stop = false; - if (!isAllowed) - stop = true; + key = 'cors' + cors.hash + '_' + headers.origin; - isAllowed = false; + if (F.temporary.other[key]) { + stop = F.temporary.other[key] === 2; + } else { - if (!stop && cors.headers) { isAllowed = false; - for (var i = 0, length = cors.headers.length; i < length; i++) { - if (headers[cors.headers[i]]) { - isAllowed = true; - break; - } - } - if (!isAllowed) - stop = true; - } - if (!stop && cors.methods) { - isAllowed = false; - var current = headers['access-control-request-method'] || req.method; - if (current !== 'OPTIONS') { - for (var i = 0, length = cors.methods.length; i < length; i++) { - if (current === cors.methods[i]) { + if (cors.headers) { + isAllowed = false; + for (var i = 0, length = cors.headers.length; i < length; i++) { + if (headers[cors.headers[i]]) { isAllowed = true; break; } @@ -7395,25 +7384,47 @@ F.$cors = function(req, res, fn, arg) { if (!isAllowed) stop = true; } - } - if (!stop && cors.origin) { - origin = headers.origin.toLowerCase(); - isAllowed = false; - for (var i = 0, length = cors.origin.length; i < length; i++) { - if (cors.origin[i].indexOf(origin) !== -1) { - isAllowed = true; - break; + if (!stop && cors.methods) { + isAllowed = false; + var current = headers['access-control-request-method'] || req.method; + if (current !== 'OPTIONS') { + for (var i = 0, length = cors.methods.length; i < length; i++) { + if (current === cors.methods[i]) { + isAllowed = true; + break; + } + } + if (!isAllowed) + stop = true; + } + } + + if (!stop && cors.origin) { + origin = headers.origin.toLowerCase().substring(headers.origin.indexOf('/') + 2); + if (origin !== headers.host) { + isAllowed = false; + for (var i = 0, length = cors.origin.length; i < length; i++) { + if (cors.origin[i].indexOf(origin) !== -1) { + isAllowed = true; + break; + } + } + if (!isAllowed) + stop = true; } } - if (!isAllowed) - stop = true; + + F.temporary.other[key] = stop ? 2 : 1; } } else if (CONF['default-cors']) { - origin = headers.origin.toLowerCase(); - if (CONF['default-cors'].indexOf(origin) === -1) { - stop = true; - isAllowed = false; + key = headers.origin; + if (F.temporary.other[key]) { + stop = F.temporary.other[key] === 2; + } else { + origin = key.toLowerCase().substring(key.indexOf('/') + 2); + stop = origin !== headers.host && CONF['default-cors'].indexOf(origin) === -1; + F.temporary.other[key] = stop ? 2 : 1; } } From a229c9c49f39fed35a1346eb47bf494aad48df42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 13 Oct 2018 21:00:08 +0200 Subject: [PATCH 125/217] Fixed CORS. --- index.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index b74dcbeef..313ea00ac 100755 --- a/index.js +++ b/index.js @@ -8736,10 +8736,9 @@ F.$configure_configs = function(arr, rewrite) { if (co) { co = co.toLowerCase(); if (co.substring(0, 2) === '//') { - tmp.push('http:' + co); - tmp.push('https:' + co); - } else tmp.push(co); + } else + tmp.push(co.substring(co.indexOf('/') + 2)); } } obj[name] = tmp.length ? tmp : null; From cccc0d5cd5eb233d2f90305b7b3d0e21621d1e8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 15 Oct 2018 12:48:43 +0200 Subject: [PATCH 126/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 92287866c..8dc06068c 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-16", + "version": "3.0.1-17", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 273da270b5c6d10089bedbe41c94910c7f3136f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 15 Oct 2018 19:05:07 +0200 Subject: [PATCH 127/217] Fixed `REQUEST()` and `controller.proxy()`. --- changes.txt | 2 ++ utils.js | 13 ++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/changes.txt b/changes.txt index 5c4ccdb18..750fce167 100755 --- a/changes.txt +++ b/changes.txt @@ -23,6 +23,7 @@ - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) - updated: routing, now it supports operations in the form `ROUTE('.. * --> @save_operation @load_operation (response)')` - updated: `ROUTE()` supports multiple HTTP method declaration `ROUTE('GET,POST,PUT /something/', action)` +- updated: `REQUEST()` can return binary data if the content-type is not `text/*` or `application/*`. - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator @@ -44,6 +45,7 @@ - fixed: RESTBuilder cache works only if the response status is `200` - fixed: compressing CSS with `\t` tabs - fixed: `controller.autoclear()` +- fixed: `controller.proxy()` - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute diff --git a/utils.js b/utils.js index 22671ee81..ef49b8004 100755 --- a/utils.js +++ b/utils.js @@ -89,6 +89,7 @@ const REG_ROOT = /@\{#\}(\/)?/g; const REG_NOREMAP = /@\{noremap\}(\n)?/g; const REG_REMAP = /href=".*?"|src=".*?"/gi; const REG_URLEXT = /(https|http|wss|ws|file):\/\/|\/\/[a-z0-9]|[a-z]:/i; +const REG_TEXTAPPLICATION = /text|application/i; exports.MONTHS = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']; exports.DAYS = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday']; @@ -973,17 +974,23 @@ function request_response(res, uri, options) { } var self = this; - var str = self._buffer ? self._buffer.toString(options.encoding) : ''; + var data; + + if (!self.headers['content-type'] || REG_TEXTAPPLICATION.test(self.headers['content-type'])) + data = self._buffer ? self._buffer.toString(options.encoding) : ''; + else + data = self._buffer; + self._buffer = undefined; if (options.evt) { - options.evt.$events.end && options.evt.emit('end', str, self.statusCode, self.headers, uri.host, options.cookies); + options.evt.$events.end && options.evt.emit('end', data, self.statusCode, self.headers, uri.host, options.cookies); options.evt.removeAllListeners(); options.evt = null; } if (options.callback) { - options.callback(null, uri.method === 'HEAD' ? self.headers : str, self.statusCode, self.headers, uri.host, options.cookies); + options.callback(null, uri.method === 'HEAD' ? self.headers : data, self.statusCode, self.headers, uri.host, options.cookies); options.callback = null; } From 4b9938ef2d16384af246225c1649569d6a1b341d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 15 Oct 2018 19:29:50 +0200 Subject: [PATCH 128/217] Fixed `controller.proxy()`. --- index.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 313ea00ac..a507e78e0 100755 --- a/index.js +++ b/index.js @@ -12661,7 +12661,11 @@ Controller.prototype.proxy = Controller.prototype.proxy2 = function(url, callbac } else { self.status = code; callback && callback(err, data, code, headers); - self.content(data, (headers['content-type'] || 'text/plain').replace(REG_ENCODINGCLEANER, '')); + var ct = (headers['content-type'] || 'text/plain').replace(REG_ENCODINGCLEANER, ''); + if (data instanceof Buffer) + self.binary(data, ct); + else + self.content(data, ct); } }, null, h, ENCODING, timeout || 10000); From 5ed1e16e6fe838d01396f2c4f796e696e5d28681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 15 Oct 2018 22:02:35 +0200 Subject: [PATCH 129/217] Component update. --- index.js | 9 +++++++-- internal.js | 24 ++++++++++++------------ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/index.js b/index.js index a507e78e0..387345d23 100755 --- a/index.js +++ b/index.js @@ -10280,9 +10280,13 @@ Controller.prototype.getSchema = function() { Controller.prototype.component = function(name, settings, model) { var filename = F.components.views[name]; if (filename) { - var generator = framework_internal.viewEngine(name, filename, this); + var self = this; + var tmp = F.components.instances[name]; + self.$hasComponents && (self.$hasComponents = []); + self.$hasComponents.push(tmp.group); + var generator = framework_internal.viewEngine(name, filename, self); if (generator) - return generator.call(this, this, this.repository, model || this.$model, this.session, this.query, this.body, this.url, F.global, F.helpers, this.user, this.config, F.functions, 0, this.outputPartial, this.req.files, this.req.mobile, settings || EMPTYOBJECT); + return generator.call(self, self, self.repository, model || self.$model, self.session, self.query, self.body, self.url, F.global, F.helpers, self.user, self.config, F.functions, 0, self.outputPartial, self.req.files, self.req.mobile, settings || EMPTYOBJECT); } return ''; }; @@ -11016,6 +11020,7 @@ Controller.prototype.$view = function(name, model, expire, key) { } var value = self.view(name, model, null, true, true, cache); + if (!value) return ''; diff --git a/internal.js b/internal.js index 7d25cd636..5999721a4 100755 --- a/internal.js +++ b/internal.js @@ -1913,14 +1913,11 @@ function view_parse(content, minify, filename, controller) { builder += '+self.$import(' + tmpimp + ')'; } } - if (tmp.indexOf('components') !== -1) - controller.$hasComponents = true; can = true; break; } } - } else if (!controller.$hasComponents && tmp.indexOf('components') !== -1) - controller.$hasComponents = true; + } if (can && !counter) { try { @@ -2184,12 +2181,10 @@ function view_prepare(command, dynamicCommand, functions, controller) { if (!controller.$hasComponents) controller.$hasComponents = []; - if (controller.$hasComponents instanceof Array) { - var group = command.match(REG_COMPONENTS_GROUP); - if (group && group.length) { - group = group[0].toString().replace(/'|"'/g, ''); - controller.$hasComponents.indexOf(group) === -1 && controller.$hasComponents.push(group); - } + var group = command.match(REG_COMPONENTS_GROUP); + if (group && group.length) { + group = group[0].toString().replace(/'|"'/g, ''); + controller.$hasComponents.indexOf(group) === -1 && controller.$hasComponents.push(group); } return 'self.$' + command + (command.indexOf('(') === -1 ? '()' : ''); @@ -2199,7 +2194,6 @@ function view_prepare(command, dynamicCommand, functions, controller) { case 'component': - controller.$hasComponents = true; tmp = command.indexOf('\''); var is = false; @@ -2216,8 +2210,14 @@ function view_prepare(command, dynamicCommand, functions, controller) { is = true; } - if (is) { + + if (tmp.group) { + !controller.$hasComponents && (controller.$hasComponents = []); + controller.$hasComponents.push(tmp.group); + } else + controller.$hasComponents = true; + var settings = command.substring(11 + name.length + 2, command.length - 1).trim(); if (settings === ')') settings = ''; From a7648642476a4e033444e37d256fb52284480baa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 16 Oct 2018 09:14:32 +0200 Subject: [PATCH 130/217] Improved components. --- index.js | 71 ++++++++++++++++++++++--------------- internal.js | 29 +++++++-------- test/controllers/default.js | 2 +- test/views/params.html | 2 +- 4 files changed, 56 insertions(+), 48 deletions(-) diff --git a/index.js b/index.js index 387345d23..b1cdf0bca 100755 --- a/index.js +++ b/index.js @@ -82,6 +82,7 @@ const REPOSITORY_META_AUTHOR = '$author'; const REPOSITORY_META_IMAGE = '$image'; const REPOSITORY_PLACE = '$place'; const REPOSITORY_SITEMAP = '$sitemap'; +const REPOSITORY_COMPONENTS = '$components'; const ATTR_END = '"'; const ETAG = '858'; const CONCAT = [null, null]; @@ -4113,7 +4114,11 @@ F.install = function(type, name, declaration, options, callback, internal, useRe } } - if (obj && obj.group) { + if (obj) { + + if (!obj.group) + obj.group = 'default'; + key = obj.group.crc32(true); temporary += '_g' + key; tmp = F.components.groups[obj.group]; @@ -4130,8 +4135,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe tmp.css = true; } - tmp.version = NOW.getTime(); - tmp.links = (tmp.js ? ''.format(link, tmp.version, key) : '') + (tmp.css ? ''.format(link, tmp.version, key) : ''); + tmp.version = GUID(5); + tmp.links = (tmp.js ? ''.format(link, tmp.version, key) : '') + (tmp.css ? ''.format(link, tmp.version, key) : ''); } !skipEmit && setTimeout(function() { @@ -10281,12 +10286,16 @@ Controller.prototype.component = function(name, settings, model) { var filename = F.components.views[name]; if (filename) { var self = this; - var tmp = F.components.instances[name]; - self.$hasComponents && (self.$hasComponents = []); - self.$hasComponents.push(tmp.group); - var generator = framework_internal.viewEngine(name, filename, self); - if (generator) + var generator = framework_internal.viewEngine(name, filename, self, true); + if (generator) { + if (generator.components.length) { + if (!self.repository[REPOSITORY_COMPONENTS]) + self.repository[REPOSITORY_COMPONENTS] = {}; + for (var i = 0; i < generator.components.length; i++) + self.repository[REPOSITORY_COMPONENTS][generator.components[i]] = 1; + } return generator.call(self, self, self.repository, model || self.$model, self.session, self.query, self.body, self.url, F.global, F.helpers, self.user, self.config, F.functions, 0, self.outputPartial, self.req.files, self.req.mobile, settings || EMPTYOBJECT); + } } return ''; }; @@ -11012,11 +11021,8 @@ Controller.prototype.$view = function(name, model, expire, key) { if (expire) { cache = '$view.' + name + '.' + (key || ''); var output = self.cache.read2(cache); - if (output) { - if (output.components) - self.$hasComponents = true; + if (output) return output.body; - } } var value = self.view(name, model, null, true, true, cache); @@ -11397,15 +11403,11 @@ Controller.prototype.head = function() { if (!arguments.length) { var author = self.repository[REPOSITORY_META_AUTHOR] || self.config.author; var plus = ''; - if (self.$hasComponents) { - if (self.$hasComponents instanceof Array) { - for (var i = 0; i < self.$hasComponents.length; i++) { - var group = F.components.groups[self.$hasComponents[i]]; - if (group) - plus += group.links; - } - } else - plus = F.components.links; + var components = self.repository[REPOSITORY_COMPONENTS]; + if (components) { + var keys = Object.keys(components); + for (var i = 0; i < keys.length; i++) + plus += F.components.groups[keys[i]].links; } return (author ? '' : '') + (self.repository[REPOSITORY_HEAD] || '') + plus; } @@ -11625,8 +11627,6 @@ Controller.prototype.$import = function() { if (filename === 'components' && F.components.has) { // Generated in controller.head() - // self.$hasComponents = true; - // builder += F.components.links; continue; } @@ -12847,7 +12847,16 @@ Controller.prototype.$viewrender = function(filename, generator, model, headers, var helpers = F.helpers; try { + value = generator.call(self, self, self.repository, model, self.session, self.query, self.body, self.url, F.global, helpers, self.user, self.config, F.functions, 0, partial ? self.outputPartial : self.output, self.req.files, self.req.mobile, EMPTYOBJECT); + + if (generator.components.length) { + if (!self.repository[REPOSITORY_COMPONENTS]) + self.repository[REPOSITORY_COMPONENTS] = {}; + for (var i = 0; i < generator.components.length; i++) + self.repository[REPOSITORY_COMPONENTS][generator.components[i]] = 1; + } + } catch (ex) { err = new Error('View "' + filename + '": ' + ex.message); @@ -13048,9 +13057,6 @@ Controller.prototype.memorize = function(key, expires, disabled, fnTo, fnFrom) { self.layoutName = output.layout; self.themeName = output.theme; - if (output.components) - self.$hasComponents = true; - var res = self.res; res.options.code = self.status || 200; @@ -13117,7 +13123,7 @@ Controller.prototype.$memorize_prepare = function(key, expires, disabled, fnTo, return; } - var options = { content: value, type: contentType || CT_TEXT, layout: self.layoutName, theme: self.themeName, components: (self.$viewasync ? true : false) || (self.$hasComponents ? true : false) }; + var options = { content: value, type: contentType || CT_TEXT, layout: self.layoutName, theme: self.themeName }; if (headers) options.headers = headers; @@ -15489,7 +15495,13 @@ function extend_response(PROTO) { if (F.components.has && F.components[req.extension] && req.uri.pathname === F.config['static-url-components'] + req.extension) { res.noCompress = true; - filename = F.path.temp('components' + (req.query.group ? '_g' + req.query.group : '') + '.' + req.extension); + res.options.components = true; + var g = req.query.group ? req.query.group.substring(0, req.query.group.length - 6) : ''; + filename = F.path.temp('components' + (g ? '_g' + g : '') + '.' + req.extension); + if (g) + req.$key = 'components_' + g + '.' + req.extension; + else + req.$key = 'components.' + req.extension; } res.options.filename = filename; @@ -15663,7 +15675,8 @@ function extend_response(PROTO) { return res; } - (DEBUG || res.$nocache) && F.isProcessed(req.$key) && (F.temporary.path[req.$key] = undefined); + if (!res.options.components && (DEBUG || res.$nocache)) + F.isProcessed(req.$key) && (F.temporary.path[req.$key] = undefined); if (name[1] && !compress) headers[HEADER_LENGTH] = name[1]; diff --git a/internal.js b/internal.js index 5999721a4..47f88241c 100755 --- a/internal.js +++ b/internal.js @@ -1769,6 +1769,7 @@ function view_parse(content, minify, filename, controller) { var isCOMPILATION = false; var builderTMP = ''; var sectionName = ''; + var components = {}; var text; while (command) { @@ -1893,7 +1894,7 @@ function view_parse(content, minify, filename, controller) { builder += '}$output+=$EMPTY'; } else { - tmp = view_prepare(command.command, newCommand, functionsName, controller, filename); + tmp = view_prepare(command.command, newCommand, functionsName, controller, components); var can = false; // Inline rendering is supported only in release mode @@ -1962,10 +1963,11 @@ function view_parse(content, minify, filename, controller) { if (RELEASE) builder = builder.replace(/(\+\$EMPTY\+)/g, '+').replace(/(\$output=\$EMPTY\+)/g, '$output=').replace(/(\$output\+=\$EMPTY\+)/g, '$output+=').replace(/(\}\$output\+=\$EMPTY)/g, '}').replace(/(\{\$output\+=\$EMPTY;)/g, '{').replace(/(\+\$EMPTY\+)/g, '+').replace(/(>'\+'<)/g, '><').replace(/'\+'/g, ''); - var fn = ('(function(self,repository,model,session,query,body,url,global,helpers,user,config,functions,index,output,files,mobile,settings){self.$hasComponents=' + (controller.$hasComponents instanceof Array ? JSON.stringify(controller.$hasComponents).replace(/"/g, '\'') : controller.$hasComponents === true ? 'true' : 'null') + ';var get=query;var post=body;var G=F.global;var R=this.repository;var M=model;var theme=this.themeName;var language=this.language;var sitemap=this.repository.$sitemap;' + (isCookie ? 'var cookie=function(name){return self.req.cookie(name)};' : '') + (functions.length ? functions.join('') + ';' : '') + 'var controller=self;' + builder + ';return $output;})'); + var fn = ('(function(self,repository,model,session,query,body,url,global,helpers,user,config,functions,index,output,files,mobile,settings){var G=F.global;var R=this.repository;var M=model;var theme=this.themeName;var language=this.language;var sitemap=this.repository.$sitemap;' + (isCookie ? 'var cookie=function(name){return self.req.cookie(name)};' : '') + (functions.length ? functions.join('') + ';' : '') + 'var controller=self;' + builder + ';return $output;})'); try { fn = eval(fn); + fn.components = Object.keys(components); } catch (e) { throw new Error(filename + ': ' + e.message.toString()); } @@ -1985,7 +1987,7 @@ function view_parse_plus(builder) { return c !== '!' && c !== '?' && c !== '+' && c !== '.' && c !== ':'; } -function view_prepare(command, dynamicCommand, functions, controller) { +function view_prepare(command, dynamicCommand, functions, controller, components) { var a = command.indexOf('.'); var b = command.indexOf('('); @@ -2050,8 +2052,6 @@ function view_prepare(command, dynamicCommand, functions, controller) { case 'G': case 'model': case 'repository': - case 'get': - case 'post': case 'query': case 'global': case 'session': @@ -2178,13 +2178,10 @@ function view_prepare(command, dynamicCommand, functions, controller) { case 'components': - if (!controller.$hasComponents) - controller.$hasComponents = []; - var group = command.match(REG_COMPONENTS_GROUP); if (group && group.length) { group = group[0].toString().replace(/'|"'/g, ''); - controller.$hasComponents.indexOf(group) === -1 && controller.$hasComponents.push(group); + components[group] = 1; } return 'self.$' + command + (command.indexOf('(') === -1 ? '()' : ''); @@ -2210,17 +2207,15 @@ function view_prepare(command, dynamicCommand, functions, controller) { is = true; } - if (is) { + if (tmp) + components[tmp.group] = 1; - if (tmp.group) { - !controller.$hasComponents && (controller.$hasComponents = []); - controller.$hasComponents.push(tmp.group); - } else - controller.$hasComponents = true; + if (is) { var settings = command.substring(11 + name.length + 2, command.length - 1).trim(); if (settings === ')') settings = ''; + $VIEWASYNC++; return '\'@{-{0}-}\'+(function(index){!controller.$viewasync&&(controller.$viewasync=[]);controller.$viewasync.push({replace:\'@{-{0}-}\',name:\'{1}\',settings:{2}});return $EMPTY})({0})'.format($VIEWASYNC, name, settings || 'null'); } @@ -2988,7 +2983,7 @@ function modificators(value, filename, type) { return value; } -function viewengine_load(name, filename, controller) { +function viewengine_load(name, filename, controller, component) { var precompiled = F.routes.views[name]; if (precompiled) @@ -3004,7 +2999,7 @@ function viewengine_load(name, filename, controller) { generator = viewengine_read(filename, controller); - if (!F.isDebug) + if (component || !F.isDebug) F.temporary.views[key] = generator; return generator; diff --git a/test/controllers/default.js b/test/controllers/default.js index 6e74ebade..c535d0328 100755 --- a/test/controllers/default.js +++ b/test/controllers/default.js @@ -183,7 +183,7 @@ function plain_post_parse() { var self = this; self.layout(''); var output = self.view('params', null, true); - assert.ok(output === '--body=total.js--query=query--post=total.js--get=query--', 'Problem with getting values from request body and URL.'); + assert.ok(output === '--body=total.js--query=query--', 'Problem with getting values from request body and URL.'); self.body.type = 'parse'; self.json(self.body); } diff --git a/test/views/params.html b/test/views/params.html index d9fe03ba6..52d79dbdc 100644 --- a/test/views/params.html +++ b/test/views/params.html @@ -1 +1 @@ ---body=@{body.name}--query=@{query.value}--post=@{post.name}--get=@{get.value}-- \ No newline at end of file +--body=@{body.name}--query=@{query.value}-- \ No newline at end of file From 3708e137e03090554fb098a35f799df14f594a81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 16 Oct 2018 09:34:21 +0200 Subject: [PATCH 131/217] Added new change. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index 750fce167..1c6590021 100755 --- a/changes.txt +++ b/changes.txt @@ -50,6 +50,7 @@ - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute - improved: CORS +- improved: rendering of components ======= 3.0.0 From a8dd5fc977002788b84b6f113b3e14b8dc6c6500 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 16 Oct 2018 13:01:04 +0200 Subject: [PATCH 132/217] Fixed rendering of group of components. --- index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index b1cdf0bca..e143d803f 100755 --- a/index.js +++ b/index.js @@ -12848,8 +12848,6 @@ Controller.prototype.$viewrender = function(filename, generator, model, headers, try { - value = generator.call(self, self, self.repository, model, self.session, self.query, self.body, self.url, F.global, helpers, self.user, self.config, F.functions, 0, partial ? self.outputPartial : self.output, self.req.files, self.req.mobile, EMPTYOBJECT); - if (generator.components.length) { if (!self.repository[REPOSITORY_COMPONENTS]) self.repository[REPOSITORY_COMPONENTS] = {}; @@ -12857,6 +12855,8 @@ Controller.prototype.$viewrender = function(filename, generator, model, headers, self.repository[REPOSITORY_COMPONENTS][generator.components[i]] = 1; } + value = generator.call(self, self, self.repository, model, self.session, self.query, self.body, self.url, F.global, helpers, self.user, self.config, F.functions, 0, partial ? self.outputPartial : self.output, self.req.files, self.req.mobile, EMPTYOBJECT); + } catch (ex) { err = new Error('View "' + filename + '": ' + ex.message); From 57e0533d2fdfdca96130bd3342fe0be50753ebcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 16 Oct 2018 13:51:01 +0200 Subject: [PATCH 133/217] Fixed components rendering. --- index.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index e143d803f..9ac80ec65 100755 --- a/index.js +++ b/index.js @@ -11406,8 +11406,11 @@ Controller.prototype.head = function() { var components = self.repository[REPOSITORY_COMPONENTS]; if (components) { var keys = Object.keys(components); - for (var i = 0; i < keys.length; i++) - plus += F.components.groups[keys[i]].links; + for (var i = 0; i < keys.length; i++) { + var com = F.components.groups[keys[i]]; + if (com) + plus += com.links; + } } return (author ? '' : '') + (self.repository[REPOSITORY_HEAD] || '') + plus; } From c311eedbbc55799a61d2bd14b41fb4606a8debe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 18 Oct 2018 22:48:27 +0200 Subject: [PATCH 134/217] Added `CACHE()`. --- changes.txt | 1 + index.js | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/changes.txt b/changes.txt index 1c6590021..97e65cd37 100755 --- a/changes.txt +++ b/changes.txt @@ -18,6 +18,7 @@ - added: a new config item `default-request-maxkeys : 33` for restricting query max. keys - added: a new config item `logger : false` which enables logging for Middleware, Schemas and Operations - added: `SchemaOptions` and `OperationOptions` supports `$.cancel()` method +- added: `CACHE(name, [value], [expire], [persistent])` alias for `F.cache.get2()` and `F.cache.set()` or `F.cache.set2()` - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) diff --git a/index.js b/index.js index 9ac80ec65..3c6747aa2 100755 --- a/index.js +++ b/index.js @@ -293,6 +293,11 @@ global.NOMEM = global.NOSQLMEMORY = (name, view) => global.framework_nosql.inmem global.CONFIG = function(name, val) { return arguments.length === 1 ? F.config[name] : (F.config[name] = val); }; + +global.CACHE = function(name, value, expire, persistent) { + return arguments.length === 1 ? F.cache.get2(name) : F.cache.set(name, value, expire, null, persistent); +}; + global.UPTODATE = (type, url, options, interval, callback) => F.uptodate(type, url, options, interval, callback); global.INSTALL = (type, name, declaration, options, callback) => F.install(type, name, declaration, options, callback); global.UNINSTALL = (type, name, options) => F.uninstall(type, name, options); From 61ea98e393280482ff15e8527a8116656924366b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 19 Oct 2018 14:37:21 +0200 Subject: [PATCH 135/217] New beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8dc06068c..6ea024b0c 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-17", + "version": "3.0.1-18", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 1314c44fc6591518ec9264e6527f85bbd00a68da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 21 Oct 2018 11:21:24 +0200 Subject: [PATCH 136/217] Renamed config keys. --- bin/totaljs | 10 +- builders.js | 8 +- bundles.js | 36 +- debug.js | 42 +- image.js | 6 +- index.js | 983 ++++++++++++++++++++---------------- internal.js | 30 +- mail.js | 18 +- nosql.js | 26 +- package.json | 2 +- test/modules/inline-view.js | 10 +- test/modules/test.js | 6 +- test/test-tmp.js | 2 +- utils.js | 28 +- 14 files changed, 657 insertions(+), 550 deletions(-) diff --git a/bin/totaljs b/bin/totaljs index 04d7be51d..2bcc7baa0 100755 --- a/bin/totaljs +++ b/bin/totaljs @@ -361,11 +361,11 @@ function main() { var port = cmd.parseInt(); if (port) { - F.config['directory-temp'] = '~' + path.join(os.tmpdir(), 'totaljs' + dir.hash()); - F.config['directory-public'] = '~' + dir; - F.config['allow-compile-html'] = false; - F.config['allow-compile-script'] = false; - F.config['allow-compile-style'] = false; + CONF.directory_temp = '~' + path.join(os.tmpdir(), 'totaljs' + dir.hash()); + CONF.directory_public = '~' + dir; + CONF.allow_compile_html = false; + CONF.allow_compile_script = false; + CONF.allow_compile_style = false; F.accept('.less', 'text/less'); diff --git a/builders.js b/builders.js index 57d880cfa..217f1253d 100755 --- a/builders.js +++ b/builders.js @@ -3111,16 +3111,16 @@ SchemaInstance.prototype.$constant = function(name) { function ErrorBuilder(onResource) { this.items = []; - this.transformName = transforms['error_default']; + this.transformName = transforms.error_default; this.onResource = onResource; - this.resourceName = F.config['default-errorbuilder-resource-name']; - this.resourcePrefix = F.config['default-errorbuilder-resource-prefix'] || ''; + this.resourceName = CONF.default_errorbuilder_resource_name; + this.resourcePrefix = CONF.default_errorbuilder_resource_prefix || ''; this.isResourceCustom = false; this.count = 0; this.replacer = []; this.isPrepared = false; this.contentType = 'application/json'; - this.status = F.config['default-errorbuilder-status'] || 200; + this.status = CONF.default_errorbuilder_status || 200; // Hidden: when the .push() contains a classic Error instance // this.unexpected; diff --git a/bundles.js b/bundles.js index 7eaa944cc..0263dc3ec 100755 --- a/bundles.js +++ b/bundles.js @@ -30,10 +30,10 @@ exports.make = function(callback) { META.ignore = makeignore(Fs.readFileSync(Path.join(path, '.bundlesignore')).toString('utf8').split('\n')); } catch (e) {} - blacklist[F.config['directory-temp']] = 1; - blacklist[F.config['directory-bundles']] = 1; - blacklist[F.config['directory-src']] = 1; - blacklist[F.config['directory-logs']] = 1; + blacklist[CONF.directory_temp] = 1; + blacklist[CONF.directory_bundles] = 1; + blacklist[CONF.directory_src] = 1; + blacklist[CONF.directory_logs] = 1; blacklist['/node_modules/'] = 1; blacklist['/debug.pid'] = 1; blacklist['/package-lock.json'] = 1; @@ -47,11 +47,11 @@ exports.make = function(callback) { async.push(cleanFiles); async.push(function(next) { - var target = F.path.root(F.config['directory-src']); - U.ls(F.path.root(F.config['directory-bundles']), function(files) { + var target = F.path.root(CONF.directory_src); + U.ls(F.path.root(CONF.directory_bundles), function(files) { var dirs = {}; files.wait(function(filename, resume) { - var dbpath = F.config['directory-databases']; + var dbpath = CONF.directory_databases; F.restore(filename, target, resume, function(p, dir) { @@ -118,9 +118,9 @@ exports.make = function(callback) { var file = files[i].substring(Length); var type = 0; - if (file.startsWith(F.config['directory-databases']) || file.startsWith('/flow/') || file.startsWith('/dashboard/')) + if (file.startsWith(CONF.directory_databases) || file.startsWith('/flow/') || file.startsWith('/dashboard/')) type = 1; - else if (file.startsWith(F.config['directory-public'])) + else if (file.startsWith(CONF.directory_public)) type = 2; else if (REGAPPEND.test(file)) { file = file.replace(/\/--/g, '/'); @@ -144,7 +144,7 @@ exports.make = function(callback) { }); async.push(function(next) { - Fs.writeFileSync(Path.join(F.path.root(F.config['directory-src']), 'bundle.json'), JSON.stringify(META, null, '\t')); + Fs.writeFileSync(Path.join(F.path.root(CONF.directory_src), 'bundle.json'), JSON.stringify(META, null, '\t')); next(); }); @@ -196,13 +196,13 @@ function normalize(path) { function cleanFiles(callback) { - var path = F.path.root(F.config['directory-src']); + var path = F.path.root(CONF.directory_src); var length = path.length - 1; var blacklist = {}; - blacklist[F.config['directory-public']] = 1; - blacklist[F.config['directory-private']] = 1; - blacklist[F.config['directory-databases']] = 1; + blacklist[CONF.directory_public] = 1; + blacklist[CONF.directory_private] = 1; + blacklist[CONF.directory_databases] = 1; var meta; @@ -240,7 +240,7 @@ function cleanFiles(callback) { function createDirectories(dirs, callback) { - var path = F.path.root(F.config['directory-src']); + var path = F.path.root(CONF.directory_src); try { Fs.mkdirSync(path); @@ -259,7 +259,7 @@ function createDirectories(dirs, callback) { } function copyFiles(files, callback) { - var path = F.path.root(F.config['directory-src']); + var path = F.path.root(CONF.directory_src); files.wait(function(file, next) { if (!META.ignore(file.name)) @@ -289,7 +289,7 @@ function copyFiles(files, callback) { append = true; if (CONSOLE && exists) { - F.config['allow-debug'] && F.consoledebug(append ? 'EXT:' : 'REW:', p); + CONF.allow_debug && F.consoledebug(append ? 'EXT:' : 'REW:', p); } else F.consoledebug(append ? 'EXT:' : 'COP:', p); @@ -299,7 +299,7 @@ function copyFiles(files, callback) { copyFile(file.filename, filename, next); if (CONSOLE && exists) - F.config['allow-debug'] && F.consoledebug('REW:', p); + CONF.allow_debug && F.consoledebug('REW:', p); else F.consoledebug('COP:', p); diff --git a/debug.js b/debug.js index 1924a6728..dd83e6d97 100644 --- a/debug.js +++ b/debug.js @@ -107,25 +107,25 @@ function runwatching() { const fork = require('child_process').fork; const directories = [ - U.combine(F.config['directory-components']), - U.combine(F.config['directory-controllers']), - U.combine(F.config['directory-definitions']), - U.combine(F.config['directory-operations']), - U.combine(F.config['directory-isomorphic']), - U.combine(F.config['directory-modules']), - U.combine(F.config['directory-models']), - U.combine(F.config['directory-schemas']), - U.combine(F.config['directory-resources']), - U.combine(F.config['directory-source']), - U.combine(F.config['directory-workers']), - U.combine(F.config['directory-packages']), - U.combine(F.config['directory-themes']), - U.combine(F.config['directory-configs']), - U.combine(F.config['directory-bundles']), + U.combine(CONF.directory_components), + U.combine(CONF.directory_controllers), + U.combine(CONF.directory_definitions), + U.combine(CONF.directory_operations), + U.combine(CONF.directory_isomorphic), + U.combine(CONF.directory_modules), + U.combine(CONF.directory_models), + U.combine(CONF.directory_schemas), + U.combine(CONF.directory_resources), + U.combine(CONF.directory_source), + U.combine(CONF.directory_workers), + U.combine(CONF.directory_packages), + U.combine(CONF.directory_themes), + U.combine(CONF.directory_configs), + U.combine(CONF.directory_bundles), U.combine('/startup/') ]; - const SRC = U.combine(F.config['directory-src']); + const SRC = U.combine(CONF.directory_src); const prefix = '----------------> '; options.watch && options.watch.forEach(function(item) { @@ -173,13 +173,13 @@ function runwatching() { } try { - Fs.statSync(F.path.root(F.config['directory-bundles'])); + Fs.statSync(F.path.root(CONF.directory_bundles)); isBUNDLE = true; } catch(e) {} if (isBUNDLE || isRELOAD) { - directories.push(U.combine(F.config['directory-public'])); - directories.push(U.combine(F.config['directory-views'])); + directories.push(U.combine(CONF.directory_public)); + directories.push(U.combine(CONF.directory_views)); } function onFilter(path, isDirectory) { @@ -225,12 +225,12 @@ function runwatching() { var index = fn.indexOf('/', 1); var dir = fn.substring(0, index + 1); - if (dir === F.config['directory-themes']) { + if (dir === CONF.directory_themes) { index = fn.indexOf('/', index + 1); dir = fn.substring(index, fn.indexOf('/', index + 1) + 1); } - return F.config['directory-views'] === dir || F.config['directory-public'] === dir ? fn : ''; + return CONF.directory_views === dir || CONF.directory_public === dir ? fn : ''; } function refresh() { diff --git a/image.js b/image.js index 04bcd26bc..2b8edeb30 100755 --- a/image.js +++ b/image.js @@ -141,7 +141,7 @@ function Image(filename, useImageMagick, width, height) { this.builder = []; this.filename = type === 'string' ? filename : null; this.currentStream = type === 'object' ? filename : null; - this.isIM = useImageMagick == null ? F.config['default-image-converter'] === 'im' : useImageMagick; + this.isIM = useImageMagick == null ? CONF.default_image_converter === 'im' : useImageMagick; this.outputType = type === 'string' ? framework_utils.getExtension(filename) : 'jpg'; this.islimit = false; } @@ -338,7 +338,7 @@ Image.prototype.cmd = function(filenameFrom, filenameTo) { var cmd = ''; if (!self.islimit) { - var tmp = F.config['default-image-consumption']; + var tmp = CONF.default_image_consumption; self.limit('memory', (1500 / 100) * tmp); self.limit('map', (3000 / 100) * tmp); } @@ -365,7 +365,7 @@ Image.prototype.arg = function(first, last) { first && arr.push(first); if (!self.islimit) { - var tmp = F.config['default-image-consumption']; + var tmp = CONF.default_image_consumption; self.limit('memory', (1500 / 100) * tmp); self.limit('map', (3000 / 100) * tmp); } diff --git a/index.js b/index.js index 3c6747aa2..ea8bbeed2 100755 --- a/index.js +++ b/index.js @@ -66,6 +66,7 @@ const REG_WWW = /^www\./i; const REG_TEXTAPPLICATION = /text|application/; const REG_ENCODINGCLEANER = /[;\s]charset=utf-8/g; const REG_SKIPERROR = /epipe|invalid\sdistance/i; +const REG_OLDCONF = /-/g; const REG_UTF8 = /[^\x20-\x7E]+/; const FLAGS_INSTALL = ['get']; const FLAGS_DOWNLOAD = ['get', 'dnscache']; @@ -291,7 +292,7 @@ global.NOSQLSTORAGE = (name) => F.nosql(name).storage; global.NOCOUNTER = global.NOSQLCOUNTER = (name) => F.nosql(name).counter; global.NOMEM = global.NOSQLMEMORY = (name, view) => global.framework_nosql.inmemory(name, view); global.CONFIG = function(name, val) { - return arguments.length === 1 ? F.config[name] : (F.config[name] = val); + return arguments.length === 1 ? CONF[name] : (CONF[name] = val); }; global.CACHE = function(name, value, expire, persistent) { @@ -655,121 +656,135 @@ function Framework() { debug: true, trace: true, - 'trace-console': true, + trace_console: true, name: 'Total.js', version: '1.0.0', author: '', secret: self.syshash, - 'secret-uid': self.syshash.substring(10), + secret_uid: self.syshash.substring(10), 'security.txt': 'Contact: mailto:support@totaljs.com\nContact: https://www.totaljs.com/contact/', - 'etag-version': '', - 'directory-src': '/.src/', - 'directory-bundles': '/bundles/', - 'directory-controllers': '/controllers/', - 'directory-components': '/components/', - 'directory-views': '/views/', - 'directory-definitions': '/definitions/', - 'directory-temp': '/tmp/', - 'directory-models': '/models/', - 'directory-schemas': '/schemas/', - 'directory-operations': '/operations/', - 'directory-resources': '/resources/', - 'directory-public': '/public/', - 'directory-public-virtual': '/app/', - 'directory-modules': '/modules/', - 'directory-source': '/source/', - 'directory-logs': '/logs/', - 'directory-tests': '/tests/', - 'directory-databases': '/databases/', - 'directory-workers': '/workers/', - 'directory-packages': '/packages/', - 'directory-private': '/private/', - 'directory-isomorphic': '/isomorphic/', - 'directory-configs': '/configs/', - 'directory-services': '/services/', - 'directory-themes': '/themes/', + etag_version: '', + directory_src: '/.src/', + directory_bundles: '/bundles/', + directory_controllers: '/controllers/', + directory_components: '/components/', + directory_views: '/views/', + directory_definitions: '/definitions/', + directory_temp: '/tmp/', + directory_models: '/models/', + directory_schemas: '/schemas/', + directory_operations: '/operations/', + directory_resources: '/resources/', + directory_public: '/public/', + directory_public_virtual: '/app/', + directory_modules: '/modules/', + directory_source: '/source/', + directory_logs: '/logs/', + directory_tests: '/tests/', + directory_databases: '/databases/', + directory_workers: '/workers/', + directory_packages: '/packages/', + directory_private: '/private/', + directory_isomorphic: '/isomorphic/', + directory_configs: '/configs/', + directory_services: '/services/', + directory_themes: '/themes/', // all HTTP static request are routed to directory-public - 'static-url': '', - 'static-url-script': '/js/', - 'static-url-style': '/css/', - 'static-url-image': '/img/', - 'static-url-video': '/video/', - 'static-url-font': '/fonts/', - 'static-url-download': '/download/', - 'static-url-components': '/components.', - 'static-accepts': { flac: true, jpg: true, jpeg: true, png: true, gif: true, ico: true, js: true, css: true, txt: true, xml: true, woff: true, woff2: true, otf: true, ttf: true, eot: true, svg: true, zip: true, rar: true, pdf: true, docx: true, xlsx: true, doc: true, xls: true, html: true, htm: true, appcache: true, manifest: true, map: true, ogv: true, ogg: true, mp4: true, mp3: true, webp: true, webm: true, swf: true, package: true, json: true, md: true, m4v: true, jsx: true, heif: true, heic: true }, + static_url: '', + static_url_script: '/js/', + static_url_style: '/css/', + static_url_image: '/img/', + static_url_video: '/video/', + static_url_font: '/fonts/', + static_url_download: '/download/', + static_url_components: '/components.', + static_accepts: { flac: true, jpg: true, jpeg: true, png: true, gif: true, ico: true, js: true, css: true, txt: true, xml: true, woff: true, woff2: true, otf: true, ttf: true, eot: true, svg: true, zip: true, rar: true, pdf: true, docx: true, xlsx: true, doc: true, xls: true, html: true, htm: true, appcache: true, manifest: true, map: true, ogv: true, ogg: true, mp4: true, mp3: true, webp: true, webm: true, swf: true, package: true, json: true, md: true, m4v: true, jsx: true, heif: true, heic: true }, // 'static-accepts-custom': [], - 'default-xpoweredby': 'Total.js', - 'default-layout': 'layout', - 'default-theme': '', - 'default-proxy': '', - 'default-request-maxkeys': 33, + default_xpoweredby: 'Total.js', + default_layout: 'layout', + default_theme: '', + default_proxy: '', + default_request_maxkeys: 33, // default maximum request size / length // default 10 kB - 'default-request-maxlength': 10, - 'default-websocket-maxlength': 2, - 'default-websocket-encodedecode': true, - 'default-maxopenfiles': 0, - 'default-timezone': '', - 'default-root': '', - 'default-response-maxage': '11111111', - 'default-errorbuilder-status': 200, + default_request_maxlength: 10, + default_websocket_maxlength: 2, + default_websocket_encodedecode: true, + default_maxopenfiles: 0, + default_timezone: '', + default_root: '', + default_response_maxage: '11111111', + default_errorbuilder_status: 200, // Default originators - 'default-cors': null, + default_cors: null, // Seconds (2 minutes) - 'default-cors-maxage': 120, + default_cors_maxage: 120, // in milliseconds - 'default-request-timeout': 3000, - 'default-dependency-timeout': 1500, - 'default-restbuilder-timeout': 10000, + default_request_timeout: 3000, + default_dependency_timeout: 1500, + default_restbuilder_timeout: 10000, // otherwise is used ImageMagick (Heroku supports ImageMagick) // gm = graphicsmagick or im = imagemagick - 'default-image-converter': 'gm', - 'default-image-quality': 93, - 'default-image-consumption': 30, - - 'allow-static-files': true, - 'allow-gzip': true, - 'allow-websocket': true, - 'allow-websocket-compression': true, - 'allow-compile': true, - 'allow-compile-script': true, - 'allow-compile-style': true, - 'allow-compile-html': true, - 'allow-compile-build': true, - 'allow-performance': false, - 'allow-custom-titles': false, - 'allow-cache-snapshot': false, - 'allow-cache-cluster': false, - 'allow-debug': false, - 'allow-head': false, - 'allow-filter-errors': true, - 'allow-clear-temp': true, - 'allow-ssc-validation': false, - 'nosql-worker': false, - 'nosql-inmemory': null, // String Array - 'nosql-cleaner': 1440, - 'nosql-logger': true, + default_image_converter: 'gm', + default_image_quality: 93, + default_image_consumption: 30, + + allow_static_files: true, + allow_gzip: true, + allow_websocket: true, + allow_websocket_compression: true, + allow_compile: true, + allow_compile_script: true, + allow_compile_style: true, + allow_compile_html: true, + allow_performance: false, + allow_custom_titles: false, + allow_cache_snapshot: false, + allow_cache_cluster: false, + allow_debug: false, + allow_head: false, + allow_filter_errors: true, + allow_clear_temp: true, + allow_ssc_validation: false, + nosql_worker: false, + nosql_inmemory: null, // String Array + nosql_cleaner: 1440, + nosql_logger: true, logger: false, // Used in F.service() // All values are in minutes - 'default-interval-clear-resources': 20, - 'default-interval-clear-cache': 10, - 'default-interval-clear-dnscache': 120, - 'default-interval-precompile-views': 61, - 'default-interval-websocket-ping': 3, - 'default-interval-uptodate': 5 + default_interval_clear_resources: 20, + default_interval_clear_cache: 10, + default_interval_clear_dnscache: 120, + default_interval_precompile_views: 61, + default_interval_websocket_ping: 3, + default_interval_uptodate: 5, + + set ['mail-address-reply'] (val) { + CONF['mail_address_reply'] = val; + return null; + }, + + set ['mail-address-from'] (val) { + CONF['mail_address_from'] = val; + return null; + }, + + set ['mail-address-copy'] (val) { + CONF['mail_address_copy'] = val; + return null; + } }; global.G = self.global = {}; @@ -1202,9 +1217,9 @@ F.useSMTP = function(smtp, options, callback) { Mail.try(smtp, options, function(err) { if (!err) { - delete F.temporary['mail-settings']; - F.config['mail-smtp'] = smtp; - F.config['mail-smtp-options'] = options; + delete F.temporary.mail_settings; + CONF.mail_smtp = smtp; + CONF.mail_smtp_options = options; } if (callback) @@ -1340,11 +1355,6 @@ global.TABLE = function(name) { return db; }; -global.GRAPHDB = function(name) { - var key = 'gdb_' + name; - return F.databases[key] ? F.databases[key] : F.databases[key] = require('./graphdb').load(name, +(F.config['graphdb.' + name] || 0)); -}; - F.stop = F.kill = function(signal) { if (F.isKilled) @@ -1362,7 +1372,7 @@ F.stop = F.kill = function(signal) { framework_nosql.kill(signal); - F.emit('exit', signal); + EMIT('exit', signal); if (!F.isWorker && process.send) TRY(() => process.send('total:stop')); @@ -1727,8 +1737,8 @@ global.CORS = F.cors = function(url, flags, credentials) { if (!methods.length) methods = 'POST,PUT,GET,DELETE,PATCH,GET,HEAD'.split(','); - if (!origin.length && CONF['default-cors']) - origin = CONF['default-cors']; + if (!origin.length && CONF.default_cors) + origin = CONF.default_cors; route.isWILDCARD = url.lastIndexOf('*') !== -1; @@ -1752,7 +1762,7 @@ global.CORS = F.cors = function(url, flags, credentials) { route.methods = methods.length ? methods : null; route.headers = headers.length ? headers : null; route.credentials = credentials; - route.age = age || F.config['default-cors-maxage']; + route.age = age || CONF.default_cors_maxage; var e = F.routes.cors.findItem(function(item) { return item.hash === route.hash; @@ -2366,7 +2376,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu method += (method ? ',' : '') + 'get'; } - if (F.config['allow-head'] && flags.indexOf('get') !== -1) { + if (CONF.allow_head && flags.indexOf('get') !== -1) { flags.append('head'); method += (method ? ',' : '') + 'head'; } @@ -2430,9 +2440,9 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu r.flags2 = flags_to_object(flags); r.method = method; r.execute = funcExecute; - r.length = (length || F.config['default-request-maxlength']) * 1024; + r.length = (length || CONF.default_request_maxlength) * 1024; r.middleware = middleware; - r.timeout = timeout === undefined ? (isDELAY ? 0 : F.config['default-request-timeout']) : timeout; + r.timeout = timeout === undefined ? (isDELAY ? 0 : CONF.default_request_timeout) : timeout; r.isGET = flags.indexOf('get') !== -1; r.isMULTIPLE = isMULTIPLE; r.isJSON = isJSON; @@ -2496,7 +2506,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu if (isMOBILE) F._request_check_mobile = true; - F.emit('route', 'web', instance); + EMIT('route', 'web', instance); return instance; }; @@ -2642,7 +2652,7 @@ global.MAP = F.map = function(url, filename, filter) { isPackage = true; } else if (c === '=') { if (F.isWindows) - filename = U.combine(F.config['directory-themes'], filename.substring(1)); + filename = U.combine(CONF.directory_themes, filename.substring(1)); else filename = F.path.themes(filename.substring(1)); isPackage = true; @@ -3095,7 +3105,7 @@ global.WEBSOCKET = F.websocket = function(url, funcInitialize, flags, length) { r.onInitialize = funcInitialize; r.protocols = protocols || EMPTYARRAY; r.allow = allow || []; - r.length = (length || F.config['default-websocket-maxlength']) * 1024; + r.length = (length || CONF.default_websocket_maxlength) * 1024; r.isWEBSOCKET = true; r.MEMBER = membertype; r.isJSON = isJSON; @@ -3115,13 +3125,13 @@ global.WEBSOCKET = F.websocket = function(url, funcInitialize, flags, length) { r.type = 'websocket'; F.routes.websockets.push(r); F.initwebsocket && F.initwebsocket(); - F.emit('route', 'websocket', r); + EMIT('route', 'websocket', r); !_controller && F.$routesSort(2); return instance; }; F.initwebsocket = function() { - if (F.routes.websockets.length && F.config['allow-websocket'] && F.server) { + if (F.routes.websockets.length && CONF.allow_websocket && F.server) { F.server.on('upgrade', F.$upgrade); F.initwebsocket = null; } @@ -3238,7 +3248,7 @@ global.FILE = F.file = function(fnValidation, fnExecute, flags) { F.routes.files.push(r); F.routes.files.sort((a, b) => !a.url ? -1 : !b.url ? 1 : a.url.length > b.url.length ? -1 : 1); - F.emit('route', 'file', r); + EMIT('route', 'file', r); F._length_files++; return F; }; @@ -3411,7 +3421,7 @@ F.errorcallback = function(err) { * @return {Framework} */ F.problem = F.wtf = function(message, name, uri, ip) { - F.$events.problem && F.emit('problem', message, name, uri, ip); + F.$events.problem && EMIT('problem', message, name, uri, ip); if (message instanceof framework_builders.ErrorBuilder) message = message.plain(); @@ -3442,7 +3452,7 @@ global.PRINTLN = function(msg) { * @return {Framework} */ F.change = function(message, name, uri, ip) { - F.$events.change && F.emit('change', message, name, uri, ip); + F.$events.change && EMIT('change', message, name, uri, ip); if (message instanceof framework_builders.ErrorBuilder) message = message.plain(); @@ -3470,10 +3480,10 @@ F.change = function(message, name, uri, ip) { */ F.trace = function(message, name, uri, ip) { - if (!F.config.trace) + if (!CONF.trace) return F; - F.$events.trace && F.emit('trace', message, name, uri, ip); + F.$events.trace && EMIT('trace', message, name, uri, ip); if (message instanceof framework_builders.ErrorBuilder) message = message.plain(); @@ -3484,7 +3494,7 @@ F.trace = function(message, name, uri, ip) { var obj = { message: message, name: name, url: uri ? typeof(uri) === 'string' ? uri : Parser.format(uri) : undefined, ip: ip, date: NOW }; F.logger('traces', obj.message, 'url: ' + obj.url, 'source: ' + obj.name, 'ip: ' + obj.ip); - F.config['trace-console'] && console.log(NOW.format('yyyy-MM-dd HH:mm:ss'), '[trace]', message, '|', 'url: ' + obj.url, 'source: ' + obj.name, 'ip: ' + obj.ip); + CONF.trace_console && console.log(NOW.format('yyyy-MM-dd HH:mm:ss'), '[trace]', message, '|', 'url: ' + obj.url, 'source: ' + obj.name, 'ip: ' + obj.ip); if (F.traces) { F.traces.push(obj); @@ -3509,7 +3519,7 @@ global.MODULE = F.module = function(name) { * @return {Framework} */ F.modify = function(fn) { - OBSOLETE('F.modify()', 'This method will be removed from in versions.'); + OBSOLETE('F.modify()', 'This method will be removed in new versions.'); if (!F.modificators) F.modificators = []; F.modificators.push(fn); @@ -3521,18 +3531,18 @@ F.$bundle = function(callback) { var makebundle = function() { require('./bundles').make(function() { - F.directory = HEADERS.workers.cwd = directory = F.path.root(F.config['directory-src']); + F.directory = HEADERS.workers.cwd = directory = F.path.root(CONF.directory_src); callback(); }); }; try { - Fs.statSync(F.path.root(F.config['directory-bundles'])); + Fs.statSync(F.path.root(CONF.directory_bundles)); if (F.$bundling) { makebundle(); return; } else - F.directory = HEADERS.workers.cwd = directory = F.path.root(F.config['directory-src']); + F.directory = HEADERS.workers.cwd = directory = F.path.root(CONF.directory_src); } catch(e) {} callback(); }; @@ -3599,7 +3609,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('modules') !== -1) { operations.push(function(resume) { - dir = U.combine(targetdirectory, isPackage ? '/modules/' : F.config['directory-modules']); + dir = U.combine(targetdirectory, isPackage ? '/modules/' : CONF.directory_modules); arr = []; listing(dir, 0, arr, '.js'); arr.forEach((item) => dependencies.push(next => F.install('module', item.name, item.filename, undefined, undefined, undefined, true, undefined, undefined, next, packageName))); @@ -3609,7 +3619,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('isomorphic') !== -1) { operations.push(function(resume) { - dir = U.combine(targetdirectory, isPackage ? '/isomorphic/' : F.config['directory-isomorphic']); + dir = U.combine(targetdirectory, isPackage ? '/isomorphic/' : CONF.directory_isomorphic); arr = []; listing(dir, 0, arr, '.js'); arr.forEach((item) => dependencies.push(next => F.install('isomorphic', item.name, item.filename, undefined, undefined, undefined, true, undefined, undefined, next, packageName))); @@ -3619,7 +3629,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('packages') !== -1) { operations.push(function(resume) { - dir = U.combine(targetdirectory, isPackage ? '/packages/' : F.config['directory-packages']); + dir = U.combine(targetdirectory, isPackage ? '/packages/' : CONF.directory_packages); arr = []; listing(dir, 0, arr, '.package'); var dirtmp = U.$normalize(dir); @@ -3666,7 +3676,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('models') !== -1) { operations.push(function(resume) { - dir = U.combine(targetdirectory, isPackage ? '/models/' : F.config['directory-models']); + dir = U.combine(targetdirectory, isPackage ? '/models/' : CONF.directory_models); arr = []; listing(dir, 0, arr); arr.forEach((item) => dependencies.push(next => F.install('model', item.name, item.filename, undefined, undefined, undefined, true, undefined, undefined, next, packageName))); @@ -3676,7 +3686,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('schemas') !== -1) { operations.push(function(resume) { - dir = U.combine(targetdirectory, isPackage ? '/schemas/' : F.config['directory-schemas']); + dir = U.combine(targetdirectory, isPackage ? '/schemas/' : CONF.directory_schemas); arr = []; listing(dir, 0, arr); arr.forEach((item) => dependencies.push(next => F.install('schema', item.name, item.filename, undefined, undefined, undefined, true, undefined, undefined, next, packageName))); @@ -3686,7 +3696,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('operations') !== -1) { operations.push(function(resume) { - dir = U.combine(targetdirectory, isPackage ? '/operations/' : F.config['directory-operations']); + dir = U.combine(targetdirectory, isPackage ? '/operations/' : CONF.directory_operations); arr = []; listing(dir, 0, arr); arr.forEach((item) => dependencies.push(next => F.install('operation', item.name, item.filename, undefined, undefined, undefined, true, undefined, undefined, next, packageName))); @@ -3697,7 +3707,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('themes') !== -1) { operations.push(function(resume) { arr = []; - dir = U.combine(targetdirectory, isPackage ? '/themes/' : F.config['directory-themes']); + dir = U.combine(targetdirectory, isPackage ? '/themes/' : CONF.directory_themes); listing(dir, 0, arr, undefined, true); arr.forEach(function(item) { var themeName = item.name; @@ -3713,7 +3723,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('definitions') !== -1) { operations.push(function(resume) { - dir = U.combine(targetdirectory, isPackage ? '/definitions/' : F.config['directory-definitions']); + dir = U.combine(targetdirectory, isPackage ? '/definitions/' : CONF.directory_definitions); arr = []; listing(dir, 0, arr); arr.forEach((item) => dependencies.push(next => F.install('definition', item.name, item.filename, undefined, undefined, undefined, true, undefined, undefined, next, packageName))); @@ -3724,7 +3734,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('controllers') !== -1) { operations.push(function(resume) { arr = []; - dir = U.combine(targetdirectory, isPackage ? '/controllers/' : F.config['directory-controllers']); + dir = U.combine(targetdirectory, isPackage ? '/controllers/' : CONF.directory_controllers); listing(dir, 0, arr); arr.forEach((item) => dependencies.push(next => F.install('controller', item.name, item.filename, undefined, undefined, undefined, true, undefined, undefined, next, packageName))); resume(); @@ -3734,7 +3744,7 @@ F.$load = function(types, targetdirectory, callback, packageName) { if (!types || types.indexOf('components') !== -1) { operations.push(function(resume) { arr = []; - dir = U.combine(targetdirectory, isPackage ? '/components/' : F.config['directory-components']); + dir = U.combine(targetdirectory, isPackage ? '/components/' : CONF.directory_components); listing(dir, 0, arr, '.html'); arr.forEach((item) => dependencies.push(next => F.install('component', item.name, item.filename, undefined, undefined, undefined, undefined, undefined, undefined, next, packageName))); resume(); @@ -3971,8 +3981,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe F.dependencies[key].count++; setTimeout(function() { - F.emit(type + '#' + name); - F.emit('install', type, name); + EMIT(type + '#' + name); + EMIT('install', type, name); F.temporary.ready[type + '#' + name] = NOW; }, 500); @@ -3983,9 +3993,9 @@ F.install = function(type, name, declaration, options, callback, internal, useRe if (type === 'config' || type === 'configuration' || type === 'settings') { F.$configure_configs(declaration instanceof Array ? declaration : declaration.toString().split('\n'), true); setTimeout(function() { - delete F.temporary['mail-settings']; - F.emit(type + '#' + name, F.config); - F.emit('install', type, name); + delete F.temporary.mail_settings; + EMIT(type + '#' + name, CONF); + EMIT('install', type, name); F.temporary.ready[type + '#' + name] = NOW; }, 500); @@ -3999,8 +4009,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe F.$configure_versions(declaration.toString().split('\n')); setTimeout(function() { - F.emit(type + '#' + name); - F.emit('install', type, name); + EMIT(type + '#' + name); + EMIT('install', type, name); F.temporary.ready[type + '#' + name] = NOW; }, 500); @@ -4014,8 +4024,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe F.$configure_workflows(declaration.toString().split('\n')); setTimeout(function() { - F.emit(type + '#' + name); - F.emit('install', type, name); + EMIT(type + '#' + name); + EMIT('install', type, name); F.temporary.ready[type + '#' + name] = NOW; F.consoledebug('install', type + '#' + name); }, 500); @@ -4029,8 +4039,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe F.$configure_sitemap(declaration.toString().split('\n')); setTimeout(function() { - F.emit(type + '#' + name); - F.emit('install', type, name); + EMIT(type + '#' + name); + EMIT('install', type, name); F.temporary.ready[type + '#' + name] = NOW; F.consoledebug('install', type + '#' + name); }, 500); @@ -4053,8 +4063,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe content = parseComponent(internal ? declaration : Fs.readFileSync(declaration).toString(ENCODING), name); if (F.$bundling) { - content.js && Fs.appendFileSync(F.path.temp(temporary + '.js'), hash + (F.config.debug ? component_debug(name, content.js, 'js') : content.js) + hash.substring(0, hash.length - 1)); - content.css && Fs.appendFileSync(F.path.temp(temporary + '.css'), hash + (F.config.debug ? component_debug(name, content.css, 'css') : content.css) + hash.substring(0, hash.length - 1)); + content.js && Fs.appendFileSync(F.path.temp(temporary + '.js'), hash + (CONF.debug ? component_debug(name, content.js, 'js') : content.js) + hash.substring(0, hash.length - 1)); + content.css && Fs.appendFileSync(F.path.temp(temporary + '.css'), hash + (CONF.debug ? component_debug(name, content.css, 'css') : content.css) + hash.substring(0, hash.length - 1)); } if (content.js) @@ -4076,7 +4086,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe F.components.has = true; - var link = F.config['static-url-components']; + var link = CONF.static_url_components; F.components.version = NOW.getTime(); F.components.links = (F.components.js ? ''.format(link, F.components.version) : '') + (F.components.css ? ''.format(link, F.components.version) : ''); @@ -4096,7 +4106,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe _controller = ''; obj.name = name; F.components.instances[name] = obj; - obj && typeof(obj.install) === 'function' && obj.install(options || F.config[_owner], name); + obj && typeof(obj.install) === 'function' && obj.install(options || CONF[_owner], name); } catch(e) { F.error(e, 'F.install(\'component\', \'{0}\')'.format(name)); } @@ -4110,7 +4120,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe obj.$owner = _owner; _controller = ''; F.components.instances[name] = obj; - typeof(obj.install) === 'function' && obj.install(options || F.config[_owner], name); + typeof(obj.install) === 'function' && obj.install(options || CONF[_owner], name); (function(name) { setTimeout(function() { delete require.cache[name]; @@ -4131,12 +4141,12 @@ F.install = function(type, name, declaration, options, callback, internal, useRe tmp = F.components.groups[obj.group] = {}; if (content.js) { - Fs.appendFileSync(F.path.temp(temporary + '.js'), hash + (F.config.debug ? component_debug(name, content.js, 'js') : content.js) + hash.substring(0, hash.length - 1)); + Fs.appendFileSync(F.path.temp(temporary + '.js'), hash + (CONF.debug ? component_debug(name, content.js, 'js') : content.js) + hash.substring(0, hash.length - 1)); tmp.js = true; } if (content.css) { - Fs.appendFileSync(F.path.temp(temporary + '.css'), hash + (F.config.debug ? component_debug(name, content.css, 'css') : content.css) + hash.substring(0, hash.length - 1)); + Fs.appendFileSync(F.path.temp(temporary + '.css'), hash + (CONF.debug ? component_debug(name, content.css, 'css') : content.css) + hash.substring(0, hash.length - 1)); tmp.css = true; } @@ -4145,8 +4155,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe } !skipEmit && setTimeout(function() { - F.emit(type + '#' + name); - F.emit('install', type, name); + EMIT(type + '#' + name); + EMIT('install', type, name); F.temporary.ready[type + '#' + name] = NOW; }, 500); @@ -4159,7 +4169,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe if (type === 'package') { var id = Path.basename(declaration, '.' + U.getExtension(declaration)); - var dir = F.config['directory-temp'][0] === '~' ? Path.join(F.config['directory-temp'].substring(1), id + '.package') : Path.join(F.path.root(), F.config['directory-temp'], id + '.package'); + var dir = CONF.directory_temp[0] === '~' ? Path.join(CONF.directory_temp.substring(1), id + '.package') : Path.join(F.path.root(), CONF.directory_temp, id + '.package'); F.routes.packages[id] = dir; @@ -4171,12 +4181,12 @@ F.install = function(type, name, declaration, options, callback, internal, useRe return; } - F.install('module', id, filename, options || F.config['package#' + name], function(err) { + F.install('module', id, filename, options || CONF['package#' + name], function(err) { setTimeout(function() { - F.emit('module#' + name); - F.emit(type + '#' + name); - F.emit('install', 'module', name); - F.emit('install', type, name); + EMIT('module#' + name); + EMIT(type + '#' + name); + EMIT('install', 'module', name); + EMIT('install', type, name); F.temporary.ready['package#' + name] = NOW; F.temporary.ready['module#' + name] = NOW; }, 500); @@ -4201,11 +4211,11 @@ F.install = function(type, name, declaration, options, callback, internal, useRe obj.$owner = _owner; F.temporary.owners[_owner] = true; - typeof(obj.install) === 'function' && obj.install(options || F.config[_owner], name); + typeof(obj.install) === 'function' && obj.install(options || CONF[_owner], name); !skipEmit && setTimeout(function() { - F.emit(type + '#' + name); - F.emit('install', type, name); + EMIT(type + '#' + name); + EMIT('install', type, name); F.temporary.ready[type + '#' + name] = NOW; }, 500); @@ -4224,14 +4234,14 @@ F.install = function(type, name, declaration, options, callback, internal, useRe if (type === 'package2') { type = type.substring(0, type.length - 1); var id = U.getName(declaration, '.package'); - var dir = F.config['directory-temp'][0] === '~' ? Path.join(F.config['directory-temp'].substring(1), id) : Path.join(F.path.root(), F.config['directory-temp'], id); + var dir = CONF.directory_temp[0] === '~' ? Path.join(CONF.directory_temp.substring(1), id) : Path.join(F.path.root(), CONF.directory_temp, id); var filename = Path.join(dir, 'index.js'); - F.install('module', id.replace(/\.package$/i, ''), filename, options || F.config['package#' + name], function(err) { + F.install('module', id.replace(/\.package$/i, ''), filename, options || CONF['package#' + name], function(err) { setTimeout(function() { - F.emit('module#' + name); - F.emit(type + '#' + name); - F.emit('install', type, name); - F.emit('install', 'module', name); + EMIT('module#' + name); + EMIT(type + '#' + name); + EMIT('install', type, name); + EMIT('install', 'module', name); F.temporary.ready['package#' + name] = NOW; F.temporary.ready['module#' + name] = NOW; }, 500); @@ -4260,8 +4270,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe Fs.writeFileSync(item.filename, framework_internal.modificators(declaration, name)); setTimeout(function() { - F.emit(type + '#' + name); - F.emit('install', type, name); + EMIT(type + '#' + name); + EMIT('install', type, name); F.temporary.ready[type + '#' + name] = NOW; }, 500); @@ -4310,8 +4320,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe callback && callback(null, name); setTimeout(function() { - F.emit(type + '#' + name); - F.emit('install', type, name); + EMIT(type + '#' + name); + EMIT('install', type, name); F.temporary.ready[type + '#' + name] = NOW; }, 500); @@ -4380,8 +4390,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe callback && callback(null, name); setTimeout(function() { - F.emit(type + '#' + name, obj); - F.emit('install', type, name, obj); + EMIT(type + '#' + name, obj); + EMIT('install', type, name, obj); F.temporary.ready[type + '#' + name] = NOW; }, 500); @@ -4479,11 +4489,11 @@ F.install = function(type, name, declaration, options, callback, internal, useRe else F.sources[name] = obj; - typeof(obj.install) === 'function' && obj.install(options || F.config[type + '#' + name], name); + typeof(obj.install) === 'function' && obj.install(options || CONF[type + '#' + name], name); !skipEmit && setTimeout(function() { - F.emit(type + '#' + name, obj); - F.emit('install', type, name, obj); + EMIT(type + '#' + name, obj); + EMIT('install', type, name, obj); F.temporary.ready[type + '#' + name] = NOW; }, 500); @@ -4571,7 +4581,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe } else { F.$configure_configs('@' + name + '/config'); - if (F.config.debug) + if (CONF.debug) F.$configure_configs('@' + name + '/config-debug'); else F.$configure_configs('@' + name + '/config-release'); @@ -4688,7 +4698,7 @@ F.install_prepare = function(noRecursive) { if (keys.length) throw new Error('Dependency exception, missing dependencies for: ' + keys.join(', ').trim()); delete F.temporary.other.dependencies; - }, F.config['default-dependency-timeout']); + }, CONF.default_dependency_timeout); if (!keys.length || noRecursive) return F; @@ -4707,7 +4717,7 @@ F.install_make = function(key, name, obj, options, callback, skipEmit, type) { _controller = routeID; _owner = type + '#' + name.replace(/\.package$/gi, ''); - typeof(obj.install) === 'function' && obj.install(options || F.config[_owner], name); + typeof(obj.install) === 'function' && obj.install(options || CONF[_owner], name); me.processed = true; var id = (type === 'module' ? '#' : '') + name; @@ -4735,8 +4745,8 @@ F.install_make = function(key, name, obj, options, callback, skipEmit, type) { if (!skipEmit) { setTimeout(function() { - F.emit(type + '#' + name, obj); - F.emit('install', type, name, obj); + EMIT(type + '#' + name, obj); + EMIT('install', type, name, obj); F.temporary.ready[type + '#' + name] = NOW; }, 500); } @@ -5005,7 +5015,7 @@ F.uninstall = function(type, name, options, skipEmit, packageName) { F.consoledebug('uninstall', type + '#' + name); } - !skipEmit && F.emit('uninstall', type, name); + !skipEmit && EMIT('uninstall', type, name); return F; }; @@ -5092,7 +5102,7 @@ F.register = function(path) { path = F.path.package(path.substring(1)); else if (c === '=') { if (path[1] === '?') - F.path.themes(F.config['default-theme'] + path.substring(2)); + F.path.themes(CONF.default_theme + path.substring(2)); else path = F.path.themes(path.substring(1)); } @@ -5181,8 +5191,8 @@ F.onMapping = function(url, def, ispublic, encode) { url = '/' + url; var tmp = url; - if (F.config['default-root']) - tmp = tmp.substring(F.config['default-root'].length - 1); + if (CONF.default_root) + tmp = tmp.substring(CONF.default_root.length - 1); // component files if (tmp[1] === '~') { @@ -5429,16 +5439,16 @@ F.onMail = function(address, subject, body, callback, replyTo) { } else message.to(address); - message.from(F.config['mail-address-from'] || '', F.config.name); + message.from(CONF.mail_address_from || '', CONF.name); if (replyTo) message.reply(replyTo); else { - tmp = F.config['mail-address-reply']; + tmp = CONF.mail_address_reply; tmp && tmp.length > 3 && message.reply(tmp); } - tmp = F.config['mail-address-copy']; + tmp = CONF.mail_address_copy; tmp && tmp.length > 3 && message.bcc(tmp); message.$sending = setImmediate(cb => message.send2(cb), callback); @@ -5459,7 +5469,7 @@ F.onMeta = function() { switch (i) { case 0: - builder += '' + (arg + (F.url !== '/' && !F.config['allow-custom-titles'] ? ' - ' + F.config.name : '')) + ''; + builder += '' + (arg + (F.url !== '/' && !CONF.allow_custom_titles ? ' - ' + CONF.name : '')) + ''; break; case 1: builder += ''; @@ -5499,7 +5509,7 @@ global.LOG = F.log = function() { } F.path.verify('logs'); - U.queue('F.log', 5, (next) => Fs.appendFile(U.combine(F.config['directory-logs'], filename + '.log'), time + ' | ' + str + '\n', next)); + U.queue('F.log', 5, (next) => Fs.appendFile(U.combine(CONF.directory_logs, filename + '.log'), time + ' | ' + str + '\n', next)); return F; }; @@ -5521,7 +5531,7 @@ global.LOGGER = F.logger = function() { } F.path.verify('logs'); - U.queue('F.logger', 5, (next) => Fs.appendFile(U.combine(F.config['directory-logs'], arguments[0] + '.log'), dt + ' | ' + str + '\n', next)); + U.queue('F.logger', 5, (next) => Fs.appendFile(U.combine(CONF.directory_logs, arguments[0] + '.log'), dt + ' | ' + str + '\n', next)); return F; }; @@ -5537,7 +5547,7 @@ global.LOGMAIL = F.logmail = function(address, subject, body, callback) { } if (!subject) - subject = F.config.name + ' v' + F.config.version; + subject = CONF.name + ' v' + CONF.version; var body = '' + subject + '
' + (typeof(body) === 'object' ? JSON.stringify(body).escape() : body) + '
'; return F.onMail(address, subject, body, callback); @@ -5574,13 +5584,13 @@ F.usage = function(detailed) { memoryTotal: (memory.heapTotal / 1024 / 1024).floor(2), memoryUsage: (memory.heapUsed / 1024 / 1024).floor(2), memoryRss: (memory.rss / 1024 / 1024).floor(2), - mode: F.config.debug ? 'debug' : 'release', + mode: CONF.debug ? 'debug' : 'release', port: F.port, ip: F.ip, directory: process.cwd() }; - if (F.config['nosql-worker']) + if (CONF.nosql_worker) output.framework.pidnosql = framework_nosql.pid(); var keys = Object.keys(U.queuecache); @@ -5730,7 +5740,7 @@ function compile_merge(res) { var merge = F.routes.merge[uri.pathname]; var filename = merge.filename; - if (!F.config.debug && existsSync(filename)) { + if (!CONF.debug && existsSync(filename)) { var stats = Fs.statSync(filename); var tmp = F.temporary.path[req.$key] = [filename, stats.size, stats.mtime.toUTCString()]; compile_gzip(tmp, function() { @@ -5867,7 +5877,7 @@ function component_debug(filename, value, extension) { F.compile_virtual = function(res) { var req = res.req; - var tmpname = res.options.filename.replace(F.config['directory-public'], F.config['directory-public-virtual']); + var tmpname = res.options.filename.replace(CONF.directory_public, CONF.directory_public_virtual); if (tmpname === res.options.filename) { F.temporary.notfound[req.$key] = true; @@ -5885,13 +5895,13 @@ F.compile_virtual = function(res) { return; } - if (!res.noCompress && COMPRESSIONSPECIAL[req.extension] && F.config['allow-compile'] && !REG_NOCOMPRESS.test(res.options.filename)) { + if (!res.noCompress && COMPRESSIONSPECIAL[req.extension] && CONF.allow_compile && !REG_NOCOMPRESS.test(res.options.filename)) { res.options.filename = tmpname; return compile_file(res); } var tmp = F.temporary.path[req.$key] = [tmpname, size, stats.mtime.toUTCString()]; - if (F.config['allow-gzip'] && COMPRESSION[U.getContentType(req.extension)]) { + if (CONF.allow_gzip && COMPRESSION[U.getContentType(req.extension)]) { compile_gzip(tmp, function() { delete F.temporary.processing[req.$key]; res.$file(); @@ -5919,11 +5929,11 @@ function compile_check(res) { if (e) { - if (!res.noCompress && COMPRESSIONSPECIAL[req.extension] && F.config['allow-compile'] && !REG_NOCOMPRESS.test(res.options.filename)) + if (!res.noCompress && COMPRESSIONSPECIAL[req.extension] && CONF.allow_compile && !REG_NOCOMPRESS.test(res.options.filename)) return compile_file(res); var tmp = F.temporary.path[req.$key] = [res.options.filename, size, stats.mtime.toUTCString()]; - if (F.config['allow-gzip'] && COMPRESSION[U.getContentType(req.extension)]) { + if (CONF.allow_gzip && COMPRESSION[U.getContentType(req.extension)]) { compile_gzip(tmp, function() { res.$file(); delete F.temporary.processing[req.$key]; @@ -5971,10 +5981,10 @@ function compile_content(extension, content, filename) { switch (extension) { case 'js': - return F.config['allow-compile-script'] ? framework_internal.compile_javascript(content, filename) : content; + return CONF.allow_compile_script ? framework_internal.compile_javascript(content, filename) : content; case 'css': - content = F.config['allow-compile-style'] ? framework_internal.compile_css(content, filename) : content; + content = CONF.allow_compile_style ? framework_internal.compile_css(content, filename) : content; var matches = content.match(REG_COMPILECSS); if (matches) { for (var i = 0, length = matches.length; i < length; i++) { @@ -6280,7 +6290,7 @@ F.exists = function(req, res, max, callback) { var filename = F.path.temp(name); var httpcachevalid = false; - RELEASE && (req.headers['if-none-match'] === ETAG + F.config['etag-version']) && (httpcachevalid = true); + RELEASE && (req.headers['if-none-match'] === ETAG + CONF.etag_version) && (httpcachevalid = true); if (F.isProcessed(name) || httpcachevalid) { res.options.filename = filename; @@ -6336,7 +6346,7 @@ F.isProcessing = function(filename) { if (index !== -1) name = name.substring(0, index); - filename = U.combine(F.config['directory-public'], $decodeURIComponent(name)); + filename = U.combine(CONF.directory_public, $decodeURIComponent(name)); return !!F.temporary.processing[filename]; }; @@ -6434,7 +6444,7 @@ global.LOAD = F.load = function(debug, types, pwd) { } F.isWorker = true; - F.config.debug = debug; + CONF.debug = debug; F.isDebug = debug; global.DEBUG = debug; @@ -6459,7 +6469,7 @@ global.LOAD = F.load = function(debug, types, pwd) { F.consoledebug('init'); F.cache.init(); - F.emit('init'); + EMIT('init'); F.$load(types, directory, function() { @@ -6469,8 +6479,8 @@ global.LOAD = F.load = function(debug, types, pwd) { setTimeout(function() { try { - F.emit('load', F); - F.emit('ready', F); + EMIT('load', F); + EMIT('ready', F); } catch (err) { F.error(err, 'F.on("load/ready")'); } @@ -6491,7 +6501,7 @@ global.LOAD = F.load = function(debug, types, pwd) { }, 500); - if (F.config['allow-debug']) { + if (CONF.allow_debug) { F.consoledebug('done'); F.usagesnapshot(); } @@ -6518,10 +6528,10 @@ F.initialize = function(http, debug, options) { var ip = options.ip; var listenpath = options.listenpath; - options.config && U.extend_headers2(F.config, options.config); + options.config && U.extend_headers2(CONF, options.config); - if (options.debug || options['allow-debug']) - F.config['allow-debug'] = true; + if (options.debug || options['allow-debug'] || options.allow_debug) + CONF.allow_debug = true; F.isHTTPS = http.STATUS_CODES === undefined; @@ -6531,7 +6541,7 @@ F.initialize = function(http, debug, options) { if (options.id) F.id = options.id; - F.config.debug = debug; + CONF.debug = debug; F.isDebug = debug; if (options.bundling != null) @@ -6556,21 +6566,21 @@ F.initialize = function(http, debug, options) { F.isTest && F.$configure_configs('config-test', true); F.cache.init(); F.consoledebug('init'); - F.emit('init'); + EMIT('init'); if (!port) { - if (F.config['default-port'] === 'auto') { + if (CONF.default_port === 'auto') { var envPort = +(process.env.PORT || ''); if (!isNaN(envPort)) port = envPort; } else - port = F.config['default-port']; + port = CONF.default_port; } F.port = port || 8000; if (ip !== null) { - F.ip = ip || F.config['default-ip'] || '0.0.0.0'; + F.ip = ip || CONF.default_ip || '0.0.0.0'; if (F.ip === 'null' || F.ip === 'undefined' || F.ip === 'auto') F.ip = null; } else @@ -6579,7 +6589,7 @@ F.initialize = function(http, debug, options) { if (F.ip == null) F.ip = '0.0.0.0'; - !listenpath && (listenpath = F.config['default-listenpath']); + !listenpath && (listenpath = CONF.default_listenpath); F.listenpath = listenpath; if (F.server) { @@ -6602,7 +6612,7 @@ F.initialize = function(http, debug, options) { else F.server = http.createServer(F.listener); - F.config['allow-performance'] && F.server.on('connection', connection_tunning); + CONF.allow_performance && F.server.on('connection', connection_tunning); F.initwebsocket && F.initwebsocket(); F.consoledebug('HTTP listening'); @@ -6626,7 +6636,7 @@ F.initialize = function(http, debug, options) { else listen(); - if (F.config['allow-debug']) { + if (CONF.allow_debug) { F.consoledebug('done'); F.usagesnapshot(); } @@ -6637,8 +6647,8 @@ F.initialize = function(http, debug, options) { setTimeout(function() { try { - F.emit('load', F); - F.emit('ready', F); + EMIT('load', F); + EMIT('ready', F); } catch (err) { F.error(err, 'F.on("load/ready")'); } @@ -6744,8 +6754,8 @@ F.mode = function(http, name, options) { debug = true; break; } - F.config.debug = debug; - F.config.trace = debug; + CONF.debug = debug; + CONF.trace = debug; F.isDebug = debug; global.DEBUG = debug; global.RELEASE = !debug; @@ -6784,7 +6794,7 @@ F.mode = function(http, name, options) { break; } - F.config.trace = debug; + CONF.trace = debug; F.consoledebug('startup'); F.$startup(function() { F.consoledebug('startup (done)'); @@ -6833,7 +6843,7 @@ F.custom = function(mode, http, request, response, options) { break; } - F.config.trace = debug; + CONF.trace = debug; F.consoledebug('startup'); F.$startup(function() { F.consoledebug('startup (done)'); @@ -6850,14 +6860,14 @@ F.console = function() { console.log('Node.js : ' + process.version); console.log('Total.js : v' + F.version_header); console.log('OS : ' + Os.platform() + ' ' + Os.release()); - F.config['nosql-worker'] && console.log('NoSQL PID : ' + framework_nosql.pid()); + CONF.nosql_worker && console.log('NoSQL PID : ' + framework_nosql.pid()); console.log('Memory : ' + memory.heapUsed.filesize(2) + ' / ' + memory.heapTotal.filesize(2)); console.log('===================================================='); - console.log('Name : ' + F.config.name); - console.log('Version : ' + F.config.version); - console.log('Author : ' + F.config.author); + console.log('Name : ' + CONF.name); + console.log('Version : ' + CONF.version); + console.log('Author : ' + CONF.author); console.log('Date : ' + NOW.format('yyyy-MM-dd HH:mm:ss')); - console.log('Mode : ' + (F.config.debug ? 'debug' : 'release')); + console.log('Mode : ' + (CONF.debug ? 'debug' : 'release')); console.log('===================================================='); console.log('Directory : ' + process.cwd()); console.log('node_modules : ' + PATHMODULES); @@ -6875,7 +6885,7 @@ F.usagesnapshot = function(filename) { F.consoledebug = function() { - if (!F.config['allow-debug']) + if (!CONF.allow_debug) return F; var arr = [new Date().format('yyyy-MM-dd HH:mm:ss'), '--------->']; @@ -6890,10 +6900,10 @@ F.consoledebug = function() { * @return {Framework} */ F.reconnect = function() { - if (F.config['default-port'] !== undefined) - F.port = F.config['default-port']; - if (F.config['default-ip'] !== undefined) - F.ip = F.config['default-ip']; + if (CONF.default_port !== undefined) + F.port = CONF.default_port; + if (CONF.default_ip !== undefined) + F.ip = CONF.default_ip; F.server.close(() => F.server.listen(F.port, F.ip)); return F; }; @@ -6937,8 +6947,8 @@ F.service = function(count) { F.temporary.notfound = {}; // every 7 minutes (default) service clears static cache - if (count % F.config['default-interval-clear-cache'] === 0) { - F.$events.clear && F.emit('clear', 'temporary', F.temporary); + if (count % CONF.default_interval_clear_cache === 0) { + F.$events.clear && EMIT('clear', 'temporary', F.temporary); F.temporary.path = {}; F.temporary.range = {}; F.temporary.views = {}; @@ -6953,24 +6963,24 @@ F.service = function(count) { F.databases[key] && F.databases[key].inmemorylastusage < dt && F.databases[key].release(); releasegc = true; - F.config['allow-debug'] && F.consoledebug('clear temporary cache'); + CONF.allow_debug && F.consoledebug('clear temporary cache'); } // every 61 minutes (default) services precompile all (installed) views - if (count % F.config['default-interval-precompile-views'] === 0) { + if (count % CONF.default_interval_precompile_views === 0) { for (var key in F.routes.views) { var item = F.routes.views[key]; F.install('view', key, item.url, null); } } - if (count % F.config['default-interval-clear-dnscache'] === 0) { - F.$events.clear && F.emit('clear', 'dns'); + if (count % CONF.default_interval_clear_dnscache === 0) { + F.$events.clear && EMIT('clear', 'dns'); U.clearDNS(); - F.config['allow-debug'] && F.consoledebug('clear DNS cache'); + CONF.allow_debug && F.consoledebug('clear DNS cache'); } - var ping = F.config['default-interval-websocket-ping']; + var ping = CONF.default_interval_websocket_ping; if (ping > 0 && count % ping === 0) { var has = false; for (var item in F.connections) { @@ -6981,10 +6991,10 @@ F.service = function(count) { has = true; } } - has && F.config['allow-debug'] && F.consoledebug('ping websocket connections'); + has && CONF.allow_debug && F.consoledebug('ping websocket connections'); } - if (F.uptodates && (count % F.config['default-interval-uptodate'] === 0) && F.uptodates.length) { + if (F.uptodates && (count % CONF.default_interval_uptodate === 0) && F.uptodates.length) { var hasUpdate = false; F.uptodates.wait(function(item, next) { @@ -6995,10 +7005,10 @@ F.service = function(count) { item.count++; setTimeout(function() { - F.config['allow-debug'] && F.consoledebug('uptodate', item.type + '#' + item.url); + CONF.allow_debug && F.consoledebug('uptodate', item.type + '#' + item.url); F.install(item.type, item.url, item.options, function(err, name, skip) { - F.config['allow-debug'] && F.consoledebug('uptodate', item.type + '#' + item.url + ' (done)'); + CONF.allow_debug && F.consoledebug('uptodate', item.type + '#' + item.url + ' (done)'); if (skip) return next(); @@ -7009,7 +7019,7 @@ F.service = function(count) { } else { hasUpdate = true; item.name = name; - F.$events.uptodate && F.emit('uptodate', item.type, name); + F.$events.uptodate && EMIT('uptodate', item.type, name); } item.callback && item.callback(err, name); @@ -7031,17 +7041,17 @@ F.service = function(count) { } // every 20 minutes (default) service clears resources - if (count % F.config['default-interval-clear-resources'] === 0) { - F.$events.clear && F.emit('clear', 'resources'); + if (count % CONF.default_interval_clear_resources === 0) { + F.$events.clear && EMIT('clear', 'resources'); F.resources = {}; releasegc = true; - F.config['allow-debug'] && F.consoledebug('clear resources'); + CONF.allow_debug && F.consoledebug('clear resources'); } // Update expires date count % 1000 === 0 && (DATE_EXPIRES = NOW.add('y', 1).toUTCString()); - if (count % F.config['nosql-cleaner'] === 0 && F.config['nosql-cleaner']) { + if (count % CONF.nosql_cleaner === 0 && CONF.nosql_cleaner) { keys = Object.keys(F.databasescleaner); keys.wait(function(item, next) { if (item[0] === '$') @@ -7051,16 +7061,16 @@ F.service = function(count) { }); } - F.$events.service && F.emit('service', count); + F.$events.service && EMIT('service', count); - if (F.config['allow-debug']) { + if (CONF.allow_debug) { F.consoledebug('service ({0}x)'.format(count)); F.usagesnapshot(); } releasegc && global.gc && setTimeout(function() { global.gc(); - F.config['allow-debug'] && F.consoledebug('gc()'); + CONF.allow_debug && F.consoledebug('gc()'); }, 1000); // Run schedules @@ -7084,7 +7094,7 @@ F.service = function(count) { else F.schedules.splice(index, 1); - F.config['allow-debug'] && F.consoledebug('schedule', schedule.id); + CONF.allow_debug && F.consoledebug('schedule', schedule.id); schedule.fn.call(F); } @@ -7114,7 +7124,7 @@ F.listener = function(req, res) { req.uri = framework_internal.parseURI(req); F.stats.request.request++; - F.$events.request && F.emit('request', req, res); + F.$events.request && EMIT('request', req, res); if (F._request_check_redirect) { var redirect = F.routes.redirects[req.$protocol + '://' + req.host]; @@ -7133,7 +7143,7 @@ F.listener = function(req, res) { req.xhr = headers['x-requested-with'] === 'XMLHttpRequest'; res.success = false; req.user = req.session = null; - req.isStaticFile = F.config['allow-static-files'] && U.isStaticFile(req.uri.pathname); + req.isStaticFile = CONF.allow_static_files && U.isStaticFile(req.uri.pathname); if (req.isStaticFile) req.extension = U.getExtension(req.uri.pathname); @@ -7254,7 +7264,7 @@ F.$requestcontinue = function(req, res, headers) { flags.push('sse'); } - if (F.config.debug) { + if (CONF.debug) { req.$flags += 'h'; flags.push('debug'); } @@ -7277,7 +7287,7 @@ F.$requestcontinue = function(req, res, headers) { } req.flags = flags; - F.$events['request-begin'] && F.emit('request-begin', req, res); + F.$events['request-begin'] && EMIT('request-begin', req, res); var isCORS = (F._length_cors || F.routes.corsall) && req.headers.origin != null; @@ -7427,13 +7437,13 @@ F.$cors = function(req, res, fn, arg) { F.temporary.other[key] = stop ? 2 : 1; } - } else if (CONF['default-cors']) { + } else if (CONF.default_cors) { key = headers.origin; if (F.temporary.other[key]) { stop = F.temporary.other[key] === 2; } else { origin = key.toLowerCase().substring(key.indexOf('/') + 2); - stop = origin !== headers.host && CONF['default-cors'].indexOf(origin) === -1; + stop = origin !== headers.host && CONF.default_cors.indexOf(origin) === -1; F.temporary.other[key] = stop ? 2 : 1; } } @@ -7467,7 +7477,7 @@ F.$cors = function(req, res, fn, arg) { if (stop) { fn = null; - F.$events['request-end'] && F.emit('request-end', req, res); + F.$events['request-end'] && EMIT('request-end', req, res); F.reqstats(false, false); F.stats.request.blocked++; res.writeHead(404); @@ -7479,7 +7489,7 @@ F.$cors = function(req, res, fn, arg) { return fn(req, res, arg); fn = null; - F.$events['request-end'] && F.emit('request-end', req, res); + F.$events['request-end'] && EMIT('request-end', req, res); F.reqstats(false, false); res.writeHead(200); res.end(); @@ -7506,7 +7516,7 @@ F.$upgrade = function(req, socket, head) { req.uri = framework_internal.parseURI(req); - F.$events.websocket && F.emit('websocket', req, socket, head); + F.$events.websocket && EMIT('websocket', req, socket, head); F.stats.request.websocket++; req.session = null; @@ -7628,7 +7638,7 @@ global.MODEL = F.model = function(name) { var obj = F.models[name]; if (obj || obj === null) return obj; - var filename = U.combine(F.config['directory-models'], name + '.js'); + var filename = U.combine(CONF.directory_models, name + '.js'); existsSync(filename) && F.install('model', name, filename, undefined, undefined, undefined, true); return F.models[name] || null; }; @@ -7643,7 +7653,7 @@ global.INCLUDE = global.SOURCE = F.source = function(name, options, callback) { var obj = F.sources[name]; if (obj || obj === null) return obj; - var filename = U.combine(F.config['directory-source'], name + '.js'); + var filename = U.combine(CONF.directory_source, name + '.js'); existsSync(filename) && F.install('source', name, filename, options, callback, undefined, true); return F.sources[name] || null; }; @@ -7725,8 +7735,8 @@ global.VIEW = F.view = function(name, model, layout, repository, language) { if (theme) { controller.themeName = theme; name = prepare_viewname(name); - } else if (this.onTheme) - controller.themeName = this.onTheme(controller); + } else if (F.onTheme) + controller.themeName = F.onTheme(controller); else controller.themeName = undefined; @@ -7787,7 +7797,7 @@ F.clear = function(callback, isInit) { var plus = F.id ? 'i-' + F.id + '_' : ''; if (isInit) { - if (!F.config['allow-clear-temp']) { + if (!CONF.allow_clear_temp) { if (F.$bundling) { // clears only JS and CSS files U.ls(dir, function(files) { @@ -7934,7 +7944,7 @@ F.encrypt = function(value, key, isUnique) { else if (type === 'object') value = JSON.stringify(value); - return value.encrypt(F.config.secret + '=' + key, isUnique); + return value.encrypt(CONF.secret + '=' + key, isUnique); }; /** @@ -7955,7 +7965,7 @@ F.decrypt = function(value, key, jsonConvert) { if (typeof(jsonConvert) !== 'boolean') jsonConvert = true; - var response = (value || '').decrypt(F.config.secret + '=' + key); + var response = (value || '').decrypt(CONF.secret + '=' + key); return response ? (jsonConvert ? (response.isJSON() ? response.parseJSON(true) : null) : response) : null; }; @@ -7973,7 +7983,7 @@ F.hash = function(type, value, salt) { if (typeof(salt) === 'string') plus = salt; else if (salt !== false) - plus = (F.config.secret || ''); + plus = (CONF.secret || ''); hash.update(value.toString() + plus, ENCODING); return hash.digest('hex'); @@ -8010,7 +8020,7 @@ F.resource = function(name, key) { } } - var filename = U.combine(F.config['directory-resources'], name + '.resource'); + var filename = U.combine(CONF.directory_resources, name + '.resource'); var empty = false; if (existsSync(filename)) body += (body ? '\n' : '') + Fs.readFileSync(filename).toString(ENCODING); @@ -8595,7 +8605,7 @@ F.$configure_configs = function(arr, rewrite) { if (!arr) { var filenameA = U.combine('/', 'config'); - var filenameB = U.combine('/', 'config-' + (F.config.debug ? 'debug' : 'release')); + var filenameB = U.combine('/', 'config-' + (CONF.debug ? 'debug' : 'release')); arr = []; @@ -8626,8 +8636,8 @@ F.$configure_configs = function(arr, rewrite) { } var done = function() { - process.title = 'total: ' + F.config.name.removeDiacritics().toLowerCase().replace(REG_EMPTY, '-').substring(0, 8); - F.isVirtualDirectory = existsSync(U.combine(F.config['directory-public-virtual'])); + process.title = 'total: ' + CONF.name.removeDiacritics().toLowerCase().replace(REG_EMPTY, '-').substring(0, 8); + F.isVirtualDirectory = existsSync(U.combine(CONF.directory_public_virtual)); }; if (!(arr instanceof Array) || !arr.length) { @@ -8671,51 +8681,80 @@ F.$configure_configs = function(arr, rewrite) { switch (name) { case 'secret': case 'secret-uid': + case 'secret_uid': + name = name.replace(REG_OLDCONF, '_'); obj[name] = value; break; case 'default-request-length': - OBSOLETE(name, 'You need to use "default-request-maxlength"'); - obj['default-request-maxlength'] = U.parseInt(value); + OBSOLETE(name, 'You need to use "default_request_maxlength"'); + obj.default_request_maxlength = U.parseInt(value); break; case 'default-websocket-request-length': - OBSOLETE(name, 'You need to use "default-websocket-maxlength"'); - obj['default-websocket-maxlength'] = U.parseInt(value); + OBSOLETE(name, 'You need to use "default_websocket_maxlength"'); + obj.default_websocket_maxlength = U.parseInt(value); break; case 'default-maximum-file-descriptors': - OBSOLETE(name, 'You need to use "default-maxopenfiles"'); - obj['default-maxopenfiles'] = U.parseInt(value); + OBSOLETE(name, 'You need to use "default_maxopenfiles"'); + obj.default_maxopenfiles = U.parseInt(value); break; - case 'default-cors-maxage': - case 'default-request-timeout': - case 'default-request-maxlength': - case 'default-request-maxkeys': - case 'default-websocket-maxlength': - case 'default-interval-clear-cache': - case 'default-interval-clear-resources': - case 'default-interval-precompile-views': - case 'default-interval-uptodate': - case 'default-interval-websocket-ping': - case 'default-interval-clear-dnscache': - case 'default-dependency-timeout': - case 'default-restbuilder-timeout': - case 'nosql-cleaner': + case 'default-cors-maxage': // old + case 'default-request-timeout': // old + case 'default-request-maxlength': // old + case 'default-request-maxkeys': // old + case 'default-websocket-maxlength': // old + case 'default-interval-clear-cache': // old + case 'default-interval-clear-resources': // old + case 'default-interval-precompile-views': // old + case 'default-interval-uptodate': // old + case 'default-interval-websocket-ping': // old + case 'default-interval-clear-dnscache': // old + case 'default-dependency-timeout': // old + case 'default-restbuilder-timeout': // old + case 'nosql-cleaner': // old + case 'default-errorbuilder-status': // old + case 'default-maxopenfiles': // old + case 'default_maxopenfiles': + case 'default_errorbuilder_status': + case 'default_cors_maxage': + case 'default_request_timeout': + case 'default_request_maxlength': + case 'default_request_maxkeys': + case 'default_websocket_maxlength': + case 'default_interval_clear_cache': + case 'default_interval_clear_resources': + case 'default_interval_precompile_views': + case 'default_interval_uptodate': + case 'default_interval_websocket_ping': + case 'default_interval_clear_dnscache': + case 'default_dependency_timeout': + case 'default_restbuilder_timeout': + case 'nosql_cleaner': + name = obsolete_config(name); obj[name] = U.parseInt(value); break; - case 'default-image-consumption': - case 'default-image-quality': + case 'default-image-consumption': // old + case 'default-image-quality': // old + case 'default_image_consumption': + case 'default_image_quality': + name = obsolete_config(name); obj[name] = U.parseInt(value.replace(/%|\s/g, '')); break; - case 'static-accepts-custom': + case 'static-accepts-custom': // old + case 'static_accepts_custom': accepts = value.replace(REG_ACCEPTCLEANER, '').split(','); break; - case 'default-root': + case 'default-root': // old + case 'default_root': + name = obsolete_config(name); if (value) obj[name] = U.path(value); break; - case 'static-accepts': + case 'static-accepts': // old + case 'static_accepts': + name = obsolete_config(name); obj[name] = {}; tmp = value.replace(REG_ACCEPTCLEANER, '').split(','); for (var j = 0; j < tmp.length; j++) @@ -8730,7 +8769,7 @@ F.$configure_configs = function(arr, rewrite) { case 'mail.address.reply': if (name === 'mail.address.bcc') - tmp = 'mail-address-copy'; + tmp = 'mail_address_copy'; else tmp = name.replace(/\./g, '-'); @@ -8738,7 +8777,9 @@ F.$configure_configs = function(arr, rewrite) { obj[tmp] = value; break; - case 'default-cors': + case 'default-cors': // old + case 'default_cors': + name = obsolete_config(name); value = value.replace(/,/g, ' ').split(' '); tmp = []; for (var j = 0; j < value.length; j++) { @@ -8755,43 +8796,73 @@ F.$configure_configs = function(arr, rewrite) { break; case 'allow-handle-static-files': - OBSOLETE('config["allow-handle-static-files"]', 'The key has been renamed to "allow-static-files"'); - obj['allow-static-files'] = true; + OBSOLETE('config["allow-handle-static-files"]', 'The key has been renamed to "allow_static_files"'); + obj.allow_static_files = true; break; case 'disable-clear-temporary-directory': - OBSOLETE('disable-clear-temporary-directory', 'You need to use "allow-clear-temp : true|false"'); - obj['allow-clear-temp'] = !(value.toLowerCase() === 'true' || value === '1' || value === 'on'); + OBSOLETE('disable-clear-temporary-directory', 'You need to use "allow_clear_temp : true|false"'); + obj.allow_clear_temp = !(value.toLowerCase() === 'true' || value === '1' || value === 'on'); break; case 'disable-strict-server-certificate-validation': - OBSOLETE('disable-strict-server-certificate-validation', 'You need to use "allow-ssc-validation : true|false"'); - obj['allow-ssc-validation'] = !(value.toLowerCase() === 'true' || value === '1' || value === 'on'); + OBSOLETE('disable-strict-server-certificate-validation', 'You need to use "allow_ssc_validation : true|false"'); + obj.allow_ssc_validation = !(value.toLowerCase() === 'true' || value === '1' || value === 'on'); break; - case 'allow-compile-html': - case 'allow-compile-script': - case 'allow-compile-style': - case 'allow-ssc-validation': - case 'allow-debug': - case 'allow-gzip': - case 'allow-performance': - case 'allow-static-files': - case 'allow-websocket': - case 'allow-clear-temp': - case 'allow-cache-snapshot': + case 'allow-compile': // old + case 'allow-compile-html': // old + case 'allow-compile-script': // old + case 'allow-compile-style': // old + case 'allow-ssc-validation': // old + case 'allow-debug': // old + case 'allow-gzip': // old + case 'allow-head': // old + case 'allow-performance': // old + case 'allow-static-files': // old + case 'allow-websocket': // old + case 'allow-websocket-compression': // old + case 'allow-clear-temp': // old + case 'allow-cache-snapshot': // old + case 'allow-cache-cluster': // old + case 'allow-custom-titles': // old + case 'nosql-worker': // old + case 'nosql-logger': // old + case 'allow-filter-errors': // old + case 'default-websocket-encodedecode': // old + case 'allow_compile': + case 'allow_compile_html': + case 'allow_compile_script': + case 'allow_compile_style': + case 'allow_ssc_validation': + case 'allow_debug': + case 'allow_gzip': + case 'allow_head': + case 'allow_performance': + case 'allow_static_files': + case 'allow_websocket': + case 'allow_websocket_compression': + case 'allow_clear_temp': + case 'allow_cache_snapshot': + case 'allow_cache_cluster': + case 'allow_filter_errors': + case 'allow_custom_titles': case 'trace': - case 'nosql-worker': - case 'nosql-logger': + case 'nosql_worker': + case 'nosql_logger': + case 'default_websocket_encodedecode': + name = obsolete_config(name); obj[name] = value.toLowerCase() === 'true' || value === '1' || value === 'on'; break; - case 'nosql-inmemory': + case 'nosql-inmemory': // old + case 'nosql_inmemory': + name = obsolete_config(name); obj[name] = typeof(value) === 'string' ? value.split(',').trim() : value instanceof Array ? value : null; break; case 'allow-compress-html': - obj['allow-compile-html'] = value.toLowerCase() === 'true' || value === '1' || value === 'on'; + obj.allow_compile_html = value.toLowerCase() === 'true' || value === '1' || value === 'on'; break; case 'version': @@ -8802,6 +8873,34 @@ F.$configure_configs = function(arr, rewrite) { obj[name] = value ? value.split(',').trim().join('\n') : ''; break; + // backward compatibility + case 'mail-smtp': // old + case 'mail-smtp-options': // old + case 'mail-address-from': // old + case 'mail-address-copy': // old + case 'mail-address-bcc': // old + case 'mail-address-reply': // old + case 'default-image-converter': // old + case 'static-url': // old + case 'static-url-script': // old + case 'static-url-style': // old + case 'static-url-image': // old + case 'static-url-video': // old + case 'static-url-font': // old + case 'static-url-download': // old + case 'static-url-components': // old + case 'default-xpoweredby': // old + case 'default-layout': // old + case 'default-theme': // old + case 'default-proxy': // old + case 'default-timezone': // old + case 'default-response-maxage': // old + case 'default-errorbuilder-resource-name': // old + case 'default-errorbuilder-resource-prefix': // old + name = obsolete_config(name); + obj[name] = value; + break; + default: if (subtype === 'string') @@ -8834,44 +8933,44 @@ F.$configure_configs = function(arr, rewrite) { } } - U.extend(F.config, obj, rewrite); + U.extend(CONF, obj, rewrite); - if (!F.config['secret-uid']) - F.config['secret-uid'] = (F.config.name).crc32(true).toString(); + if (!CONF.secret_uid) + CONF.secret_uid = (CONF.name).crc32(true).toString(); - var tmp = F.config['mail-smtp-options']; + var tmp = CONF.mail_smtp_options; if (typeof(tmp) === 'string' && tmp) { tmp = new Function('return ' + tmp)(); - F.config['mail-smtp-options'] = tmp; + CONF.mail_smtp_options = tmp; } - if (!F.config['directory-temp']) - F.config['directory-temp'] = '~' + U.path(Path.join(Os.tmpdir(), 'totaljs' + F.directory.hash())); + if (!CONF.directory_temp) + CONF.directory_temp = '~' + U.path(Path.join(Os.tmpdir(), 'totaljs' + F.directory.hash())); - if (!F.config['etag-version']) - F.config['etag-version'] = F.config.version.replace(/\.|\s/g, ''); + if (!CONF.etag_version) + CONF.etag_version = CONF.version.replace(/\.|\s/g, ''); - if (F.config['default-timezone']) - process.env.TZ = F.config['default-timezone']; + if (CONF.default_timezone) + process.env.TZ = CONF.default_timezone; - F.config['nosql-worker'] && framework_nosql.worker(); - F.config['nosql-inmemory'] && F.config['nosql-inmemory'].forEach(n => framework_nosql.inmemory(n)); - accepts && accepts.length && accepts.forEach(accept => F.config['static-accepts'][accept] = true); + CONF.nosql_worker && framework_nosql.worker(); + CONF.nosql_inmemory && CONF.nosql_inmemory.forEach(n => framework_nosql.inmemory(n)); + accepts && accepts.length && accepts.forEach(accept => CONF.static_accepts[accept] = true); - if (F.config['allow-ssc-validation'] === false) + if (CONF.allow_ssc_validation === false) process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; - if (F.config['allow-performance']) + if (CONF.allow_performance) http.globalAgent.maxSockets = 9999; - QUERYPARSEROPTIONS.maxKeys = F.config['default-request-maxkeys'] || 33; + QUERYPARSEROPTIONS.maxKeys = CONF.default_request_maxkeys || 33; - var xpowered = F.config['default-xpoweredby']; + var xpowered = CONF.default_xpoweredby; Object.keys(HEADERS).forEach(function(key) { Object.keys(HEADERS[key]).forEach(function(subkey) { if (RELEASE && subkey === 'Cache-Control') - HEADERS[key][subkey] = HEADERS[key][subkey].replace(/max-age=\d+/, 'max-age=' + F.config['default-response-maxage']); + HEADERS[key][subkey] = HEADERS[key][subkey].replace(/max-age=\d+/, 'max-age=' + CONF.default_response_maxage); if (subkey === 'X-Powered-By') { if (xpowered) HEADERS[key][subkey] = xpowered; @@ -8881,19 +8980,27 @@ F.$configure_configs = function(arr, rewrite) { }); }); - IMAGEMAGICK = F.config['default-image-converter'] === 'im'; + IMAGEMAGICK = CONF.default_image_converter === 'im'; done(); - F.emit('configure', F.config); + EMIT('configure', CONF); return F; }; +function obsolete_config(name) { + if (name.indexOf('-') === -1) + return name; + var n = name.replace(REG_OLDCONF, '_'); + OBSOLETE('config[\'' + name + '\']', 'Replace key "{0}" to "{1}" in your config file'.format(name, n)); + return n; +} + /** * Create URL: JavaScript (according to config['static-url-script']) * @param {String} name * @return {String} */ F.routeScript = function(name, theme) { - return F.$routeStatic(name, F.config['static-url-script'], theme); + return F.$routeStatic(name, CONF.static_url_script, theme); }; /** @@ -8902,27 +9009,27 @@ F.routeScript = function(name, theme) { * @return {String} */ F.routeStyle = function(name, theme) { - return F.$routeStatic(name, F.config['static-url-style'], theme); + return F.$routeStatic(name, CONF.static_url_style, theme); }; F.routeImage = function(name, theme) { - return F.$routeStatic(name, F.config['static-url-image'], theme); + return F.$routeStatic(name, CONF.static_url_image, theme); }; F.routeVideo = function(name, theme) { - return F.$routeStatic(name, F.config['static-url-video'], theme); + return F.$routeStatic(name, CONF.static_url_video, theme); }; F.routeFont = function(name, theme) { - return F.$routeStatic(name, F.config['static-url-font'], theme); + return F.$routeStatic(name, CONF.static_url_font, theme); }; F.routeDownload = function(name, theme) { - return F.$routeStatic(name, F.config['static-url-download'], theme); + return F.$routeStatic(name, CONF.static_url_download, theme); }; F.routeStatic = function(name, theme) { - return F.$routeStatic(name, F.config['static-url'], theme); + return F.$routeStatic(name, CONF.static_url, theme); }; F.$routeStatic = function(name, directory, theme) { @@ -8941,7 +9048,7 @@ F.$routeStatic = function(name, directory, theme) { if (index !== -1) { theme = name.substring(1, index); if (theme === '?') { - theme = F.config['default-theme']; + theme = CONF.default_theme; name = name.substring(index); } else name = name.substring(index + 1); @@ -9155,7 +9262,7 @@ F.lookup_websocket = function(req, url, membertype) { F.accept = function(extension, contentType) { if (extension[0] === '.') extension = extension.substring(1); - F.config['static-accepts'][extension] = true; + CONF.static_accepts[extension] = true; contentType && U.setContentType(extension, contentType); return F; }; @@ -9196,7 +9303,7 @@ F.worker = function(name, id, timeout, args) { if (fork) return fork; - var filename = name[0] === '@' ? F.path.package(name.substring(1)) : U.combine(F.config['directory-workers'], name); + var filename = name[0] === '@' ? F.path.package(name.substring(1)) : U.combine(CONF.directory_workers, name); if (!args) args = []; @@ -9380,7 +9487,7 @@ FrameworkPath.prototype.verify = function(name) { var prop = '$directory-' + name; if (F.temporary.path[prop]) return F; - var directory = F.config['directory-' + name] || name; + var directory = CONF['directory_' + name] || name; var dir = U.combine(directory); !existsSync(dir) && Fs.mkdirSync(dir); F.temporary.path[prop] = true; @@ -9435,43 +9542,43 @@ FrameworkPath.prototype.exists = function(path, callback) { }; FrameworkPath.prototype.public = function(filename) { - return U.combine(F.config['directory-public'], filename); + return U.combine(CONF.directory_public, filename); }; FrameworkPath.prototype.public_cache = function(filename) { var key = 'public_' + filename; var item = F.temporary.other[key]; - return item ? item : F.temporary.other[key] = U.combine(F.config['directory-public'], filename); + return item ? item : F.temporary.other[key] = U.combine(CONF.directory_public, filename); }; FrameworkPath.prototype.private = function(filename) { - return U.combine(F.config['directory-private'], filename); + return U.combine(CONF.directory_private, filename); }; FrameworkPath.prototype.isomorphic = function(filename) { - return U.combine(F.config['directory-isomorphic'], filename); + return U.combine(CONF.directory_isomorphic, filename); }; FrameworkPath.prototype.configs = function(filename) { - return U.combine(F.config['directory-configs'], filename); + return U.combine(CONF.directory_configs, filename); }; FrameworkPath.prototype.virtual = function(filename) { - return U.combine(F.config['directory-public-virtual'], filename); + return U.combine(CONF.directory_public_virtual, filename); }; FrameworkPath.prototype.logs = function(filename) { this.verify('logs'); - return U.combine(F.config['directory-logs'], filename); + return U.combine(CONF.directory_logs, filename); }; FrameworkPath.prototype.models = function(filename) { - return U.combine(F.config['directory-models'], filename); + return U.combine(CONF.directory_models, filename); }; FrameworkPath.prototype.temp = function(filename) { this.verify('temp'); - return U.combine(F.config['directory-temp'], filename); + return U.combine(CONF.directory_temp, filename); }; FrameworkPath.prototype.temporary = function(filename) { @@ -9479,52 +9586,52 @@ FrameworkPath.prototype.temporary = function(filename) { }; FrameworkPath.prototype.views = function(filename) { - return U.combine(F.config['directory-views'], filename); + return U.combine(CONF.directory_views, filename); }; FrameworkPath.prototype.workers = function(filename) { - return U.combine(F.config['directory-workers'], filename); + return U.combine(CONF.directory_workers, filename); }; FrameworkPath.prototype.databases = function(filename) { this.verify('databases'); - return U.combine(F.config['directory-databases'], filename); + return U.combine(CONF.directory_databases, filename); }; FrameworkPath.prototype.modules = function(filename) { - return U.combine(F.config['directory-modules'], filename); + return U.combine(CONF.directory_modules, filename); }; FrameworkPath.prototype.controllers = function(filename) { - return U.combine(F.config['directory-controllers'], filename); + return U.combine(CONF.directory_controllers, filename); }; FrameworkPath.prototype.definitions = function(filename) { - return U.combine(F.config['directory-definitions'], filename); + return U.combine(CONF.directory_definitions, filename); }; FrameworkPath.prototype.tests = function(filename) { - return U.combine(F.config['directory-tests'], filename); + return U.combine(CONF.directory_tests, filename); }; FrameworkPath.prototype.resources = function(filename) { - return U.combine(F.config['directory-resources'], filename); + return U.combine(CONF.directory_resources, filename); }; FrameworkPath.prototype.services = function(filename) { - return U.combine(F.config['directory-services'], filename); + return U.combine(CONF.directory_services, filename); }; FrameworkPath.prototype.packages = function(filename) { - return U.combine(F.config['directory-packages'], filename); + return U.combine(CONF.directory_packages, filename); }; FrameworkPath.prototype.themes = function(filename) { - return U.combine(F.config['directory-themes'], filename); + return U.combine(CONF.directory_themes, filename); }; FrameworkPath.prototype.components = function(filename) { - return U.combine(F.config['directory-components'], filename); + return U.combine(CONF.directory_components, filename); }; FrameworkPath.prototype.root = function(filename) { @@ -9542,7 +9649,7 @@ FrameworkPath.prototype.package = function(name, filename) { } } - var tmp = F.config['directory-temp']; + var tmp = CONF.directory_temp; var p = tmp[0] === '~' ? Path.join(tmp.substring(1), name + '.package', filename || '') : Path.join(directory, tmp, name + '.package', filename || ''); return F.isWindows ? p.replace(REG_WINDOWSPATH, '/') : p; }; @@ -9561,7 +9668,7 @@ FrameworkCache.prototype.init = function() { var self = this; clearInterval(self.interval); self.interval = setInterval(() => F.cache.recycle(), 1000 * 60); - if (F.config['allow-cache-snapshot']) + if (CONF.allow_cache_snapshot) self.load(() => self.loadpersistent()); else self.loadpersistent(); @@ -9631,7 +9738,7 @@ FrameworkCache.prototype.stop = function() { FrameworkCache.prototype.clear = function(sync) { this.items = {}; - F.isCluster && sync !== false && F.config['allow-cache-cluster'] && process.send(CLUSTER_CACHE_CLEAR); + F.isCluster && sync !== false && CONF.allow_cache_cluster && process.send(CLUSTER_CACHE_CLEAR); this.savePersist(); return this; }; @@ -9652,13 +9759,13 @@ FrameworkCache.prototype.recycle = function() { else if (value.expire < NOW) { if (value.persist) isPersist = true; - F.emit('cache-expire', o, value.value); + EMIT('cache-expire', o, value.value); delete items[o]; } } isPersist && this.savePersist(); - F.config['allow-cache-snapshot'] && this.save(); + CONF.allow_cache_snapshot && this.save(); F.service(this.count); return this; }; @@ -9670,7 +9777,7 @@ FrameworkCache.prototype.set2 = function(name, value, expire, sync) { FrameworkCache.prototype.set = FrameworkCache.prototype.add = function(name, value, expire, sync, persist) { var type = typeof(expire); - if (F.isCluster && sync !== false && F.config['allow-cache-cluster']) { + if (F.isCluster && sync !== false && CONF.allow_cache_cluster) { CLUSTER_CACHE_SET.key = name; CLUSTER_CACHE_SET.value = value; CLUSTER_CACHE_SET.expire = expire; @@ -9694,7 +9801,7 @@ FrameworkCache.prototype.set = FrameworkCache.prototype.add = function(name, val } this.items[name] = obj; - F.$events['cache-set'] && F.emit('cache-set', name, value, expire, sync !== false); + F.$events['cache-set'] && EMIT('cache-set', name, value, expire, sync !== false); return value; }; @@ -9708,7 +9815,7 @@ FrameworkCache.prototype.read = FrameworkCache.prototype.get = function(key, def if (value.expire < NOW) { this.items[key] = undefined; - F.$events['cache-expire'] && F.emit('cache-expire', key, value.value); + F.$events['cache-expire'] && EMIT('cache-expire', key, value.value); return def; } @@ -9723,7 +9830,7 @@ FrameworkCache.prototype.read2 = FrameworkCache.prototype.get2 = function(key, d if (value.expire < NOW) { this.items[key] = undefined; - F.$events['cache-expire'] && F.emit('cache-expire', key, value.value); + F.$events['cache-expire'] && EMIT('cache-expire', key, value.value); return def; } @@ -9745,7 +9852,7 @@ FrameworkCache.prototype.remove = function(name, sync) { this.items[name] = undefined; } - if (F.isCluster && sync !== false && F.config['allow-cache-cluster']) { + if (F.isCluster && sync !== false && CONF.allow_cache_cluster) { CLUSTER_CACHE_REMOVE.key = name; process.send(CLUSTER_CACHE_REMOVE); } @@ -9771,7 +9878,7 @@ FrameworkCache.prototype.removeAll = function(search, sync) { count++; } - if (F.isCluster && sync !== false && F.config['allow-cache-cluster']) { + if (F.isCluster && sync !== false && CONF.allow_cache_cluster) { CLUSTER_CACHE_REMOVEALL.key = search; process.send(CLUSTER_CACHE_REMOVEALL); } @@ -9837,8 +9944,8 @@ function Controller(name, req, res, currentView) { // controller.type === 1 - server sent events // this.type = 0; - // this.layoutName = F.config['default-layout']; - // this.themeName = F.config['default-theme']; + // this.layoutName =CONF.default_layout; + // this.themeName =CONF.default_theme; // this.status = 200; // this.isLayout = false; @@ -9941,7 +10048,7 @@ Controller.prototype = { }, get config() { - return F.config; + return CONF; }, get controllers() { @@ -9949,7 +10056,7 @@ Controller.prototype = { }, get isDebug() { - return F.config.debug; + return CONF.debug; }, get isTest() { @@ -10299,7 +10406,7 @@ Controller.prototype.component = function(name, settings, model) { for (var i = 0; i < generator.components.length; i++) self.repository[REPOSITORY_COMPONENTS][generator.components[i]] = 1; } - return generator.call(self, self, self.repository, model || self.$model, self.session, self.query, self.body, self.url, F.global, F.helpers, self.user, self.config, F.functions, 0, self.outputPartial, self.req.files, self.req.mobile, settings || EMPTYOBJECT); + return generator.call(self, self, self.repository, model || self.$model, self.session, self.query, self.body, self.url, F.global, F.helpers, self.user, CONF, F.functions, 0, self.outputPartial, self.req.files, self.req.mobile, settings || EMPTYOBJECT); } } return ''; @@ -10689,7 +10796,7 @@ Controller.prototype.$meta = function() { return ''; } - F.$events['controller-render-meta'] && F.emit('controller-render-meta', self); + F.$events['controller-render-meta'] && EMIT('controller-render-meta', self); var repository = self.repository; return F.onMeta.call(self, repository[REPOSITORY_META_TITLE], repository[REPOSITORY_META_DESCRIPTION], repository[REPOSITORY_META_KEYWORDS], repository[REPOSITORY_META_IMAGE]); }; @@ -11406,7 +11513,7 @@ Controller.prototype.head = function() { var self = this; if (!arguments.length) { - var author = self.repository[REPOSITORY_META_AUTHOR] || self.config.author; + var author = self.repository[REPOSITORY_META_AUTHOR] || CONF.author; var plus = ''; var components = self.repository[REPOSITORY_COMPONENTS]; if (components) { @@ -12587,7 +12694,7 @@ Controller.prototype.close = function(end) { self.isConnected = false; self.res.success = true; F.reqstats(false, false); - F.$events['request-end'] && F.emit('request-end', self.req, self.res); + F.$events['request-end'] && EMIT('request-end', self.req, self.res); self.type = 0; end && self.res.end(); self.req.clear(true); @@ -12601,7 +12708,7 @@ Controller.prototype.close = function(end) { self.res.success = true; F.reqstats(false, false); - F.$events['request-end'] && F.emit('request-end', self.req, self.res); + F.$events['request-end'] && EMIT('request-end', self.req, self.res); end && self.res.end(); self.req.clear(true); return self; @@ -12710,9 +12817,9 @@ Controller.prototype.view = function(name, model, headers, partial, noasync, cac return self; if (self.layoutName === undefined) - self.layoutName = F.config['default-layout']; + self.layoutName = CONF.default_layout; if (self.themeName === undefined) - self.themeName = F.config['default-theme']; + self.themeName = CONF.default_theme; // theme root `~some_view` // views root `~~some_view` @@ -12863,7 +12970,7 @@ Controller.prototype.$viewrender = function(filename, generator, model, headers, self.repository[REPOSITORY_COMPONENTS][generator.components[i]] = 1; } - value = generator.call(self, self, self.repository, model, self.session, self.query, self.body, self.url, F.global, helpers, self.user, self.config, F.functions, 0, partial ? self.outputPartial : self.output, self.req.files, self.req.mobile, EMPTYOBJECT); + value = generator.call(self, self, self.repository, model, self.session, self.query, self.body, self.url, F.global, helpers, self.user, CONF, F.functions, 0, partial ? self.outputPartial : self.output, self.req.files, self.req.mobile, EMPTYOBJECT); } catch (ex) { @@ -13202,7 +13309,7 @@ WebSocket.prototype = { }, get config() { - return F.config; + return CONF; }, get cache() { @@ -13210,7 +13317,7 @@ WebSocket.prototype = { }, get isDebug() { - return F.config.debug; + return CONF.debug; }, get path() { @@ -13530,7 +13637,7 @@ WebSocket.prototype.destroy = function(problem) { return self; self.close(); - self.$events.destroy && self.emit('destroy'); + self.$events.destroy && selEMIT('destroy'); setTimeout(function() { @@ -13778,7 +13885,7 @@ WebSocketClient.prototype.prepare = function(flags, protocols, allow, length) { } } - var compress = (F.config['allow-websocket-compression'] && self.req.headers['sec-websocket-extensions'] || '').indexOf('permessage-deflate') !== -1; + var compress = (CONF.allow_websocket_compression && self.req.headers['sec-websocket-extensions'] || '').indexOf('permessage-deflate') !== -1; var header = protocols.length ? (compress ? SOCKET_RESPONSE_PROTOCOL_COMPRESS : SOCKET_RESPONSE_PROTOCOL).format(self.$websocket_key(self.req), protocols.join(', ')) : (compress ? SOCKET_RESPONSE_COMPRESS : SOCKET_RESPONSE).format(self.$websocket_key(self.req)); self.socket.write(U.createBuffer(header, 'binary')); @@ -13841,7 +13948,7 @@ WebSocketClient.prototype.upgrade = function(container) { self.socket.on('end', websocket_close); self.container.$add(self); self.container.$refresh(); - F.$events['websocket-begin'] && F.emit('websocket-begin', self.container, self); + F.$events['websocket-begin'] && EMIT('websocket-begin', self.container, self); self.container.$events.open && self.container.emit('open', self); return self; }; @@ -13924,7 +14031,7 @@ WebSocketClient.prototype.$ondata = function(data) { this.closemessage = current.buffer.slice(4).toString('utf8'); this.closecode = current.buffer[2] << 8 | current.buffer[3]; - if (this.closemessage && F.config['default-websocket-encodedecode']) + if (this.closemessage && CONF.default_websocket_encodedecode) this.closemessage = $decodeURIComponent(this.closemessage); this.close(); @@ -14080,7 +14187,7 @@ WebSocketClient.prototype.$decode = function() { case 3: // JSON if (data instanceof Buffer) data = data.toString(ENCODING); - F.config['default-websocket-encodedecode'] === true && (data = $decodeURIComponent(data)); + CONF.default_websocket_encodedecode === true && (data = $decodeURIComponent(data)); if (data.isJSON()) { var tmp = F.onParseJSON(data, this.req); if (tmp !== undefined) @@ -14091,7 +14198,7 @@ WebSocketClient.prototype.$decode = function() { default: // TEXT if (data instanceof Buffer) data = data.toString(ENCODING); - this.container.emit('message', this, F.config['default-websocket-encodedecode'] === true ? $decodeURIComponent(data) : data); + this.container.emit('message', this, CONF.default_websocket_encodedecode === true ? $decodeURIComponent(data) : data); break; } @@ -14174,7 +14281,7 @@ WebSocketClient.prototype.$onclose = function() { this.container.$refresh(); this.container.$events.close && this.container.emit('close', this, this.closecode, this.closemessage); this.socket.removeAllListeners(); - F.$events['websocket-end'] && F.emit('websocket-end', this.container, this); + F.$events['websocket-end'] && EMIT('websocket-end', this.container, this); }; /** @@ -14190,7 +14297,7 @@ WebSocketClient.prototype.send = function(message, raw, replacer) { if (this.type !== 1) { var data = this.type === 3 ? (raw ? message : JSON.stringify(message, replacer)) : (message || '').toString(); - if (F.config['default-websocket-encodedecode'] === true && data) + if (CONF.default_websocket_encodedecode === true && data) data = encodeURIComponent(data); if (this.deflate) { this.deflatepending.push(U.createBuffer(data)); @@ -14256,7 +14363,7 @@ WebSocketClient.prototype.close = function(message, code) { if (!self.isClosed) { self.isClosed = true; if (self.ready) - self.socket.end(U.getWebSocketFrame(code || 1000, message ? (F.config['default-websocket-encodedecode'] ? encodeURIComponent(message) : message) : '', 0x08)); + self.socket.end(U.getWebSocketFrame(code || 1000, message ? (CONF.default_websocket_encodedecode ? encodeURIComponent(message) : message) : '', 0x08)); else self.socket.end(); self.req.connection.destroy(); @@ -14542,7 +14649,7 @@ function extend_request(PROTO) { this.$total_header = header; if (this.$total_route) { F.path.verify('temp'); - framework_internal.parseMULTIPART(this, header, this.$total_route, F.config['directory-temp']); + framework_internal.parseMULTIPART(this, header, this.$total_route, CONF.directory_temp); } else this.$total_status(404); }; @@ -14568,7 +14675,7 @@ function extend_request(PROTO) { F.reqstats(false, false); this.res.writeHead(status); this.res.end(U.httpStatus(status)); - F.$events['request-end'] && F.emit('request-end', this, this.res); + F.$events['request-end'] && EMIT('request-end', this, this.res); this.clear(true); }; @@ -14590,7 +14697,7 @@ function extend_request(PROTO) { if (isError || !route) { F.stats.response['error' + status]++; - status !== 500 && F.$events['error'] && F.emit('error' + status, this, res, this.$total_exception); + status !== 500 && F.$events['error'] && EMIT('error' + status, this, res, this.$total_exception); } if (!route) { @@ -14659,8 +14766,8 @@ function extend_request(PROTO) { return; var ctrlname = '@' + name; - F.$events.controller && F.emit('controller', controller, name, this.$total_route.options); - F.$events[ctrlname] && F.emit(ctrlname, controller, name, this.$total_route.options); + F.$events.controller && EMIT('controller', controller, name, this.$total_route.options); + F.$events[ctrlname] && EMIT(ctrlname, controller, name, this.$total_route.options); if (controller.isCanceled) return; @@ -15144,7 +15251,7 @@ function extend_response(PROTO) { if (!accept && isGZIP(req)) accept = 'gzip'; - var compress = F.config['allow-gzip'] && accept.indexOf('gzip') !== -1; + var compress = CONF.allow_gzip && accept.indexOf('gzip') !== -1; if (isHEAD) { compress && (headers['Content-Encoding'] = 'gzip'); res.writeHead(200, headers); @@ -15351,7 +15458,7 @@ function extend_response(PROTO) { var options = { protocol: uri.protocol, auth: uri.auth, method: 'GET', hostname: uri.hostname, port: uri.port, path: uri.path, agent: false, headers: headers }; var connection = options.protocol === 'https:' ? require('https') : http; - var gzip = F.config['allow-gzip'] && (res.req.headers['accept-encoding'] || '').lastIndexOf('gzip') !== -1; + var gzip = CONF.allow_gzip && (res.req.headers['accept-encoding'] || '').lastIndexOf('gzip') !== -1; var client = connection.get(options, function(response) { @@ -15457,13 +15564,13 @@ function extend_response(PROTO) { if (res.success || res.headersSent) return res; - if (!F.config['static-accepts'][req.extension]) { + if (!CONF.static_accepts[req.extension]) { res.throw404(); return res; } - if (SECURITYTXT[req.url] && F.config['security.txt']) { - res.send(200, F.config['security.txt'], 'text/plain'); + if (SECURITYTXT[req.url] && CONF['security.txt']) { + res.send(200, CONF['security.txt'], 'text/plain'); return; } @@ -15501,7 +15608,7 @@ function extend_response(PROTO) { if (!canresize) { - if (F.components.has && F.components[req.extension] && req.uri.pathname === F.config['static-url-components'] + req.extension) { + if (F.components.has && F.components[req.extension] && req.uri.pathname === CONF.static_url_components + req.extension) { res.noCompress = true; res.options.components = true; var g = req.query.group ? req.query.group.substring(0, req.query.group.length - 6) : ''; @@ -15611,7 +15718,7 @@ function extend_response(PROTO) { if (name === undefined) { if (F.temporary.processing[req.$key]) { - if (req.processing > F.config['default-request-timeout']) { + if (req.processing > CONF.default_request_timeout) { res.throw408(); } else { req.processing += 500; @@ -15632,7 +15739,7 @@ function extend_response(PROTO) { !accept && isGZIP(req) && (accept = 'gzip'); - var compress = F.config['allow-gzip'] && COMPRESSION[contentType] && accept.indexOf('gzip') !== -1 && name.length > 2; + var compress = CONF.allow_gzip && COMPRESSION[contentType] && accept.indexOf('gzip') !== -1 && name.length > 2; var range = req.headers.range; var canCache = !res.$nocache && RELEASE && contentType !== 'text/cache-manifest'; @@ -15676,7 +15783,7 @@ function extend_response(PROTO) { else if (!res.options.lastmodified) headers['Last-Modified'] = name[2]; - headers.Etag = ETAG + F.config['etag-version']; + headers.Etag = ETAG + CONF.etag_version; if (range) { $file_range(name[0], range, headers, res); @@ -15756,7 +15863,7 @@ function extend_response(PROTO) { var accept = req.headers['accept-encoding'] || ''; !accept && isGZIP(req) && (accept = 'gzip'); - var compress = F.config['allow-gzip'] && COMPRESSION[options.type] && accept.indexOf('gzip') !== -1; + var compress = CONF.allow_gzip && COMPRESSION[options.type] && accept.indexOf('gzip') !== -1; var headers = compress ? HEADERS.binary_compress : HEADERS.binary; headers['Vary'] = 'Accept-Encoding' + (req.$mobile ? ', User-Agent' : ''); @@ -15815,7 +15922,7 @@ function extend_response(PROTO) { var accept = req.headers['accept-encoding'] || ''; !accept && isGZIP(req) && (accept = 'gzip'); - var compress = (options.compress === undefined || options.compress) && F.config['allow-gzip'] && COMPRESSION[options.type] && accept.indexOf('gzip') !== -1; + var compress = (options.compress === undefined || options.compress) && CONF.allow_gzip && COMPRESSION[options.type] && accept.indexOf('gzip') !== -1; var headers; if (RELEASE) { @@ -15916,7 +16023,7 @@ function extend_response(PROTO) { } if (F.temporary.processing[req.$key]) { - if (req.processing > F.config['default-request-timeout']) { + if (req.processing > CONF.default_request_timeout) { res.throw408(); } else { req.processing += 500; @@ -15964,7 +16071,7 @@ function extend_response(PROTO) { var accept = req.headers['accept-encoding'] || ''; !accept && isGZIP(req) && (accept = 'gzip'); - var gzip = F.config['allow-gzip'] && (options.compress === undefined || options.compress) ? accept.indexOf('gzip') !== -1 : false; + var gzip = CONF.allow_gzip && (options.compress === undefined || options.compress) ? accept.indexOf('gzip') !== -1 : false; var headers; if (req.$mobile) @@ -16079,7 +16186,7 @@ function extend_response(PROTO) { req.$total_execute(res.options.code, true); } - F.$events[key] && F.emit(key, req, res, res.options.problem); + F.$events[key] && EMIT(key, req, res, res.options.problem); return res; }; } @@ -16109,7 +16216,7 @@ function $file_notmodified(res, name) { if (res.getHeader('ETag')) delete headers.Etag; else - headers.Etag = ETAG + F.config['etag-version']; + headers.Etag = ETAG + CONF.etag_version; headers[HEADER_TYPE] = U.getContentType(req.extension); res.writeHead(304, headers); @@ -16313,7 +16420,7 @@ function $image_filename(exists, size, isFile, stats, res) { function response_end(res) { F.reqstats(false, res.req.isStaticFile); res.success = true; - !res.req.isStaticFile && F.$events['request-end'] && F.emit('request-end', res.req, res); + !res.req.isStaticFile && F.$events['request-end'] && EMIT('request-end', res.req, res); res.req.clear(true); res.controller && res.req.$total_success(); @@ -16356,14 +16463,14 @@ process.on('uncaughtException', function(e) { console.log('\nThe IP address and the PORT is already in use.\nYou must change the PORT\'s number or IP address.\n'); process.exit('SIGTERM'); return; - } else if (F.config['allow-filter-errors'] && REG_SKIPERROR.test(err)) + } else if (CONF.allow_filter_errors && REG_SKIPERROR.test(err)) return; F.error(e, '', null); }); function fsFileRead(filename, callback, a, b, c) { - U.queue('F.files', F.config['default-maxopenfiles'], function(next) { + U.queue('F.files', CONF.default_maxopenfiles, function(next) { Fs.readFile(filename, function(err, result) { next(); callback(err, result, a, b, c); @@ -16372,7 +16479,7 @@ function fsFileRead(filename, callback, a, b, c) { } function fsFileExists(filename, callback, a, b, c) { - U.queue('F.files', F.config['default-maxopenfiles'], function(next) { + U.queue('F.files', CONF.default_maxopenfiles, function(next) { Fs.lstat(filename, function(err, stats) { next(); callback(!err && stats.isFile(), stats ? stats.size : 0, stats ? stats.isFile() : false, stats, a, b, c); @@ -16401,7 +16508,7 @@ function fsStreamRead(filename, options, callback, res) { } else opt = HEADERS.fsStreamRead; - U.queue('F.files', F.config['default-maxopenfiles'], function(next) { + U.queue('F.files', CONF.default_maxopenfiles, function(next) { var stream = Fs.createReadStream(filename, opt); stream.on('error', NOOP); callback(stream, next, res); @@ -16491,11 +16598,11 @@ process.on('message', function(msg, h) { msg.TYPE === 'cache-clear' && F.cache.clear(false); msg.TYPE === 'req' && F.cluster.req(msg); msg.TYPE === 'res' && msg.target === F.id && F.cluster.res(msg); - msg.TYPE === 'emit' && F.$events[msg.name] && F.emit(msg.name, msg.data); + msg.TYPE === 'emit' && F.$events[msg.name] && EMIT(msg.name, msg.data); msg.TYPE === 'nosql-meta' && NOSQL(msg.name).meta(msg.key, msg.value, true); msg.TYPE === 'table-meta' && TABLE(msg.name).meta(msg.key, msg.value, true); } - F.$events.message && F.emit('message', msg, h); + F.$events.message && EMIT('message', msg, h); }); function prepare_error(e) { @@ -16508,7 +16615,7 @@ function prepare_error(e) { } function prepare_filename(name) { - return name[0] === '@' ? (F.isWindows ? U.combine(F.config['directory-temp'], name.substring(1)) : F.path.package(name.substring(1))) : U.combine('/', name); + return name[0] === '@' ? (F.isWindows ? U.combine(CONF.directory_temp, name.substring(1)) : F.path.package(name.substring(1))) : U.combine('/', name); } function prepare_staticurl(url, isDirectory) { @@ -16878,7 +16985,7 @@ function parseSchema(name) { function ilogger(body) { F.path.verify('logs'); - U.queue('F.ilogger', 5, (next) => Fs.appendFile(U.combine(F.config['directory-logs'], 'logger.log'), body, next)); + U.queue('F.ilogger', 5, (next) => Fs.appendFile(U.combine(CONF.directory_logs, 'logger.log'), body, next)); } F.ilogger = function(name, req, ts) { diff --git a/internal.js b/internal.js index 47f88241c..c9b54755e 100755 --- a/internal.js +++ b/internal.js @@ -771,7 +771,7 @@ HFP.isAudio = function() { HFP.image = function(im) { if (im === undefined) - im = F.config['default-image-converter'] === 'im'; + im = CONF.default_image_converter === 'im'; return framework_image.init(this.path, im, this.width, this.height); }; @@ -1813,7 +1813,7 @@ function view_parse(content, minify, filename, controller) { if (cmd[0] === '\'' || cmd[0] === '"') { if (cmd[1] === '%') { - var t = F.config[cmd.substring(2, cmd.length - 1)]; + var t = CONF[cmd.substring(2, cmd.length - 1)]; if (t != null) builder += '+' + DELIMITER + t + DELIMITER; } else @@ -1922,7 +1922,7 @@ function view_parse(content, minify, filename, controller) { if (can && !counter) { try { - var r = (new Function('self', 'config', 'return ' + tmp))(controller, F.config).replace(REG_7, '\\\\').replace(REG_8, '\\\''); + var r = (new Function('self', 'config', 'return ' + tmp))(controller, CONF).replace(REG_7, '\\\\').replace(REG_8, '\\\''); if (r) { txtindex = $VIEWCACHE.indexOf(r); if (txtindex === -1) { @@ -2044,7 +2044,7 @@ function view_prepare(command, dynamicCommand, functions, controller, components return '$STRING(' + command + ')'; case 'root': - var r = F.config['default-root']; + var r = CONF.default_root; return '\'' + (r ? r.substring(0, r.length - 1) : r) + '\''; case 'M': @@ -2503,7 +2503,7 @@ function compressView(html, minify) { */ function compressJS(html, index, filename, nomarkup) { - if (!F.config['allow-compile-script']) + if (!CONF.allow_compile_script) return html; var strFrom = '', 'framework.view()'); - }, 100); + exports.installed = true; + ROUTE('/inline-view-route/'); + setTimeout(function() { + assert.ok(VIEW('view') === '
Total.js
', 'VIEW()'); + }, 100); }; \ No newline at end of file diff --git a/test/modules/test.js b/test/modules/test.js index 88825c6f6..0699116ed 100755 --- a/test/modules/test.js +++ b/test/modules/test.js @@ -5,9 +5,9 @@ exports.install = function() { app = framework; assert.ok(typeof(framework.modules) === 'object', 'module install'); - setTimeout(function() { - assert.ok(MODULE('inline-view').installed, 'module install dependencies'); - }, 3000); + setTimeout(function() { + assert.ok(MODULE('inline-view').installed, 'module install dependencies'); + }, 3000); }; exports.message = function() { diff --git a/test/test-tmp.js b/test/test-tmp.js index b6aceaf82..97fa07942 100644 --- a/test/test-tmp.js +++ b/test/test-tmp.js @@ -7,7 +7,7 @@ require('../index'); // NOSQL('test').on('modify', console.log); // NOSQL('test').update({ name: GUID(5) }).between('index', 1, 3).callback(console.log); -F.config['table.test'] = 'index:number | name:string'; +CONF['table.test'] = 'index:number | name:string'; TABLE('test').find().take(10).skip(10).callback(console.log); diff --git a/utils.js b/utils.js index ef49b8004..e1c2d0ea9 100755 --- a/utils.js +++ b/utils.js @@ -485,7 +485,7 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, if (callback === NOOP) callback = null; - var options = { length: 0, timeout: timeout || CONF['default-restbuilder-timeout'], evt: new EventEmitter2(), encoding: typeof(encoding) !== 'string' ? ENCODING : encoding, callback: callback, post: false, redirect: 0 }; + var options = { length: 0, timeout: timeout || CONF.default_restbuilder_timeout, evt: new EventEmitter2(), encoding: typeof(encoding) !== 'string' ? ENCODING : encoding, callback: callback, post: false, redirect: 0 }; var method; var type = 0; var isCookies = false; @@ -641,8 +641,8 @@ global.REQUEST = exports.request = function(url, flags, data, callback, cookies, if (options.resolve && (uri.hostname === 'localhost' || uri.hostname.charCodeAt(0) < 64)) options.resolve = null; - if (F.config['default-proxy'] && !proxy && !PROXYBLACKLIST[uri.hostname]) - proxy = parseProxy(F.config['default-proxy']); + if (CONF.default_proxy && !proxy && !PROXYBLACKLIST[uri.hostname]) + proxy = parseProxy(CONF.default_proxy); if (proxy && (uri.hostname === 'localhost' || uri.hostname === '127.0.0.1')) proxy = null; @@ -1180,8 +1180,8 @@ exports.download = function(url, flags, data, callback, cookies, headers, encodi headers['Content-Length'] = options.data.length; } - if (F.config['default-proxy'] && !proxy && !PROXYBLACKLIST[uri.hostname]) - proxy = parseProxy(F.config['default-proxy']); + if (CONF.default_proxy && !proxy && !PROXYBLACKLIST[uri.hostname]) + proxy = parseProxy(CONF.default_proxy); options.proxy = proxy; @@ -3023,7 +3023,7 @@ SP.ROOT = function(noremap) { return ''; }).replace(REG_ROOT, $urlmaker); - if (!noremap && F.config['default-root']) + if (!noremap && CONF.default_root) str = str.replace(REG_REMAP, $urlremap); return str; @@ -3031,12 +3031,12 @@ SP.ROOT = function(noremap) { function $urlremap(text) { var pos = text[0] === 'h' ? 6 : 5; - return REG_URLEXT.test(text) ? text : ((text[0] === 'h' ? 'href' : 'src') + '="' + F.config['default-root'] + (text[pos] === '/' ? text.substring(pos + 1) : text)); + return REG_URLEXT.test(text) ? text : ((text[0] === 'h' ? 'href' : 'src') + '="' + CONF.default_root + (text[pos] === '/' ? text.substring(pos + 1) : text)); } function $urlmaker(text) { var c = text[4]; - return F.config['default-root'] ? F.config['default-root'] : (c || ''); + return CONF.default_root ? CONF.default_root : (c || ''); } if (!SP.trim) { @@ -3492,7 +3492,7 @@ SP.parseConfig = function(def, onerr) { obj[name] = (/true|on|1|enabled/i).test(value); break; case 'config': - obj[name] = F.config[value]; + obj[name] = CONF[value]; break; case 'eval': case 'object': @@ -3857,7 +3857,7 @@ SP.encrypt = function(key, isUnique, secret) { for (var i = 0; i < str.length; i++) sum += str.charCodeAt(i); - return (sum + checksum((secret || F.config.secret) + key)) + '-' + str; + return (sum + checksum((secret || CONF.secret) + key)) + '-' + str; }; SP.decrypt = function(key, secret) { @@ -3871,7 +3871,7 @@ SP.decrypt = function(key, secret) { return null; var hash = this.substring(index + 1); - var sum = checksum((secret || F.config.secret) + key); + var sum = checksum((secret || CONF.secret) + key); for (var i = 0; i < hash.length; i++) sum += hash.charCodeAt(i); @@ -3910,7 +3910,7 @@ exports.encryptUID = function(val, key) { var sum = 0; if (!key) - key = F.config.secret; + key = CONF.secret; val = val.toString(); @@ -3920,7 +3920,7 @@ exports.encryptUID = function(val, key) { for (var i = 0; i < key.length; i++) sum += key.charCodeAt(i); - return (num ? 'n' : 'x') + (F.config['secret-uid'] + val + sum + key).crc32(true).toString(16) + 'x' + val; + return (num ? 'n' : 'x') + (CONF.secret_uid + val + sum + key).crc32(true).toString(16) + 'x' + val; }; exports.decryptUID = function(val, key) { @@ -5792,7 +5792,7 @@ exports.parseTheme = function(value) { if (index === -1) return ''; value = value.substring(1, index); - return value === '?' ? F.config['default-theme'] : value; + return value === '?' ? CONF.default_theme : value; }; exports.set = function(obj, path, value) { From a46441d52e3abf3d6bc5bad52aae3d53a3201987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 21 Oct 2018 19:00:41 +0200 Subject: [PATCH 137/217] Added config encryption. --- changes.txt | 1 + index.js | 5 +++++ test/config-debug | 5 ++++- test/config-release | 5 ++++- test/test-framework-debug.js | 2 ++ test/test-framework-release.js | 2 ++ 6 files changed, 18 insertions(+), 2 deletions(-) diff --git a/changes.txt b/changes.txt index 97e65cd37..752c7f1cf 100755 --- a/changes.txt +++ b/changes.txt @@ -19,6 +19,7 @@ - added: a new config item `logger : false` which enables logging for Middleware, Schemas and Operations - added: `SchemaOptions` and `OperationOptions` supports `$.cancel()` method - added: `CACHE(name, [value], [expire], [persistent])` alias for `F.cache.get2()` and `F.cache.set()` or `F.cache.set2()` +- added: encryption of config values - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) diff --git a/index.js b/index.js index ea8bbeed2..f1c8b0c4f 100755 --- a/index.js +++ b/index.js @@ -8672,6 +8672,11 @@ F.$configure_configs = function(arr, rewrite) { value = str.substring(index + 1).trim(); index = name.indexOf('('); + if (value.substring(0, 7) === 'base64 ' && value.length > 8) + value = U.createBuffer(value.substring(7).trim(), 'base64').toString('utf8'); + else if (value.substring(0, 4) === 'hex ' && value.length > 6) + value = U.createBuffer(value.substring(4).trim(), 'hex').toString('utf8'); + if (index !== -1) { subtype = name.substring(index + 1, name.indexOf(')')).trim().toLowerCase(); name = name.substring(0, index).trim(); diff --git a/test/config-debug b/test/config-debug index ef433010f..7a700c7d6 100755 --- a/test/config-debug +++ b/test/config-debug @@ -1,3 +1,6 @@ etag-version : 1 secret : total.js test -array (Array) : [1, 2, 3, 4] \ No newline at end of file +array (Array) : [1, 2, 3, 4] + +testbase : base64 MTIzNDU2 +testhex : hex 313233343536 \ No newline at end of file diff --git a/test/config-release b/test/config-release index ef433010f..7a700c7d6 100755 --- a/test/config-release +++ b/test/config-release @@ -1,3 +1,6 @@ etag-version : 1 secret : total.js test -array (Array) : [1, 2, 3, 4] \ No newline at end of file +array (Array) : [1, 2, 3, 4] + +testbase : base64 MTIzNDU2 +testhex : hex 313233343536 \ No newline at end of file diff --git a/test/test-framework-debug.js b/test/test-framework-debug.js index 5f723711b..7b814a460 100755 --- a/test/test-framework-debug.js +++ b/test/test-framework-debug.js @@ -29,6 +29,8 @@ framework.on('ready', function() { }); t.on('exit', () => assert.ok(a === true, 'F.load() in worker')); assert.ok(F.config.array.length === 4, 'Problem with config sub types.'); + assert.ok(CONF.testhex === 123456, 'config: hex encode'); + assert.ok(CONF.testbase === 123456, 'config: base encode'); }); framework.onAuthorize = function(req, res, flags, cb) { diff --git a/test/test-framework-release.js b/test/test-framework-release.js index 3cfccb3ac..e5a2efb7d 100755 --- a/test/test-framework-release.js +++ b/test/test-framework-release.js @@ -29,6 +29,8 @@ framework.on('ready', function() { }); t.on('exit', () => assert.ok(a === true, 'F.load() in worker')); assert.ok(F.config.array.length === 4, 'Problem with config sub types.'); + assert.ok(CONF.testhex === 123456, 'config: hex encode'); + assert.ok(CONF.testbase === 123456, 'config: base encode'); }); framework.onAuthorize = function(req, res, flags, cb) { From b74c04beb80ac449c4fc24a3f3930e6bed249d3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 21 Oct 2018 20:29:33 +0200 Subject: [PATCH 138/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bdff2dd63..ba55e40af 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-19", + "version": "3.0.1-20", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From d8bc00e616eccc3126ed386c877659e21d5e66fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 22 Oct 2018 11:04:01 +0200 Subject: [PATCH 139/217] Fixed config values. --- index.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/index.js b/index.js index f1c8b0c4f..5fffc760f 100755 --- a/index.js +++ b/index.js @@ -771,6 +771,16 @@ function Framework() { default_interval_websocket_ping: 3, default_interval_uptodate: 5, + set ['mail-smtp'] (val) { + CONF['mail_smtp'] = val; + return null; + }, + + set ['mail-smtp-options'] (val) { + CONF['mail_smtp_options'] = val; + return null; + }, + set ['mail-address-reply'] (val) { CONF['mail_address_reply'] = val; return null; From e081cc478027ba9c7787d5ad85d7e0b1d1571714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 22 Oct 2018 11:12:57 +0200 Subject: [PATCH 140/217] Updated TABLE reference to `CONF.table_{name}`. --- nosql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 6e1964058..ffbb0a7ba 100755 --- a/nosql.js +++ b/nosql.js @@ -567,7 +567,7 @@ function Table(name, filename) { t.counter = new Counter(t); t.$meta(); - var schema = CONF['table.' + name]; + var schema = CONF['table_' + name] || CONF['table.' + name]; Fs.createReadStream(t.filename, { end: 1200 }).once('data', function(chunk) { From efc113c58ff2ca6d9f0fed70d6932a2004775a99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 23 Oct 2018 20:01:52 +0200 Subject: [PATCH 141/217] Fixed `repeat` mode in `SCHEDULE()`. --- changes.txt | 1 + index.js | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 752c7f1cf..a07cb7468 100755 --- a/changes.txt +++ b/changes.txt @@ -48,6 +48,7 @@ - fixed: compressing CSS with `\t` tabs - fixed: `controller.autoclear()` - fixed: `controller.proxy()` +- fixed: `repeat` mode in `SCHEDULE()` - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute diff --git a/index.js b/index.js index 5fffc760f..3776b2f9d 100755 --- a/index.js +++ b/index.js @@ -1476,7 +1476,7 @@ F.schedule = function(date, repeat, fn) { if (type === 'string') { date = date.parseDate(); - repeat && date < NOW && (date = NOW.add(repeat)); + repeat && date < NOW && (date = date.add(repeat)); } else if (type === 'number') date = new Date(date); @@ -1492,6 +1492,7 @@ F.clearSchedule = function(id) { return F; }; + /** * Auto resize picture according the path * @param {String} url Relative path. From ad273be75a3c5a42a5714d99421a3d3e4398361a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 23 Oct 2018 20:29:50 +0200 Subject: [PATCH 142/217] Fixed filtering in NoSQL. --- nosql.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nosql.js b/nosql.js index ffbb0a7ba..f0ad598d0 100755 --- a/nosql.js +++ b/nosql.js @@ -6573,6 +6573,7 @@ NoSQLReader.prototype.add = function(builder, noTrimmer) { } else { var item = {}; item.scalarcount = 0; + item.all = 0; item.count = 0; item.counter = 0; item.builder = builder; @@ -6586,7 +6587,6 @@ NoSQLReader.prototype.add = function(builder, noTrimmer) { NoSQLReader.prototype.compare2 = function(docs, custom, done) { var self = this; - for (var i = 0; i < docs.length; i++) { var doc = docs[i]; @@ -6605,7 +6605,7 @@ NoSQLReader.prototype.compare2 = function(docs, custom, done) { if (item.canceled) continue; - var output = item.compare(doc, item.filter, j); + var output = item.compare(doc, item.filter, item.all++); if (!output) continue; @@ -6647,7 +6647,7 @@ NoSQLReader.prototype.compare = function(docs) { if (item.canceled) continue; - var output = item.compare(doc, item.filter, j); + var output = item.compare(doc, item.filter, item.all++); if (!output) continue; From c25093e3a4fad7b7fdb52c5d75db7c551ea05147 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 23 Oct 2018 20:40:34 +0200 Subject: [PATCH 143/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ba55e40af..b47557352 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-20", + "version": "3.0.1-21", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 560f7d3f0f298f9a3f5f526d27c47fe068b144e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 24 Oct 2018 09:44:39 +0200 Subject: [PATCH 144/217] Fixed `SCHEDULE()` and `UTC()`. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 3776b2f9d..2f97992a2 100755 --- a/index.js +++ b/index.js @@ -1475,7 +1475,7 @@ F.schedule = function(date, repeat, fn) { var type = typeof(date); if (type === 'string') { - date = date.parseDate(); + date = date.parseDate().toUTC(); repeat && date < NOW && (date = date.add(repeat)); } else if (type === 'number') date = new Date(date); From 03fdcd9482ee2d9ef10cd64088beeec8de0c5a1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 24 Oct 2018 15:18:36 +0200 Subject: [PATCH 145/217] Fixed saving data of `NoSQL.Counter`. --- nosql.js | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/nosql.js b/nosql.js index f0ad598d0..fe5a562bd 100755 --- a/nosql.js +++ b/nosql.js @@ -3061,6 +3061,13 @@ function Counter(db) { t.key = (db instanceof Table ? 'table' : 'nosql') + db.name.hash(); t.type = 0; // 1 === saving, 2 === reading t.$events = {}; + t.$cb_save = function() { + t.tid = undefined; + if (F.isCluster) + clusterlock(t, '$save'); + else + t.$save(); + }; } const CP = Counter.prototype; @@ -3161,7 +3168,7 @@ CP.min = function(id, count) { arr[1] = count; } - setTimeout2(self.key, () => self.save(), self.TIMEOUT, 5); + self.save(); this.$events.min && self.emit('min', id, count || 1); return self; }; @@ -3187,8 +3194,8 @@ CP.max = function(id, count) { arr[1] = count; } - setTimeout2(self.key, () => self.save(), self.TIMEOUT, 5); - this.$events.max && self.emit('max', id, count || 1); + self.save(); + self.$events.max && self.emit('max', id, count || 1); return self; }; @@ -3208,7 +3215,7 @@ CP.inc = CP.hit = function(id, count) { else self.cache[key] += count || 1; - setTimeout2(self.key, () => self.save(), self.TIMEOUT, 5); + self.save(); this.$events.sum && self.emit('sum', id, count || 1); this.$events.hits && self.emit('hit', id, count || 1); return self; @@ -3224,7 +3231,7 @@ CP.remove = function(id) { else self.cache[id] = null; - setTimeout2(self.key, () => self.save(), self.TIMEOUT, 5); + self.save(); self.emit('remove', id); return self; }; @@ -3987,16 +3994,15 @@ function counter_parse_days_all(output, value, year, opt) { CP.save = function() { var self = this; - if (F.isCluster) - clusterlock(self, '$save'); - else - self.$save(); + !self.tid && (self.tid = setTimeout(self.$cb_save, self.TIMEOUT)); return self; }; CP.$save = function() { var self = this; + + self.tid && clearTimeout(self.tid); self.db.readonly && self.db.throwReadonly(); if (self.type) { From 03b8bf12c0f83713fe329d0fc238932c122385a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 24 Oct 2018 15:18:53 +0200 Subject: [PATCH 146/217] Fixed `setTimeout2()` with a limit. --- index.js | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/index.js b/index.js index 2f97992a2..cc37797ff 100755 --- a/index.js +++ b/index.js @@ -6975,6 +6975,11 @@ F.service = function(count) { releasegc = true; CONF.allow_debug && F.consoledebug('clear temporary cache'); + + keys = Object.keys(F.temporary.internal); + for (var i = 0; i < keys.length; i++) + if (!F.temporary.internal[keys[i]]) + delete F.temporary.internal[keys[i]]; } // every 61 minutes (default) services precompile all (installed) views @@ -16752,20 +16757,35 @@ function async_middleware(index, req, res, middleware, callback, options, contro global.setTimeout2 = function(name, fn, timeout, limit, param) { var key = ':' + name; + var internal = F.temporary.internal; + if (limit > 0) { - var key2 = key + ':limit'; - if (F.temporary.internal[key2] >= limit) + + var key2 = key + '_limit'; + var key3 = key + '_fn'; + + if (internal[key2] >= limit) { + internal[key] && clearTimeout(internal[key]); + internal[key] = internal[key2] = internal[key3] = undefined; + fn(); return; - F.temporary.internal[key2] = (F.temporary.internal[key2] || 0) + 1; - F.temporary.internal[key] && clearTimeout(F.temporary.internal[key]); - return F.temporary.internal[key] = setTimeout(function(param) { - F.temporary.internal[key2] = undefined; + } + + internal[key] && clearTimeout(internal[key]); + internal[key2] = (internal[key2] || 0) + 1; + + return internal[key] = setTimeout(function(param, key) { + F.temporary.internal[key] = F.temporary.internal[key + '_limit'] = F.temporary.internal[key + '_fn'] = undefined; fn && fn(param); - }, timeout, param); + }, timeout, param, key); + } + + if (internal[key]) { + clearTimeout(internal[key]); + internal[key] = undefined; } - F.temporary.internal[key] && clearTimeout(F.temporary.internal[key]); - return F.temporary.internal[key] = setTimeout(fn, timeout, param); + return internal[key] = setTimeout(fn, timeout, param); }; global.clearTimeout2 = function(name) { From 32f0299a1d3bb1a809fb7aecbea10fe483f4b90a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 24 Oct 2018 15:51:41 +0200 Subject: [PATCH 147/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b47557352..b9cace217 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-21", + "version": "3.0.1-22", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 4b64e02fb699c161f1d1ff251111089006fd6d2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 24 Oct 2018 20:34:04 +0200 Subject: [PATCH 148/217] Added new fix. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index a07cb7468..65d129162 100755 --- a/changes.txt +++ b/changes.txt @@ -29,6 +29,7 @@ - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator +- fixed: a critical bug with NoSQL counter and freezing app - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function - fixed: `F.wait()` in the test mode - fixed: `LOCALIZE()` for nested directories From b9cda59ec18234be37fd2fe2425334df21b4e235 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 24 Oct 2018 20:46:29 +0200 Subject: [PATCH 149/217] Added new fix. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index 65d129162..fd543d663 100755 --- a/changes.txt +++ b/changes.txt @@ -30,6 +30,7 @@ - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator - fixed: a critical bug with NoSQL counter and freezing app +- fixed: a critical bug with rendering of multiple async components - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function - fixed: `F.wait()` in the test mode - fixed: `LOCALIZE()` for nested directories From 1d700fdf8985842a5b7f27ab91ca1e4b2b35b8dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 28 Oct 2018 08:51:49 +0100 Subject: [PATCH 150/217] Improved `String.toUnicode()` and `String.fromUnicode()`. --- utils.js | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/utils.js b/utils.js index e1c2d0ea9..b6b7a9d3a 100755 --- a/utils.js +++ b/utils.js @@ -3763,22 +3763,28 @@ SP.capitalize = function(first) { return builder; }; - -SP.toUnicode = function() { - var result = ''; - var self = this; - var length = self.length; - for(var i = 0; i < length; ++i){ - if(self.charCodeAt(i) > 126 || self.charCodeAt(i) < 32) - result += '\\u' + self.charCodeAt(i).hex(4); +String.prototype.toUnicode = function() { + var output = ''; + for (var i = 0; i < this.length; i++) { + var c = this[i].charCodeAt(0); + if(c > 126 || c < 32) + output += '\\u' + ('000' + c.toString(16)).substr(-4); else - result += self[i]; + output += this[i]; } - return result; + return output; }; -SP.fromUnicode = function() { - return unescape(this.replace(regexpUNICODE, (match, v) => String.fromCharCode(parseInt(v, 16)))); +String.prototype.fromUnicode = function() { + var output = ''; + for (var i = 0; i < this.length; i++) { + if (this[i] === '\\' && this[i + 1] === 'u') { + output += String.fromCharCode(parseInt(this[i + 2] + this[i + 3] + this[i + 4] + this[i + 5], 16)); + i += 5; + } else + output += this[i]; + } + return output; }; SP.sha1 = function(salt) { From b1a2b76f83da8b570a4747af9b9edfc8ef908b7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 28 Oct 2018 17:28:06 +0100 Subject: [PATCH 151/217] Improved `TABLE()`. --- nosql.js | 221 ++++++++++++++++++++++++------- nosqlstream.js | 343 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 425 insertions(+), 139 deletions(-) diff --git a/nosql.js b/nosql.js index fe5a562bd..bf414d818 100755 --- a/nosql.js +++ b/nosql.js @@ -577,17 +577,22 @@ function Table(name, filename) { } t.parseSchema(chunk.toString('utf8').split('\n', 1)[0].split('|')); - t.ready = true; - t.next(0); - if (schema && t.stringifySchema() !== schema) + if (schema && t.stringifySchema() !== schema) { + t.$header = Buffer.byteLength(t.stringifySchema()) + 1; t.extend(schema); + } else + t.$header = Buffer.byteLength(schema ? schema : t.stringifySchema()) + 1; + + t.next(0); }).on('error', function(e) { if (schema) { t.parseSchema(schema.replace(/;|,/g, '|').trim().split('|')); - Fs.writeFileSync(t.filename, t.stringifySchema() + NEWLINE, 'utf8'); + var bschema = t.stringifySchema(); + t.$header = Buffer.byteLength(bschema) + 1; + Fs.writeFileSync(t.filename, bschema + NEWLINE, 'utf8'); t.ready = true; t.next(0); } else { @@ -1146,8 +1151,11 @@ DP.find2 = function(builder) { var self = this; if (builder instanceof DatabaseBuilder) builder.db = self; - else + else { builder = new DatabaseBuilder(self); + builder.$options.notall = true; + } + if (self.readonly) return self.find(builder); self.pending_reader2.push(builder); @@ -5535,8 +5543,11 @@ TP.find2 = function(builder) { var self = this; if (builder) builder.db = self; - else + else { builder = new DatabaseBuilder(self); + builder.$options.notall = true; + } + self.pending_reader2.push(builder); setImmediate(next_operation, self, 11); return builder; @@ -5563,13 +5574,14 @@ TP.extend = function(schema, callback) { var olds = self.$schema; var oldk = self.$keys; + var oldl = self.$size; + var oldh = Buffer.byteLength(self.stringifySchema() + NEWLINE); self.parseSchema(schema.replace(/;|,/g, '|').trim().split('|')); var meta = self.stringifySchema() + NEWLINE; var news = self.$schema; var newk = self.$keys; - self.$schema = olds; self.$keys = oldk; @@ -5594,8 +5606,14 @@ TP.extend = function(schema, callback) { }); data.keys = self.$keys; + fs.start = oldh; fs.divider = '\n'; + if (oldl) + self.linesize = oldl; + + var size = self.$size; + fs.ondocuments = function() { var lines = fs.docs.split(fs.divider); @@ -5603,8 +5621,9 @@ TP.extend = function(schema, callback) { self.$schema = olds; self.$keys = oldk; + self.$size = oldl; - for (var a = count ? 0 : 1; a < lines.length; a++) { + for (var a = 0; a < lines.length; a++) { data.line = lines[a].split('|'); data.index = count++; var doc = self.parseData(data); @@ -5614,6 +5633,7 @@ TP.extend = function(schema, callback) { self.$schema = news; self.$keys = newk; + self.$size = size; var buffer = ''; for (var i = 0; i < items.length; i++) buffer += self.stringify(items[i], true) + NEWLINE; @@ -5623,6 +5643,7 @@ TP.extend = function(schema, callback) { fs.$callback = function() { self.$schema = news; self.$keys = newk; + self.$header = Buffer.byteLength(meta); writer.end(); fs = null; }; @@ -5816,7 +5837,11 @@ TP.$reader = function() { var data = {}; var indexer = 0; + fs.array = true; + fs.start = self.$header; + fs.linesize = self.$size; fs.divider = '\n'; + data.keys = self.$keys; if (self.buffersize) @@ -5827,16 +5852,16 @@ TP.$reader = function() { fs.ondocuments = function() { - var lines = fs.docs.split(fs.divider); + var lines = fs.docs; var arr = []; - for (var j = indexer ? 0 : 1; j < lines.length; j++) { + for (var j = 0; j < lines.length; j++) { data.line = lines[j].split('|'); data.index = indexer++; arr.push(self.parseData(data)); } - return filters.compare(arr, jsonparser); + return filters.compare(arr); }; fs.$callback = function() { @@ -5868,6 +5893,9 @@ TP.$reader3 = function() { var data = {}; var indexer = 0; + fs.array = true; + fs.start = self.$header; + fs.linesize = self.$size; fs.divider = '\n'; data.keys = self.$keys; @@ -5879,18 +5907,18 @@ TP.$reader3 = function() { fs.ondocuments = function() { - var lines = fs.docs.split(fs.divider); + var lines = fs.docs; var arr = []; for (var j = 0; j < lines.length; j++) { data.line = lines[j].split('|'); - if (!TABLERECORD[data.line[0]]) - continue; - data.index = indexer++; - arr.push(self.parseData(data)); + if (TABLERECORD[data.line[0]]) { + data.index = indexer++; + arr.push(self.parseData(data)); + } } - return filters.compare(arr, jsonparser); + return filters.compare(arr); }; fs.$callback = function() { @@ -5926,6 +5954,9 @@ TP.$update = function() { for (var i = 0; i < filter.length; i++) filters.add(filter[i].builder, true); + fs.array = true; + fs.start = self.$header; + fs.linesize = self.$size; fs.divider = '\n'; if (self.buffersize) @@ -5937,7 +5968,6 @@ TP.$update = function() { var update = function(docs, doc, dindex, f, findex) { var rec = fs.docsbuffer[dindex]; - var fil = filter[findex]; var e = fil.keys ? 'modify' : 'update'; var old = self.$events[e] ? CLONE(doc) : 0; @@ -5987,6 +6017,7 @@ TP.$update = function() { var rec = fs.docsbuffer[dindex]; var upd = self.stringify(doc, null, rec.length); + if (upd === rec.doc) return; @@ -6004,13 +6035,10 @@ TP.$update = function() { fs.ondocuments = function() { - var lines = fs.docs.split(fs.divider); + var lines = fs.docs; var arr = []; - if (!indexer) - arr.push(EMPTYOBJECT); - - for (var a = indexer ? 0 : 1; a < lines.length; a++) { + for (var a = 0; a < lines.length; a++) { data.line = lines[a].split('|'); data.length = lines[a].length; data.index = indexer++; @@ -6069,6 +6097,9 @@ TP.$remove = function() { var change = false; var indexer = 0; + fs.array = true; + fs.start = self.$header; + fs.linesize = self.$size; fs.divider = '\n'; if (self.buffersize) @@ -6094,13 +6125,13 @@ TP.$remove = function() { fs.ondocuments = function() { - var lines = fs.docs.split(fs.divider); + var lines = fs.docs; var arr = []; if (!indexer) arr.push(EMPTYOBJECT); - for (var a = indexer ? 0 : 1; a < lines.length; a++) { + for (var a = 0; a < lines.length; a++) { data.line = lines[a].split('|'); data.index = indexer++; arr.push(self.parseData(data)); @@ -6143,6 +6174,8 @@ TP.$clean = function() { var fs = new NoSQLStream(self.filename); var writer = Fs.createWriteStream(self.filename + '-tmp'); + fs.start = self.$header; + fs.linesize = self.$size; fs.divider = NEWLINE; if (self.buffersize) @@ -6232,6 +6265,9 @@ TP.$streamer = function() { var data = {}; data.keys = self.$keys; + + fs.array = true; + fs.start = self.$header; fs.divider = '\n'; if (self.buffersize) @@ -6241,8 +6277,8 @@ TP.$streamer = function() { fs.buffercount = self.buffercount; fs.ondocuments = function() { - var lines = fs.docs.split(fs.divider); - for (var a = count ? 0 : 1; a < lines.length; a++) { + var lines = fs.docs; + for (var a = 0; a < lines.length; a++) { data.line = lines[a].split('|'); data.index = count++; var doc = self.parseData(data); @@ -6271,37 +6307,62 @@ TP.allocations = function(enable) { TP.parseSchema = function() { var self = this; var arr = arguments[0] instanceof Array ? arguments[0] : arguments; + var sized = true; self.$schema = {}; self.$keys = []; + self.$size = 2; for (var i = 0; i < arr.length; i++) { var arg = arr[i].split(':'); var type = 0; - switch ((arg[1] || '').toLowerCase().trim()) { + var T = (arg[1] || '').toLowerCase().trim(); + var size = 0; + + var index = T.indexOf('('); + if (index != -1) { + size = +T.substring(index + 1, T.lastIndexOf(')')); + T = T.substring(0, index); + } + + switch (T) { case 'number': type = 2; + !size && (size = 16); break; case 'boolean': case 'bool': type = 3; + size = 1; break; case 'date': type = 4; + size = 13; break; case 'object': type = 5; + size = 0; + sized = false; break; case 'string': default: type = 1; + if (!size) + sized = false; break; } var name = arg[0].trim(); - self.$schema[name] = { type: type, pos: i }; + self.$schema[name] = { type: type, pos: i, size: size }; self.$keys.push(name); + self.$size += size + 1; } + if (sized) { + self.$allocations = false; + self.$size++; // newline + } else + self.$size = 0; + return self; }; @@ -6318,8 +6379,15 @@ TP.stringifySchema = function() { switch (meta.type) { case 2: + type = 'number'; + + // string + if (self.$size && meta.size !== 16) + type += '(' + (meta.size) + ')'; + break; + case 3: type = 'boolean'; break; @@ -6329,6 +6397,11 @@ TP.stringifySchema = function() { case 5: type = 'object'; break; + default: + // string + if (meta.size) + type += '(' + (meta.size) + ')'; + break; } data.push(key + ':' + type); @@ -6362,29 +6435,43 @@ TP.parseData = function(data, cache) { continue; var pos = meta.pos + 1; + var line = data.line[pos]; + + if (self.$size) { + for (var j = line.length - 1; j > -1; j--) { + if (line[j] !== ' ') { + line = line.substring(0, j + 1); + break; + } + } + } switch (meta.type) { case 1: // String - obj[key] = data.line[pos]; + obj[key] = line; if (esc && obj[key]) obj[key] = obj[key].replace(REGTUNESCAPE, regtescapereverse); + if (self.$size && obj[key].indexOf('\\u') !== -1) + obj[key] = obj[key].fromUnicode(); break; case 2: // Number - val = +data.line[pos]; + val = +line; obj[key] = val < 0 || val > 0 ? val : 0; break; case 3: // Boolean - val = data.line[pos]; + val = line; obj[key] = BOOLEAN[val] == 1; break; case 4: // Date - val = data.line[pos]; + val = line; obj[key] = val ? new Date(val[10] === 'T' ? val : +val) : null; break; case 5: // Object - val = data.line[pos]; + val = line; if (esc && val) val = val.replace(REGTUNESCAPE, regtescapereverse); + if (self.$size && obj[key].indexOf('\\u') !== -1) + obj[key] = obj[key].fromUnicode(); obj[key] = val ? val.parseJSON(true) : null; break; } @@ -6408,21 +6495,67 @@ TP.stringify = function(doc, insert, byteslen) { switch (meta.type) { case 1: // String - val = val ? val : ''; - size += 4; + + if (self.$size) { + switch (typeof(val)) { + case 'number': + val = val + ''; + break; + case 'boolean': + val = val ? '1' : '0'; + break; + case 'object': + var is = !!val; + val = JSON.stringify(val); + if (!is) + val = val.toUnicode(); + break; + case 'string': + val = val.toUnicode(); + break; + } + + if (val.length > meta.size) + val = val.substring(0, meta.size); + else + val = val.padRight(meta.size, ' '); + + // bytes + var diff = meta.size - Buffer.byteLength(val); + if (diff > 0) { + for (var j = 0; j < diff; j++) + val += ' '; + } + + } else { + val = val ? val : ''; + if (meta.size && val.length > meta.sized) + val = val.substring(0, meta.size); + size += 4; + } + break; case 2: // Number - val = (val || 0); - size += 2; + val = (val || 0) + ''; + if (self.$size) { + if (val.length < meta.size) + val = val.padRight(meta.size, ' '); + } else + size += 2; break; + case 3: // Boolean val = (val == true ? '1' : '0'); break; + case 4: // Date - // val = val ? val.toISOString() : ''; val = val ? val instanceof Date ? val.getTime() : val : ''; - !val && (size += 13); + if (self.$size) + val = (val + '').padRight(meta.size, ' '); + else if (!val) + size += 10; break; + case 5: // Object val = val ? JSON.stringify(val) : ''; size += 4; @@ -6440,7 +6573,9 @@ TP.stringify = function(doc, insert, byteslen) { output += '|' + val; } - if (doc.$$alloc) { + if (self.$size && (insert || byteslen)) { + output += '|'; + } else if (doc.$$alloc) { var l = output.length; var a = doc.$$alloc; if (l <= a.length) { @@ -6665,7 +6800,7 @@ NoSQLReader.prototype.compare = function(docs) { item.counter++; - if (!b.$inlinesort && !item.done) + if (b.$options.notall && !b.$inlinesort && !item.done) item.done = b.$options.take && b.$options.take <= item.counter; if (b.$options.readertype) @@ -6729,7 +6864,7 @@ NoSQLReader.prototype.compare = function(docs) { break; } - if (item.first) { + if (item.first || item.done) { item.canceled = true; self.canceled++; } diff --git a/nosqlstream.js b/nosqlstream.js index 6665e067e..4c39f619e 100644 --- a/nosqlstream.js +++ b/nosqlstream.js @@ -46,6 +46,8 @@ function NoSQLStream(filename) { this.remchar = '-'; this.buffercount = BUFFERDOCS; this.buffersize = BUFFERSIZE; + this.linesize = 0; + this.start = 0; // this.canceled = false; // this.docs = ''; // this.docscount = 0; @@ -81,44 +83,84 @@ NoSQLStream.prototype.readhelpers = function() { } else self.buffer = chunk; - var index = self.buffer.indexOf(NEWLINEBUFFER, beg); - while (index !== -1) { + if (self.linesize) { - var tmp = self.buffer.toString('utf8', 0, index); - if (tmp[0] === self.remchar) { - self.buffer = self.buffer.slice(index + 1); - index = self.buffer.indexOf(NEWLINEBUFFER); - if (index === -1) - break; - continue; + while (self.buffer.length >= self.linesize) { + + var tmp = self.buffer.toString('utf8', 0, self.linesize - 1); + if (tmp[0] === self.remchar) { + self.buffer = self.buffer.slice(self.linesize); + if (self.buffer.length > self.linesize) + continue; + else + break; + } + + if (self.array) + self.docs.push(tmp); + else + self.docs += (self.docs ? self.divider : '') + tmp; + + self.docscount++; + self.indexer++; + + if (self.docscount >= self.buffercount) { + if (self.ondocuments() === false) + self.canceled = true; + self.docs = self.array ? [] : ''; + self.docscount = 0; + if (self.canceled) { + self.read(self.$callback, self.$noclose); + return; + } + } + + self.buffer = self.buffer.slice(self.linesize); } - self.docs += (self.docs ? self.divider : '') + tmp; - self.docscount++; - self.indexer++; + } else { + var index = self.buffer.indexOf(NEWLINEBUFFER, beg); + while (index !== -1) { + + var tmp = self.buffer.toString('utf8', 0, index); + if (tmp[0] === self.remchar) { + self.buffer = self.buffer.slice(index + 1); + index = self.buffer.indexOf(NEWLINEBUFFER); + if (index === -1) + break; + continue; + } - if (self.docscount >= self.buffercount) { + if (self.array) + self.docs.push(tmp); + else + self.docs += (self.docs ? self.divider : '') + tmp; - if (self.ondocuments() === false) - self.canceled = true; + self.docscount++; + self.indexer++; - self.docs = ''; - self.docscount = 0; + if (self.docscount >= self.buffercount) { - if (self.canceled) { - self.read(self.$callback, self.$noclose); - return; + if (self.ondocuments() === false) + self.canceled = true; + + self.docs = self.array ? [] : ''; + self.docscount = 0; + + if (self.canceled) { + self.read(self.$callback, self.$noclose); + return; + } } - } - self.buffer = self.buffer.slice(index + 1); - index = self.buffer.indexOf(NEWLINEBUFFER); - if (index === -1) - break; + self.buffer = self.buffer.slice(index + 1); + index = self.buffer.indexOf(NEWLINEBUFFER); + if (index === -1) + break; + } } self.ticks++; - if (self.ticks % 5 === 0) setImmediate(self.cb_readticks); else @@ -144,38 +186,78 @@ NoSQLStream.prototype.readhelpers = function() { } else self.buffer = chunk; - var index = self.buffer.lastIndexOf(NEWLINEBUFFER, self.buffer.length - 2); + if (self.linesize) { - while (index !== -1) { + while (self.buffer.length >= self.linesize) { - var tmp = self.buffer.toString('utf8', index); - if (tmp[1] === self.remchar) { - self.buffer = self.buffer.slice(0, index); - index = self.buffer.lastIndexOf(NEWLINEBUFFER); - if (index === -1) - break; - continue; - } + var tmp = self.buffer.toString('utf8', self.buffer.length - self.linesize, self.buffer.length - 1); + if (tmp[0] === self.remchar) { + self.buffer = self.buffer.slice(0, self.buffer.length - self.linesize); + if (self.buffer.length > self.linesize) + continue; + else + break; + } - self.docs += (self.docs ? self.divider : '') + tmp.trim(); - self.docscount++; + if (self.array) + self.docs.push(tmp); + else + self.docs += (self.docs ? self.divider : '') + tmp; - if (self.docscount >= self.buffercount) { - if (self.ondocuments() === false) - self.canceled = true; - self.docs = ''; - self.docscount = 0; - } + self.docscount++; + self.indexer++; - if (self.canceled) { - self.readreverse2(); - return; + if (self.docscount >= self.buffercount) { + if (self.ondocuments() === false) + self.canceled = true; + self.docs = self.array ? [] : ''; + self.docscount = 0; + if (self.canceled) { + self.read(self.$callback, self.$noclose); + return; + } + } + + self.buffer = self.buffer.slice(0, self.buffer.length - self.linesize); } - self.buffer = self.buffer.slice(0, index); - index = self.buffer.lastIndexOf(NEWLINEBUFFER, self.buffer.length - 2); - if (index === -1) - break; + } else { + var index = self.buffer.lastIndexOf(NEWLINEBUFFER, self.buffer.length - 2); + while (index !== -1) { + + var tmp = self.buffer.toString('utf8', index); + if (tmp[1] === self.remchar) { + self.buffer = self.buffer.slice(0, index); + index = self.buffer.lastIndexOf(NEWLINEBUFFER); + if (index === -1) + break; + continue; + } + + if (self.array) + self.docs.push(tmp.trim()); + else + self.docs += (self.docs ? self.divider : '') + tmp.trim(); + + self.docscount++; + + if (self.docscount >= self.buffercount) { + if (self.ondocuments() === false) + self.canceled = true; + self.docs = self.array ? [] : ''; + self.docscount = 0; + } + + if (self.canceled) { + self.readreverse2(); + return; + } + + self.buffer = self.buffer.slice(0, index); + index = self.buffer.lastIndexOf(NEWLINEBUFFER, self.buffer.length - 2); + if (index === -1) + break; + } } self.ticks++; @@ -199,7 +281,6 @@ NoSQLStream.prototype.readhelpers = function() { self.cache[0] = self.buffer; self.cache[1] = chunk; self.buffer = Buffer.concat(self.cache); - beg = self.cache[0].length - 1; if (beg < 0) @@ -208,34 +289,68 @@ NoSQLStream.prototype.readhelpers = function() { } else self.buffer = chunk; - var index = self.buffer.indexOf(NEWLINEBUFFER, beg); - while (index !== -1) { + if (self.linesize) { + while (self.buffer.length >= self.linesize) { + var tmp = self.buffer.toString('utf8', 0, self.linesize - 1); - var tmp = self.buffer.toString('utf8', 0, index).trim(); + if (self.array) + self.docs.push(tmp); + else + self.docs += (self.docs ? self.divider : '') + tmp; - self.docs += (self.docs ? self.divider : '') + tmp; - self.docscount++; - self.indexer++; + self.docscount++; + self.indexer++; - if (self.docscount >= self.buffercount) { + if (self.docscount >= self.buffercount) { - if (self.ondocuments() === false) - self.canceled = true; + if (self.ondocuments() === false) + self.canceled = true; - self.docs = ''; - self.docscount = 0; + self.docs = self.array ? [] : ''; + self.docscount = 0; - if (self.canceled) { - self.stream.destroy && self.stream.destroy(); - return; + if (self.canceled) { + self.stream.destroy && self.stream.destroy(); + return; + } } + self.buffer = self.buffer.slice(self.linesize); } + } else { + var index = self.buffer.indexOf(NEWLINEBUFFER, beg); + while (index !== -1) { + + var tmp = self.buffer.toString('utf8', 0, index).trim(); + + if (self.array) + self.docs.push(tmp); + else + self.docs += (self.docs ? self.divider : '') + tmp; + + + self.docscount++; + self.indexer++; + + if (self.docscount >= self.buffercount) { - self.buffer = self.buffer.slice(index + 1); - index = self.buffer.indexOf(NEWLINEBUFFER); - if (index === -1) - break; + if (self.ondocuments() === false) + self.canceled = true; + + self.docs = self.array ? [] : ''; + self.docscount = 0; + + if (self.canceled) { + self.stream.destroy && self.stream.destroy(); + return; + } + } + + self.buffer = self.buffer.slice(index + 1); + index = self.buffer.indexOf(NEWLINEBUFFER); + if (index === -1) + break; + } } }; @@ -305,6 +420,7 @@ NoSQLStream.prototype.writehelpers = function() { self.position += size; var beg = 0; + var index; if (self.buffer) { self.cache[0] = self.buffer; @@ -319,32 +435,66 @@ NoSQLStream.prototype.writehelpers = function() { } else self.buffer = chunk; - var index = self.buffer.indexOf(NEWLINEBUFFER, beg); + if (self.linesize) { + while (self.buffer.length >= self.linesize) { + + var tmp = self.buffer.toString('utf8', 0, self.linesize - 1); + + if (self.array) + self.docs.push(tmp); + else + self.docs += (self.docs ? self.divider : '') + tmp; - while (index !== -1) { - var tmp = self.buffer.toString('utf8', 0, index); - if (tmp[0] !== self.remchar) { - self.docs += (self.docs ? self.divider : '') + tmp; - self.docsbuffer.push({ length: index, doc: tmp, position: self.positionupdate }); + self.docsbuffer.push({ length: self.linesize - 1, doc: tmp, position: self.positionupdate }); self.docscount++; + if (self.docsbuffer.length >= self.buffercount) { if (self.ondocuments() === false) self.canceled = true; self.docsbuffer = []; - self.docs = ''; + self.docs = self.array ? [] : ''; if (self.canceled) break; } + + self.positionupdate += self.linesize; + self.buffer = self.buffer.slice(self.linesize); } - self.positionupdate += Buffer.byteLength(tmp, 'utf8') + 1; - self.buffer = self.buffer.slice(index + 1); - index = self.buffer.indexOf(NEWLINEBUFFER); - if (index === -1) - break; + } else { + + index = self.buffer.indexOf(NEWLINEBUFFER, beg); + while (index !== -1) { + var tmp = self.buffer.toString('utf8', 0, index); + + if (tmp[0] !== self.remchar) { + + if (self.array) + self.docs.push(tmp); + else + self.docs += (self.docs ? self.divider : '') + tmp; + + self.docsbuffer.push({ length: index, doc: tmp, position: self.positionupdate }); + self.docscount++; + if (self.docsbuffer.length >= self.buffercount) { + if (self.ondocuments() === false) + self.canceled = true; + self.docsbuffer = []; + self.docs = self.array ? [] : ''; + if (self.canceled) + break; + } + } + + self.positionupdate += Buffer.byteLength(tmp, 'utf8') + 1; + self.buffer = self.buffer.slice(index + 1); + index = self.buffer.indexOf(NEWLINEBUFFER); + if (index === -1) + break; + } } if (self.bufferstack.length || self.bufferstacknew.length) @@ -364,7 +514,7 @@ NoSQLStream.prototype.writehelpers = function() { NoSQLStream.prototype.openread = function() { var self = this; self.type = 'r'; - self.position = 0; + self.position = self.start; self.open(); return self; }; @@ -372,7 +522,7 @@ NoSQLStream.prototype.openread = function() { NoSQLStream.prototype.openreadreverse = function() { var self = this; self.type = 'r'; - self.position = 0; + self.position = self.start; self.$reverse = true; self.open(); return self; @@ -390,7 +540,7 @@ NoSQLStream.prototype.openupdate = function() { Fs.fstat(fd, function(err, stats) { - self.docs = ''; + self.docs = self.array ? [] : ''; self.docscount = 0; if (err) { @@ -399,13 +549,13 @@ NoSQLStream.prototype.openupdate = function() { return; } - self.docs = ''; + self.docs = self.array ? [] : ''; self.docscount = 0; self.fd = fd; self.stats = stats; - self.position = 0; + self.position = self.start; self.positionappend = self.stats.size; - self.positionupdate = 0; + self.positionupdate = self.start; self.bufferstack = []; self.bufferstacknew = []; self.docsbuffer = []; @@ -426,13 +576,13 @@ NoSQLStream.prototype.openstream = function(stream) { if (self.docscount) { self.ondocuments(); self.docscount = 0; - self.docs = ''; + self.docs = self.array ? [] : ''; } self.$callback && self.$callback(); self.$callback = null; }; - self.docs = ''; + self.docs = self.array ? [] : ''; self.docscount = 0; self.readhelpers(); self.stream = stream; @@ -452,11 +602,11 @@ NoSQLStream.prototype.open = function() { } Fs.fstat(fd, function(err, stats) { - self.docs = ''; + self.docs = self.array ? [] : ''; self.docscount = 0; self.fd = fd; self.stats = stats; - self.position = 0; + self.position = self.start; if (err) { Fs.close(fd, NOOP); @@ -553,6 +703,7 @@ NoSQLStream.prototype.read = function() { var self = this; var size = self.stats.size - self.position; + if (!self.fd || size <= 0 || self.canceled) { if (!self.canceled && self.buffer && self.buffer.length) { @@ -564,7 +715,7 @@ NoSQLStream.prototype.read = function() { if (self.docscount) { self.ondocuments(); self.docscount = 0; - self.docs = ''; + self.docs = self.array ? [] : ''; } self.close(); @@ -586,7 +737,7 @@ NoSQLStream.prototype.readreverse = function() { NoSQLStream.prototype.readreverse2 = function() { var self = this; - if (!self.fd || self.position <= 0 || self.canceled) { + if (!self.fd || self.position <= self.start || self.canceled) { if (!self.canceled && self.buffer && self.buffer.length) { self.cb_readreversebuffer(null, 1, NEWLINEBUFFER); @@ -596,7 +747,7 @@ NoSQLStream.prototype.readreverse2 = function() { if (self.docscount) { self.ondocuments(); - self.docs = ''; + self.docs = self.array ? [] : ''; self.docscount = 0; } @@ -628,7 +779,7 @@ NoSQLStream.prototype.readupdate = function() { if (self.docsbuffer.length) { self.ondocuments(); self.docsbuffer = []; - self.docs = ''; + self.docs = self.array ? [] : ''; } self.flush(); From ebed53f34a52b5407077780c50ecc5330e37a7c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 28 Oct 2018 23:42:02 +0100 Subject: [PATCH 152/217] Added `F.refresh()`. --- changes.txt | 1 + index.js | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/changes.txt b/changes.txt index fd543d663..f3e4ad3e5 100755 --- a/changes.txt +++ b/changes.txt @@ -20,6 +20,7 @@ - added: `SchemaOptions` and `OperationOptions` supports `$.cancel()` method - added: `CACHE(name, [value], [expire], [persistent])` alias for `F.cache.get2()` and `F.cache.set()` or `F.cache.set2()` - added: encryption of config values +- added: `F.refresh()` for refreshing of internal cache - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) diff --git a/index.js b/index.js index cc37797ff..daa680dfe 100755 --- a/index.js +++ b/index.js @@ -1023,6 +1023,38 @@ F.dir = function(path) { directory = path; }; +F.refresh = function() { + + NOW = new Date(); + + F.$events.clear && EMIT('clear', 'temporary', F.temporary); + F.temporary.path = {}; + F.temporary.range = {}; + F.temporary.views = {}; + F.temporary.other = {}; + global.$VIEWCACHE && global.$VIEWCACHE.length && (global.$VIEWCACHE = []); + + // Clears command cache + Image.clear(); + + CONF.allow_debug && F.consoledebug('clear temporary cache'); + + var keys = Object.keys(F.temporary.internal); + for (var i = 0; i < keys.length; i++) + if (!F.temporary.internal[keys[i]]) + delete F.temporary.internal[keys[i]]; + + F.$events.clear && EMIT('clear', 'resources'); + F.resources = {}; + CONF.allow_debug && F.consoledebug('clear resources'); + + F.$events.clear && EMIT('clear', 'dns'); + U.clearDNS(); + CONF.allow_debug && F.consoledebug('clear DNS cache'); + + return F; +}; + F.prototypes = function(fn) { var proto = {}; proto.Chunker = framework_utils.Chunker.prototype; From ae4cbc1d94b5a319009748dd45062dbd367010c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 29 Oct 2018 09:37:26 +0100 Subject: [PATCH 153/217] Improved NoSQLReader. --- nosql.js | 119 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 82 insertions(+), 37 deletions(-) diff --git a/nosql.js b/nosql.js index bf414d818..021875a50 100755 --- a/nosql.js +++ b/nosql.js @@ -2206,6 +2206,24 @@ function DatabaseBuilder(db) { DatabaseBuilder.prototype.promise = promise; +DatabaseBuilder.prototype.reset = function() { + var self = this; + var reader = self.$nosqlreader; + if (reader) { + for (var i = 0; i < reader.builders.length; i++) { + var item = reader.builders[i]; + if (item.builder === self) { + item.response = null; + item.scalar = null; + item.counter = 0; + item.count = 0; + item.scalarcount = 0; + } + } + } + return self; +}; + DatabaseBuilder.prototype.makefilter = function() { return { repository: this.$repository, options: this.$options, arg: this.$args, fn: this.$functions }; }; @@ -6721,6 +6739,7 @@ NoSQLReader.prototype.add = function(builder, noTrimmer) { item.compare = builder.compile(noTrimmer); item.filter = builder.makefilter(); item.first = builder.$options.first && !builder.$options.sort; + builder.$nosqlreader = self; self.builders.push(item); } return self; @@ -6872,55 +6891,81 @@ NoSQLReader.prototype.compare = function(docs) { } }; -NoSQLReader.prototype.done = function() { - +NoSQLReader.prototype.reset = function() { var self = this; for (var i = 0; i < self.builders.length; i++) { - var item = self.builders[i]; - var builder = item.builder; - var output; - var opt = builder.$options; - - if (opt.scalar || !opt.sort) { - if (opt.scalar) - output = opt.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; - else if (opt.first) - output = item.response ? item.response[0] : undefined; - else if (opt.listing) - output = listing(builder, item); - else - output = item.response || []; - builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count); - continue; - } + item.canceled = false; + item.response = null; + item.scalar = null; + item.counter = 0; + item.count = 0; + item.scalarcount = 0; + } + self.canceled = 0; + return self; +}; - if (item.count) { - if (opt.sort.name) { - if (!builder.$inlinesort || opt.take !== item.response.length) - item.response.quicksort(opt.sort.name, opt.sort.asc); - } else if (opt.sort === null) - item.response.random(); - else - item.response.sort(opt.sort); - - if (opt.skip && opt.take) - item.response = item.response.splice(opt.skip, opt.take); - else if (opt.skip) - item.response = item.response.splice(opt.skip); - else if (!builder.$inlinesort && opt.take) - item.response = item.response.splice(0, opt.take); - } +NoSQLReader.prototype.callback = function(item) { + + var self = this; + var builder = item.builder; + var output; + var opt = builder.$options; - if (opt.first) + if (item.canceled) { + item.canceled = false; + if (self.canceled) + self.canceled--; + } + + if (opt.scalar || !opt.sort) { + if (opt.scalar) + output = opt.scalar === 'avg' ? item.scalar / item.scalarcount : item.scalar; + else if (opt.first) output = item.response ? item.response[0] : undefined; else if (opt.listing) output = listing(builder, item); else output = item.response || []; - builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count); + return self; + } + + if (item.count) { + if (opt.sort.name) { + if (!builder.$inlinesort || opt.take !== item.response.length) + item.response.quicksort(opt.sort.name, opt.sort.asc); + } else if (opt.sort === null) + item.response.random(); + else + item.response.sort(opt.sort); + + if (opt.skip && opt.take) + item.response = item.response.splice(opt.skip, opt.take); + else if (opt.skip) + item.response = item.response.splice(opt.skip); + else if (!builder.$inlinesort && opt.take) + item.response = item.response.splice(0, opt.take); } + + if (opt.first) + output = item.response ? item.response[0] : undefined; + else if (opt.listing) + output = listing(builder, item); + else + output = item.response || []; + + builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count); + return self; +}; + +NoSQLReader.prototype.done = function() { + var self = this; + for (var i = 0; i < self.builders.length; i++) + self.callback(self.builders[i]); + self.canceled = 0; + return self; }; exports.NoSQLReader = NoSQLReader; \ No newline at end of file From f32ab1a71cb112be9af6b513e4b73aacd30847be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 29 Oct 2018 09:43:18 +0100 Subject: [PATCH 154/217] Improved DataReader. --- utils.js | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/utils.js b/utils.js index b6b7a9d3a..a2131098f 100755 --- a/utils.js +++ b/utils.js @@ -6166,19 +6166,15 @@ exports.Callback = function(count, callback) { function Reader() {} const RP = Reader.prototype; -RP.clear = function() { - +RP.done = function() { var self = this; - var builders = self.reader.builders; - - for (var i = 0; i < builders.length; i++) { - var builder = builders[i]; - builder.scalarcount = 0; - builder.count = 0; - builder.counter = 0; - builder.response = null; - } + self.reader.done(); + return self; +}; +RP.reset = function() { + var self = this; + self.reader.reset(); return self; }; From eaf8961776db1563c2d26050946354cf70a884ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 30 Oct 2018 08:53:56 +0100 Subject: [PATCH 155/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b9cace217..2cbd58c99 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-22", + "version": "3.0.1-23", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 71ef13ad510cdb61aacc306252ad7085176f87e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 30 Oct 2018 16:31:41 +0100 Subject: [PATCH 156/217] Added `DatabaseBuilder.each()`. --- nosql.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/nosql.js b/nosql.js index 021875a50..aee23a65b 100755 --- a/nosql.js +++ b/nosql.js @@ -3075,7 +3075,23 @@ DatabaseBuilder.prototype.prepare = function(fn) { !self.$scope && self.$code.push('if(!$is)return;'); } - return this; + return self; +}; + +DatabaseBuilder.prototype.each = function(fn) { + var self = this; + + if (!self.$functions) + self.$functions = []; + + var index = self.$functions.push(fn) - 1; + + if (!self.$iscache) { + var code = '$tmp=fn[{0}].call($F,doc,index,repository);'.format(index); + self.$code.push(code); + } + + return self; }; function Counter(db) { From a7dad6900517c331f680e06f4ba4ad67ba6c4ac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 30 Oct 2018 16:31:52 +0100 Subject: [PATCH 157/217] Added `CONF.bundling`. --- bundles.js | 15 ++++++++++++++- changes.txt | 2 ++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/bundles.js b/bundles.js index 0263dc3ec..c900a0431 100755 --- a/bundles.js +++ b/bundles.js @@ -3,16 +3,17 @@ require('./index'); const Fs = require('fs'); const Path = require('path'); const CONSOLE = process.argv.indexOf('restart') === -1; -const META = {}; const INTERNAL = { '/sitemap': 1, '/versions': 1, '/workflows': 1, '/dependencies': 1, '/config': 1, '/config-release': 1, '/config-debug': 1 }; const isWindows = require('os').platform().substring(0, 3).toLowerCase() === 'win'; const REGAPPEND = /\/--[a-z0-9]+/i; +const META = {}; META.version = 1; META.created = new Date(); META.total = 'v' + F.version_header; META.node = F.version_node; META.files = []; +META.skip = false; META.directories = []; META.ignore = () => true; @@ -46,6 +47,11 @@ exports.make = function(callback) { async.push(cleanFiles); + async.push(function(next) { + META.skip && (async.length = 0); + next(); + }); + async.push(function(next) { var target = F.path.root(CONF.directory_src); U.ls(F.path.root(CONF.directory_bundles), function(files) { @@ -208,6 +214,13 @@ function cleanFiles(callback) { try { meta = U.parseJSON(Fs.readFileSync(Path.join(path, 'bundle.json')).toString('utf8'), true) || {}; + + if (CONF.bundling === 'shallow') { + META.skip = true; + callback(); + return; + } + } catch (e) { meta = {}; } diff --git a/changes.txt b/changes.txt index f3e4ad3e5..8492563e7 100755 --- a/changes.txt +++ b/changes.txt @@ -17,10 +17,12 @@ - added: a new config item `default-cors : https://www.totaljs.com, https://www.componentator.com` which allows originators for `CORS()` method - added: a new config item `default-request-maxkeys : 33` for restricting query max. keys - added: a new config item `logger : false` which enables logging for Middleware, Schemas and Operations +- added: a new config item `bundling : shallow` which enables shallow bundling (if `bundle.json` exists then the bundles won't be extracted) - added: `SchemaOptions` and `OperationOptions` supports `$.cancel()` method - added: `CACHE(name, [value], [expire], [persistent])` alias for `F.cache.get2()` and `F.cache.set()` or `F.cache.set2()` - added: encryption of config values - added: `F.refresh()` for refreshing of internal cache +- added: `DatabaseBuilder.each(fn)` for browsing of evaluated records - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) From aab692b84c8e55f07490e4eee5dfad463d02a63b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 30 Oct 2018 17:41:41 +0100 Subject: [PATCH 158/217] Fixed NoSQL `repository` in the `callback`. --- nosql.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/nosql.js b/nosql.js index aee23a65b..c817b858f 100755 --- a/nosql.js +++ b/nosql.js @@ -2940,6 +2940,7 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { if (cache) { self.$mappers = cache.mitems; self.$mappersexec = cache.mexec; + self.$each = cache.each; return cache.filter; } } @@ -2956,10 +2957,14 @@ DatabaseBuilder.prototype.compile = function(noTrimmer) { self.$mappersexec = new Function('doc', 'item', tmp); } + if (opt.each) + self.$each = new Function('item', 'doc', 'repository', 'R', opt.each.join('')); + var cache = {}; cache.filter = new Function('doc', '$F', 'index', code); cache.mexec = self.$mappersexec; cache.mitems = self.$mappers; + cache.each = self.$each; CACHE[key] = cache; return cache.filter; }; @@ -3087,8 +3092,11 @@ DatabaseBuilder.prototype.each = function(fn) { var index = self.$functions.push(fn) - 1; if (!self.$iscache) { - var code = '$tmp=fn[{0}].call($F,doc,index,repository);'.format(index); - self.$code.push(code); + var code = 'item.filter.fn[{0}].call(item,doc,item.filter.repository,item.filter.repository);'.format(index); + if (self.$options.each) + self.$options.each.push(code); + else + self.$options.each = [code]; } return self; @@ -6841,6 +6849,7 @@ NoSQLReader.prototype.compare = function(docs) { if (b.$options.readertype) continue; + b.$each && b.$each(item, output); b.$mappersexec && b.$mappersexec(output, item); var val; @@ -6944,7 +6953,7 @@ NoSQLReader.prototype.callback = function(item) { output = listing(builder, item); else output = item.response || []; - builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count); + builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count, item.filter.repository); return self; } @@ -6972,7 +6981,7 @@ NoSQLReader.prototype.callback = function(item) { else output = item.response || []; - builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count); + builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count, item.filter.repository); return self; }; From 6dee6487809381fbb8af7ac0880b7797299c5ef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 30 Oct 2018 20:31:04 +0100 Subject: [PATCH 159/217] Fixed bugs in `repository`. --- nosql.js | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/nosql.js b/nosql.js index c817b858f..028283f0b 100755 --- a/nosql.js +++ b/nosql.js @@ -1585,15 +1585,15 @@ DP.$update = function() { for (var i = 0; i < filters.builders.length; i++) { var item = filters.builders[i]; var fil = filter[i]; - if (fil.insert && !item.count) { - item.builder.$insertcallback && item.builder.$insertcallback(fil.insert, item.builder.$repository || EMPTYOBJECT); + if (fil.insert && !item.counter) { + item.builder.$insertcallback && item.builder.$insertcallback(fil.insert, item.filter.repository || EMPTYOBJECT); var tmp = self.insert(fil.insert); tmp.$callback = item.builder.$callback; tmp.$options.log = item.builder.$options.log; item.builder.$callback = null; } else { item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.repository); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.counter), item.counter, item.count, item.filter.repository); } } @@ -1688,15 +1688,15 @@ DP.$update_inmemory = function() { for (var i = 0; i < filter.length; i++) { var item = filter[i]; - if (item.insert && !item.count) { - item.builder.$insertcallback && item.builder.$insertcallback(item.insert, item.builder.$repository || EMPTYOBJECT); + if (item.insert && !item.counter) { + item.builder.$insertcallback && item.builder.$insertcallback(item.insert, item.filter.repository || EMPTYOBJECT); var tmp = self.insert(item.insert); tmp.$callback = item.builder.$callback; tmp.$options.log = item.builder.$options.log; item.builder.$callback = null; } else { item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.repository); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.counter), item.counter, item.count, item.filter.repository); } } @@ -6100,14 +6100,14 @@ TP.$update = function() { var item = filters.builders[i]; var fil = filter[i]; if (fil.insert && !item.count) { - item.builder.$insertcallback && item.builder.$insertcallback(fil.insert, item.builder.$repository || EMPTYOBJECT); + item.builder.$insertcallback && item.builder.$insertcallback(fil.insert, item.filter.repository); var tmp = self.insert(fil.insert); tmp.$callback = item.builder.$callback; tmp.$options.log = item.builder.$options.log; item.builder.$callback = null; } else { item.builder.$options.log && item.builder.log(); - item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.count), item.count, item.repository); + item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.counter), item.counter, item.count, item.filter.repository); } } @@ -6170,9 +6170,6 @@ TP.$remove = function() { var lines = fs.docs; var arr = []; - if (!indexer) - arr.push(EMPTYOBJECT); - for (var a = 0; a < lines.length; a++) { data.line = lines[a].split('|'); data.index = indexer++; @@ -6771,6 +6768,7 @@ NoSQLReader.prototype.add = function(builder, noTrimmer) { NoSQLReader.prototype.compare2 = function(docs, custom, done) { var self = this; + for (var i = 0; i < docs.length; i++) { var doc = docs[i]; @@ -6793,9 +6791,19 @@ NoSQLReader.prototype.compare2 = function(docs, custom, done) { if (!output) continue; - item.is = false; - !is && (is = true); + // WTF? + // item.is = false; + item.count++; + + if ((item.builder.$options.skip && item.builder.$options.skip >= item.count) || (item.builder.$options.take && item.builder.$options.take <= item.counter)) + continue; + + !is && (is = true); + + item.counter++; + item.builder.$each && item.builder.$each(item, output); + var canceled = item.canceled; var c = custom(docs, output, i, item, j); @@ -6953,7 +6961,7 @@ NoSQLReader.prototype.callback = function(item) { output = listing(builder, item); else output = item.response || []; - builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count, item.filter.repository); + builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.counter : output, item.count, item.filter.repository); return self; } @@ -6981,7 +6989,7 @@ NoSQLReader.prototype.callback = function(item) { else output = item.response || []; - builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.count : output, item.count, item.filter.repository); + builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.counter : output, item.count, item.filter.repository); return self; }; From 09005b567f455a9dade65ece60355f3e4bba0696 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 30 Oct 2018 21:20:04 +0100 Subject: [PATCH 160/217] Fixed `TABLE` cleaning. --- nosql.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nosql.js b/nosql.js index 028283f0b..1aa23d547 100755 --- a/nosql.js +++ b/nosql.js @@ -6213,6 +6213,8 @@ TP.$clean = function() { var fs = new NoSQLStream(self.filename); var writer = Fs.createWriteStream(self.filename + '-tmp'); + writer.write(self.stringifySchema() + NEWLINE); + fs.start = self.$header; fs.linesize = self.$size; fs.divider = NEWLINE; @@ -6456,10 +6458,8 @@ TP.parseData = function(data, cache) { var esc = data.line[0] === '*'; var val, alloc; - if (cache && data.keys.length === data.line.length - 2) { - // alloc = data.line[data.line.length - 1].length - 1; + if (cache && !self.$size && data.keys.length === data.line.length - 2) alloc = data.line[data.line.length - 1].length; - } for (var i = 0; i < data.keys.length; i++) { var key = data.keys[i]; @@ -6802,7 +6802,7 @@ NoSQLReader.prototype.compare2 = function(docs, custom, done) { !is && (is = true); item.counter++; - item.builder.$each && item.builder.$each(item, output); + item.builder.$each && item.builder.$each(item, doc); var canceled = item.canceled; var c = custom(docs, output, i, item, j); From f44d4e67d2f7ce7b8fbbe3be9623901df5358b9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 30 Oct 2018 21:20:52 +0100 Subject: [PATCH 161/217] Updated beta. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2cbd58c99..237b9a13e 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-23", + "version": "3.0.1-24", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 89cdc9e9f456446cd6492d7b07164e9440f935a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 1 Nov 2018 15:45:00 +0100 Subject: [PATCH 162/217] Fixed `versions` with `auto` value. --- index.js | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index daa680dfe..3117672ec 100755 --- a/index.js +++ b/index.js @@ -2587,6 +2587,8 @@ F.routing = function(name) { */ global.MERGE = F.merge = function(url) { + F.temporary.other['merge_' + url] = 1; + if (url[0] === '#') url = sitemapurl(url.substring(1)); @@ -5244,6 +5246,9 @@ F.onMapping = function(url, def, ispublic, encode) { return F.components.files[name] && F.components.files[name][tmp.substring(index + 1)] ? (F.path.temp() + tmp.substring(1)) : null; } + if (F.routes.mapping[url]) + return F.routes.mapping[url]; + if (F._length_themes) { var index = tmp.indexOf('/', 2); if (index !== -1) { @@ -5253,9 +5258,6 @@ F.onMapping = function(url, def, ispublic, encode) { } } - if (F.routes.mapping[url]) - return F.routes.mapping[url]; - def = framework_internal.preparePath(def, true); if (encode) @@ -8591,7 +8593,22 @@ F.$configure_versions = function(arr, clean) { if (hash) { var index = key.lastIndexOf('.'); filename = key.substring(0, index) + '-' + hash + key.substring(index); + F.versions[key] = filename; + + if (!F.routes.merge[key] && !F.temporary.other['merge_' + key]) { + var index = key.indexOf('/', 1); + var theme = index === -1 ? null : key.substring(1, index); + if (theme) { + if (F.themes[theme]) + key = F.themes[theme] + 'public' + key.substring(index); + else + key = F.path.public(key); + } else + key = F.path.public(key); + F.map(filename, key); + } + F.temporary.views = {}; F.temporary.other = {}; global.$VIEWCACHE = []; From 8b9a4ae49d50b062305b9863426708730cb5ef16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 1 Nov 2018 15:47:14 +0100 Subject: [PATCH 163/217] Updated beta version. --- changes.txt | 1 + package.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 8492563e7..191ade48d 100755 --- a/changes.txt +++ b/changes.txt @@ -39,6 +39,7 @@ - fixed: `LOCALIZE()` for nested directories - fixed: sending of error handling when WebSocketClient is starting (for example: `unauthorized`) - fixed: `versions` and `auto` feature with enabled `F.wait()` +- fixed: `versions` and `auto` feature with direct link to file - fixed: `LOAD('release')` a release mode - fixed: `SchemaInstance.$clean()` for nested schemas - fixed: extracting `bundles` (added `/flow/` and `/dashboard/`) diff --git a/package.json b/package.json index 237b9a13e..f06817e1c 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-24", + "version": "3.0.1-25", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 59dbcc4b71db1d8be1dac42194f250a0f426d39f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 1 Nov 2018 18:14:47 +0100 Subject: [PATCH 164/217] Added bundle downloading. --- bundles.js | 4 ++++ changes.txt | 1 + index.js | 47 +++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/bundles.js b/bundles.js index c900a0431..f78443399 100755 --- a/bundles.js +++ b/bundles.js @@ -57,6 +57,10 @@ exports.make = function(callback) { U.ls(F.path.root(CONF.directory_bundles), function(files) { var dirs = {}; files.wait(function(filename, resume) { + + if (!filename.endsWith('.bundle')) + return resume(); + var dbpath = CONF.directory_databases; F.restore(filename, target, resume, function(p, dir) { diff --git a/changes.txt b/changes.txt index 191ade48d..0aa5a3116 100755 --- a/changes.txt +++ b/changes.txt @@ -23,6 +23,7 @@ - added: encryption of config values - added: `F.refresh()` for refreshing of internal cache - added: `DatabaseBuilder.each(fn)` for browsing of evaluated records +- added: Bundles can be downloaded from URL address - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) diff --git a/index.js b/index.js index 3117672ec..d1ac27c1f 100755 --- a/index.js +++ b/index.js @@ -3574,15 +3574,54 @@ F.modify = function(fn) { F.$bundle = function(callback) { + var bundledir = F.path.root(CONF.directory_bundles); + var makebundle = function() { - require('./bundles').make(function() { - F.directory = HEADERS.workers.cwd = directory = F.path.root(CONF.directory_src); - callback(); + + var arr = Fs.readdirSync(bundledir); + var url = []; + + for (var i = 0; i < arr.length; i++) { + if (arr[i].endsWith('.url')) + url.push(arr[i]); + } + + url.wait(function(item, next) { + + var filename = F.path.root(CONF.directory_bundles) + item.replace('.url', '.bundle'); + var link = Fs.readFileSync(F.path.root(CONF.directory_bundles) + item).toString('utf8'); + + F.consoledebug('Download bundle: ' + link); + + U.download(link, FLAGS_INSTALL, function(err, response) { + + if (err) { + F.error(err, 'Bundle: ' + link); + next(); + return; + } + + var stream = Fs.createWriteStream(filename); + + response.pipe(stream); + response.on('error', function(err) { + F.error(err, 'Bundle: ' + link); + next(); + }); + + CLEANUP(stream, next); + }); + + }, function() { + require('./bundles').make(function() { + F.directory = HEADERS.workers.cwd = directory = F.path.root(CONF.directory_src); + callback(); + }); }); }; try { - Fs.statSync(F.path.root(CONF.directory_bundles)); + Fs.statSync(bundledir); if (F.$bundling) { makebundle(); return; From 0607bcb821fc81a38f425133656a244d02d3ca84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 1 Nov 2018 20:59:09 +0100 Subject: [PATCH 165/217] Fix grammar. --- changes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 0aa5a3116..f1e84a94d 100755 --- a/changes.txt +++ b/changes.txt @@ -23,7 +23,7 @@ - added: encryption of config values - added: `F.refresh()` for refreshing of internal cache - added: `DatabaseBuilder.each(fn)` for browsing of evaluated records -- added: Bundles can be downloaded from URL address +- added: Bundles can be downloaded from URL addresses - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) From aa232192549d1a6f2ce0efe3d6282687885cf7b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 2 Nov 2018 09:08:40 +0100 Subject: [PATCH 166/217] Fixed NoSQL in-memory. --- nosql.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nosql.js b/nosql.js index 1aa23d547..598191604 100755 --- a/nosql.js +++ b/nosql.js @@ -1686,8 +1686,8 @@ DP.$update_inmemory = function() { change && self.$save(); - for (var i = 0; i < filter.length; i++) { - var item = filter[i]; + for (var i = 0; i < filters.builders.length; i++) { + var item = filters.builders[i]; if (item.insert && !item.counter) { item.builder.$insertcallback && item.builder.$insertcallback(item.insert, item.filter.repository || EMPTYOBJECT); var tmp = self.insert(item.insert); @@ -6961,6 +6961,7 @@ NoSQLReader.prototype.callback = function(item) { output = listing(builder, item); else output = item.response || []; + builder.$callback2(errorhandling(null, builder, output), opt.readertype === 1 ? item.counter : output, item.count, item.filter.repository); return self; } From 657e5c3fa52e656162f516d57abcaa9187f52ad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 2 Nov 2018 12:37:57 +0100 Subject: [PATCH 167/217] Fixed bundles. --- debug.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debug.js b/debug.js index dd83e6d97..12b8da8e9 100644 --- a/debug.js +++ b/debug.js @@ -258,6 +258,14 @@ function runwatching() { var ticks = stat.mtime.getTime(); if (files[filename] != null && files[filename] !== ticks) { + + if (filename.endsWith('.bundle') && files[filename.replace(/\.bundle$/, '.url')]) { + // Bundle from URL address + files[filename] = ticks; + next(); + return; + } + var log = stamp.replace('#', files[filename] === 0 ? 'ADD' : 'UPD') + prefix + normalize(filename.replace(directory, '')); if (files[filename]) { var tmp = isViewPublic(filename); From 9df8c5b475c3ab5ebb33ed530b71c03918777ee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 2 Nov 2018 13:12:13 +0100 Subject: [PATCH 168/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f06817e1c..5b7b9a641 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-25", + "version": "3.0.1-26", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From ea694da798cedef9d0e9de000093cc4d0c2e6efd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 2 Nov 2018 20:12:59 +0100 Subject: [PATCH 169/217] Updated joins. --- changes.txt | 3 ++- nosql.js | 44 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/changes.txt b/changes.txt index f1e84a94d..da63a84b1 100755 --- a/changes.txt +++ b/changes.txt @@ -29,7 +29,8 @@ - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) - updated: routing, now it supports operations in the form `ROUTE('.. * --> @save_operation @load_operation (response)')` - updated: `ROUTE()` supports multiple HTTP method declaration `ROUTE('GET,POST,PUT /something/', action)` -- updated: `REQUEST()` can return binary data if the content-type is not `text/*` or `application/*`. +- updated: `REQUEST()` can return binary data if the content-type is not `text/*` or `application/*` +- updated: NoSQL joins support array values - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator diff --git a/nosql.js b/nosql.js index 598191604..aacfcd67a 100755 --- a/nosql.js +++ b/nosql.js @@ -2262,8 +2262,15 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { for (var i = 0; i < response.length; i++) { var item = response[i]; var val = item[join.b]; - if (val !== undefined && unique.indexOf(val) === -1) - unique.push(val); + if (val !== undefined) { + if (val instanceof Array) { + for (var j = 0; j < val.length; j++) { + if (unique.indexOf(val[j]) === -1) + unique.push(val[j]); + } + } else if (unique.indexOf(val) === -1) + unique.push(val); + } } } else if (response) { var val = response[join.b]; @@ -2341,8 +2348,8 @@ DatabaseBuilder.prototype.$callback2 = function(err, response, count, repository self.$callbackjoin(function() { var keys = Object.keys(self.$join); - var jl = keys.length; + if (response instanceof Array) { for (var i = 0, length = response.length; i < length; i++) { var item = response[i]; @@ -2368,23 +2375,44 @@ DatabaseBuilder.prototype.$callback2 = function(err, response, count, repository function findItem(items, field, value) { for (var i = 0, length = items.length; i < length; i++) { - if (items[i][field] === value) + if (value instanceof Array) { + for (var j = 0; j < value.length; j++) { + if (items[i][field] === value[j]) + return items[i]; + } + } else if (items[i][field] === value) return items[i]; } } function findScalar(items, value) { + var sum = null; for (var i = 0, length = items.length; i < length; i++) { - if (items[i].id === value) - return items[i].response || null; + var item = items[i]; + if (value instanceof Array) { + for (var j = 0; j < value.length; j++) { + if (item.id === value[j]) { + sum = sum == null ? item.response : (sum + item.response); + break; + } + } + } else if (item.id === value) + sum = sum == null ? item.response : (sum + item.response); } - return null; + return sum; } function findItems(items, field, value) { var arr = []; for (var i = 0, length = items.length; i < length; i++) { - if (items[i][field] === value) + if (value instanceof Array) { + for (var j = 0; j < value.length; j++) { + if (items[i][field] === value[j]) { + arr.push(items[i]); + break; + } + } + } else if (items[i][field] === value) arr.push(items[i]); } return arr; From 694a4ee8d16b3228af87515dd983fe185f2ed13b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 5 Nov 2018 15:59:45 +0100 Subject: [PATCH 170/217] Added `ONCE()`. --- changes.txt | 1 + index.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index da63a84b1..595390fcc 100755 --- a/changes.txt +++ b/changes.txt @@ -24,6 +24,7 @@ - added: `F.refresh()` for refreshing of internal cache - added: `DatabaseBuilder.each(fn)` for browsing of evaluated records - added: Bundles can be downloaded from URL addresses +- added: `ONCE()` alias to `F.once()` - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) diff --git a/index.js b/index.js index d1ac27c1f..366894b36 100755 --- a/index.js +++ b/index.js @@ -1148,7 +1148,7 @@ global.EMIT = F.emit = function(name, a, b, c, d, e, f, g) { return F; }; -F.once = function(name, fn) { +global.ONCE = F.once = function(name, fn) { fn.$once = true; return F.on(name, fn); }; From 156dfd0c656232fb304be517cbc324a3eb9e5ef2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 5 Nov 2018 18:19:18 +0100 Subject: [PATCH 171/217] Fixed DateTime in NoSQL backups. --- nosql.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nosql.js b/nosql.js index aacfcd67a..72eaa611e 100755 --- a/nosql.js +++ b/nosql.js @@ -2167,7 +2167,7 @@ DatabaseBuilder2.prototype.log = function(msg, user) { var self = this; if (msg) { NOW = new Date(); - self.$options.log = (self.$options.log ? self.$options.log : '') + NOW.format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; + self.$options.log = (self.$options.log ? self.$options.log : '') + NOW.toUTC().format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; } else if (self.$options.log) { self.db.filenameLog && Fs.appendFile(self.db.filenameLog, self.$options.log, F.errorcallback); self.$options.log = ''; @@ -2242,7 +2242,7 @@ DatabaseBuilder.prototype.log = function(msg, user) { var self = this; if (msg) { NOW = new Date(); - self.$options.log = (self.$options.log ? self.$options.log : '') + NOW.format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; + self.$options.log = (self.$options.log ? self.$options.log : '') + NOW.toUTC().format('yyyy-MM-dd HH:mm:ss') + ' | ' + (user ? user.padRight(20) + ' | ' : '') + msg + NEWLINE; } else if (self.$options.log) { self.db.filenameLog && Fs.appendFile(self.db.filenameLog, self.$options.log, F.errorcallback); self.$options.log = ''; @@ -2570,7 +2570,7 @@ DatabaseBuilder.prototype.backup = function(user) { }; DatabaseBuilder.prototype.$backupdoc = function(doc) { - this.db.filenameBackup && Fs.appendFile(this.db.filenameBackup, NOW.format('yyyy-MM-dd HH:mm') + ' | ' + this.$options.backup.padRight(20) + ' | ' + (typeof(doc) === 'string' ? doc : JSON.stringify(doc)) + NEWLINE, F.errorcallback); + this.db.filenameBackup && Fs.appendFile(this.db.filenameBackup, NOW.toUTC().format('yyyy-MM-dd HH:mm') + ' | ' + this.$options.backup.padRight(20) + ' | ' + (typeof(doc) === 'string' ? doc : JSON.stringify(doc)) + NEWLINE, F.errorcallback); return this; }; From 7e6ee9cdd811b4ff9af2effa35cfbd0de2d4220e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 6 Nov 2018 20:03:48 +0100 Subject: [PATCH 172/217] Improved `ROUTING()` method. --- changes.txt | 1 + index.js | 59 +++++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/changes.txt b/changes.txt index 595390fcc..60b8d9c3e 100755 --- a/changes.txt +++ b/changes.txt @@ -32,6 +32,7 @@ - updated: `ROUTE()` supports multiple HTTP method declaration `ROUTE('GET,POST,PUT /something/', action)` - updated: `REQUEST()` can return binary data if the content-type is not `text/*` or `application/*` - updated: NoSQL joins support array values +- updated: `ROUTING(id:|search, [flags])` method - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator diff --git a/index.js b/index.js index 366894b36..3a5313efb 100755 --- a/index.js +++ b/index.js @@ -311,7 +311,6 @@ global.CREATE = (group, name) => framework_builders.getschema(group, name).defau global.SCRIPT = (body, value, callback, param) => F.script(body, value, callback, param); global.SINGLETON = (name, def) => SINGLETONS[name] || (SINGLETONS[name] = (new Function('return ' + (def || '{}')))()); global.FUNCTION = (name) => F.functions[name] || NOOP; -global.ROUTING = (name) => F.routing(name); global.SCHEDULE = (date, each, fn, param) => F.schedule(date, each, fn, param); global.FINISHED = framework_internal.onFinished; global.DESTROY = framework_internal.destroyStream; @@ -1896,7 +1895,6 @@ global.GROUP = F.group = function() { * @param {Number} timeout Response timeout. * @return {Framework} */ - global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, language) { var name; @@ -1922,6 +1920,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu flags = tmp; } + var search = (typeof(url) === 'string' ? url.toLowerCase().replace(/\s{2,}/g, ' ') : '') + (flags ? (' ' + flags.where(n => typeof(n) === 'string' && n.substring(0, 2) !== '//' && n[2] !== ':').join(' ')).toLowerCase() : ''); var method = ''; var CUSTOM = typeof(url) === 'function' ? url : null; if (CUSTOM) @@ -2463,6 +2462,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu var instance = new FrameworkRoute(); var r = instance.route; r.hash = hash; + r.search = search.split(' '); r.id = id; r.name = name.trim(); r.groups = flags_to_object(groups); @@ -2516,6 +2516,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu r.regexp = reg; r.regexpIndexer = regIndex; r.type = 'web'; + r.remove = remove_route_web; if (r.isUPLOAD) PERF.upload = true; @@ -2559,20 +2560,62 @@ function flags_to_object(flags) { return obj; } +function remove_route_web() { + + if (this.isSYSTEM) { + var keys = Object.keys(F.routes.system); + for (var i = 0; i < keys.length; i++) { + if (F.routes.system[keys[i]] === this) { + delete F.routes.system[keys]; + F.temporary.other = {}; + return; + } + } + } + + var index = F.routes.web.indexOf(this); + if (index !== -1) { + F.routes.web.splice(index, 1); + F.$routesSort(); + F.temporary.other = {}; + } +} + /** * Get routing by name * @param {String} name * @return {Object} */ -F.routing = function(name) { +global.ROUTING = F.routing = function(name, flags) { + + var id = name.substring(0, 3) === 'id:' ? name.substring(3) : null; + if (id) + name = null; + + var search = id ? null : (name.toLowerCase().replace(/\s{2,}/g, ' ') + (flags ? (' ' + flags.where(n => typeof(n) === 'string' && n.substring(0, 2) !== '//' && n[2] !== ':').join(' ')).toLowerCase() : '')).split(' '); + for (var i = 0, length = F.routes.web.length; i < length; i++) { var route = F.routes.web[i]; - if (route.name === name) { - var url = U.path(route.url.join('/')); - if (url[0] !== '/') - url = '/' + url; - return { controller: route.controller, url: url, id: route.id, flags: route.flags, middleware: route.middleware, execute: route.execute, timeout: route.timeout, options: route.options, length: route.length }; + var is = true; + if (id && route.id !== id) + is = false; + else if (search) { + for (var j = 0; j < search.length; j++) { + if (route.search.indexOf(search[j]) === -1) { + is = false; + break; + } + } } + + if (!is) + continue; + + var url = U.path(route.url.join('/')); + if (url[0] !== '/') + url = '/' + url; + + return route; } }; From b82e83b1af792443fdd5176d715b1ad6f2b276d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 7 Nov 2018 18:13:44 +0100 Subject: [PATCH 173/217] Improved code. --- internal.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/internal.js b/internal.js index c9b54755e..234126582 100755 --- a/internal.js +++ b/internal.js @@ -171,7 +171,6 @@ exports.parseMULTIPART = function(req, contentType, route, tmpDirectory) { } header = parse_multipart_header(header); - tmp.$step = 1; tmp.$is = header[1] !== null; tmp.name = header[0]; @@ -1413,11 +1412,14 @@ MultipartParser.prototype.write = function(buffer) { for (i = 0; i < len; i++) { c = buffer[i]; switch (state) { + case S.PARSER_UNINITIALIZED: return i; + case S.START: index = 0; state = S.START_BOUNDARY; + case S.START_BOUNDARY: if (index == boundary.length - 2) { if (c === HYPHEN) @@ -1445,10 +1447,12 @@ MultipartParser.prototype.write = function(buffer) { if (c === boundary[index + 2]) index++; break; + case S.HEADER_FIELD_START: state = S.HEADER_FIELD; mark('headerField'); index = 0; + case S.HEADER_FIELD: if (c === CR) { clear('headerField'); @@ -1472,12 +1476,15 @@ MultipartParser.prototype.write = function(buffer) { cl = lower(c); if (cl < A || cl > Z) return i; + break; + case S.HEADER_VALUE_START: if (c === SPACE) break; mark('headerValue'); state = S.HEADER_VALUE; + case S.HEADER_VALUE: if (c === CR) { dataCallback('headerValue', true); @@ -1485,23 +1492,26 @@ MultipartParser.prototype.write = function(buffer) { state = S.HEADER_VALUE_ALMOST_DONE; } break; + case S.HEADER_VALUE_ALMOST_DONE: if (c !== LF) return i; state = S.HEADER_FIELD_START; break; + case S.HEADERS_ALMOST_DONE: if (c !== LF) return i; callback('headersEnd'); state = S.PART_DATA_START; break; + case S.PART_DATA_START: state = S.PART_DATA; mark('partData'); + case S.PART_DATA: prevIndex = index; - if (!index) { // boyer-moore derrived algorithm to safely skip non-boundary data i += boundaryEnd; @@ -1566,8 +1576,10 @@ MultipartParser.prototype.write = function(buffer) { i--; } break; + case S.END: break; + default: return i; } From cebce48e809490489ffd01023631e4cde0d0274c Mon Sep 17 00:00:00 2001 From: Tema Smirnov Date: Wed, 7 Nov 2018 21:22:00 +0300 Subject: [PATCH 174/217] Fix workers crash when debugging main process with --inspect or --debug --- index.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/index.js b/index.js index 3a5313efb..04619f4cd 100755 --- a/index.js +++ b/index.js @@ -868,6 +868,16 @@ function Framework() { self.isLE = Os.endianness ? Os.endianness() === 'LE' : true; self.isHTTPS = false; + // Fix for workers crash (port in use) when debugging main process with --inspect or --debug + // See: https://github.com/nodejs/node/issues/14325 and https://github.com/nodejs/node/issues/9435 + process.execArgv.forEach((val, i, arr) => { + if (val.indexOf('inspect') != -1 || val.indexOf('debug') != -1) { + // Setting inspect/debug port to random unused + arr[i] = '--inspect=0'; + } + }); + HEADERS.workers.execArgv = process.execArgv; + // It's hidden // self.waits = {}; From 8753b903b1a68576cafb3572d9cb35f11919cefa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 7 Nov 2018 19:47:56 +0100 Subject: [PATCH 175/217] Improved code. --- index.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/index.js b/index.js index 04619f4cd..a035ec2a0 100755 --- a/index.js +++ b/index.js @@ -870,12 +870,14 @@ function Framework() { // Fix for workers crash (port in use) when debugging main process with --inspect or --debug // See: https://github.com/nodejs/node/issues/14325 and https://github.com/nodejs/node/issues/9435 - process.execArgv.forEach((val, i, arr) => { - if (val.indexOf('inspect') != -1 || val.indexOf('debug') != -1) { - // Setting inspect/debug port to random unused - arr[i] = '--inspect=0'; + for (var i = 0; i < process.execArgv.length; i++) { + // Setting inspect/debug port to random unused + if ((/inspect|debug/).test(process.execArgv[i])) { + process.execArgv[i] = '--inspect=0'; + break; } - }); + } + HEADERS.workers.execArgv = process.execArgv; // It's hidden From 37a306601ba26269ce94cbbaa5aa7eecd792a03b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 7 Nov 2018 19:49:35 +0100 Subject: [PATCH 176/217] Added new change. --- changes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changes.txt b/changes.txt index 60b8d9c3e..5e828e991 100755 --- a/changes.txt +++ b/changes.txt @@ -59,6 +59,7 @@ - fixed: `controller.autoclear()` - fixed: `controller.proxy()` - fixed: `repeat` mode in `SCHEDULE()` +- fixed: `--inspect` argument for Workers - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute From adeaf108fc311ba29c5cfb2e12fe8ca9e6a71803 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 7 Nov 2018 19:50:10 +0100 Subject: [PATCH 177/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5b7b9a641..fd1f69d69 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-26", + "version": "3.0.1-27", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 32195fde9b4667ecaee16941c23e323c08dc6b6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 8 Nov 2018 21:58:15 +0100 Subject: [PATCH 178/217] Fixed `ROUTE()`. --- index.js | 14 ++++++++------ package.json | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/index.js b/index.js index a035ec2a0..818258b84 100755 --- a/index.js +++ b/index.js @@ -1939,18 +1939,20 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu url = '/'; if (url) { + url = url.replace(/\t/g, ' '); - var index = url.indexOf(' '); - if (index !== -1) { - method = url.substring(0, index).toLowerCase().trim(); - url = url.substring(index + 1).trim(); - } url = url.replace(/(^|\s?)\*([a-z0-9]|\s).*?$/i, function(text) { !flags && (flags = []); flags.push(text.trim()); return ''; - }); + }).trim(); + + var index = url.indexOf(' '); + if (index !== -1) { + method = url.substring(0, index).toLowerCase().trim(); + url = url.substring(index + 1).trim(); + } if (method.indexOf(',') !== -1) { !flags && (flags = []); diff --git a/package.json b/package.json index fd1f69d69..40259b217 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-27", + "version": "3.0.1-28", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From f365c08162c814427c767f2d9d4adefc026ccde5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 9 Nov 2018 20:28:47 +0100 Subject: [PATCH 179/217] Added `image.define()`. --- changes.txt | 1 + image.js | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 5e828e991..1aae99a03 100755 --- a/changes.txt +++ b/changes.txt @@ -25,6 +25,7 @@ - added: `DatabaseBuilder.each(fn)` for browsing of evaluated records - added: Bundles can be downloaded from URL addresses - added: `ONCE()` alias to `F.once()` +- added: `image.define(value)` performs `convert -define 'value'` - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) diff --git a/image.js b/image.js index 2b8edeb30..022dd8dfa 100755 --- a/image.js +++ b/image.js @@ -21,7 +21,7 @@ /** * @module FrameworkImage - * @version 3.0.0 + * @version 3.1.0 */ 'use strict'; @@ -667,6 +667,10 @@ Image.prototype.flop = function() { return this.push('-flop', null, 10); }; +Image.prototype.define = function(value) { + return this.push('-define', value, 10, true); +}; + Image.prototype.minify = function() { return this.push('+profile', '*', null, 10, true); }; From 21bbaa15601cd9a05fa41d95a97c18efa53c9b27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 9 Nov 2018 20:28:51 +0100 Subject: [PATCH 180/217] Removed useless file. --- tools/merge.sh | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 tools/merge.sh diff --git a/tools/merge.sh b/tools/merge.sh deleted file mode 100644 index c03a40689..000000000 --- a/tools/merge.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -cd "$DIR" -cd .. -cd merged -echo "MERGING" -node merge.js -echo "UGLIFY" -node merge.js --minify -echo "DONE" \ No newline at end of file From d44093a271ae6f07752401aac069ac6cc849b2bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 10 Nov 2018 12:59:10 +0100 Subject: [PATCH 181/217] Improved code. --- index.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/index.js b/index.js index 818258b84..8dccfb8d7 100755 --- a/index.js +++ b/index.js @@ -6429,9 +6429,7 @@ F.exists = function(req, res, max, callback) { var name = req.$key = createTemporaryKey(req); var filename = F.path.temp(name); - var httpcachevalid = false; - - RELEASE && (req.headers['if-none-match'] === ETAG + CONF.etag_version) && (httpcachevalid = true); + var httpcachevalid = RELEASE && (req.headers['if-none-match'] === (ETAG + CONF.etag_version)); if (F.isProcessed(name) || httpcachevalid) { res.options.filename = filename; From 76fabb834e552cc90e8fe98ab6ed73d8e99fdfec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 11 Nov 2018 00:38:13 +0100 Subject: [PATCH 182/217] Fixed SMTP TLS. --- mail.js | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/mail.js b/mail.js index dc4eca0af..9448c66fa 100755 --- a/mail.js +++ b/mail.js @@ -21,7 +21,7 @@ /** * @module FrameworkMail - * @version 3.0.0 + * @version 3.1.0 */ 'use strict'; @@ -36,6 +36,8 @@ const REG_STATE = /\d+/; const REG_WINLINE = /\r\n/g; const REG_NEWLINE = /\n/g; const REG_AUTH = /(AUTH LOGIN|AUTH PLAIN)/i; +const REG_TLS = /TLS/; +const REG_STARTTLS = /STARTTLS/; const EMPTYARRAY = []; var INDEXSENDER = 0; @@ -577,6 +579,7 @@ Mailer.prototype.send = function(smtp, options, messages, callback) { return self; } + obj.smtpoptions = options; obj.socket.$host = smtp; obj.host = smtp.substring(smtp.lastIndexOf('.', smtp.lastIndexOf('.') - 1) + 1); obj.socket.on('error', function(err) { @@ -811,20 +814,28 @@ Mailer.prototype.$send = function(obj, options, autosend) { switch (code) { case 220: - if (obj.isTLS) { + if (obj.isTLS || REG_TLS.test(line)) { mailer.switchToTLS(obj, options); - return; + } else { + obj.secured = REG_ESMTP.test(line); + command = obj.isTLS || (options.user && options.password) || obj.secured ? 'EHLO' : 'HELO'; + mailer.$writeline(obj, command + ' ' + host); } - command = obj.isTLS || (options.user && options.password) || REG_ESMTP.test(line) ? 'EHLO' : 'HELO'; - mailer.$writeline(obj, command + ' ' + host); - break; + return; case 250: // OPERATION case 251: // FORWARD case 235: // VERIFY case 999: // Total.js again + if (obj.secured && !obj.isTLS && !obj.logged && obj.smtpoptions.user && obj.smtpoptions.password) { + // maybe TLS + obj.isTLS = true; + mailer.$writeline(obj, 'STARTTLS'); + return; + } + mailer.$writeline(obj, buffer.shift()); if (buffer.length) @@ -857,6 +868,7 @@ Mailer.prototype.$send = function(obj, options, autosend) { var value = auth.shift(); if (value) { + obj.logged = true; mailer.$writeline(obj, value); } else { var err = new Error('Forbidden.'); @@ -879,6 +891,12 @@ Mailer.prototype.$send = function(obj, options, autosend) { if (code < 400) return; + if (!obj.isTLS && code === 530 && REG_STARTTLS.test(line)) { + obj.isTLS = true; + mailer.$writeline(obj, 'STARTTLS'); + return; + } + var err = new Error(line); mailer.$events.error && !obj.try && mailer.emit('error', err, obj); From 0775ec42501e95c2e26f22b3a6506bf42d42c524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 11 Nov 2018 00:38:26 +0100 Subject: [PATCH 183/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 40259b217..b8873567e 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-28", + "version": "3.0.1-29", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 824688b845412029de003696bb9fe2f5c319ca1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 11 Nov 2018 00:42:04 +0100 Subject: [PATCH 184/217] Added new changes. --- changes.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 1aae99a03..4c439f748 100755 --- a/changes.txt +++ b/changes.txt @@ -60,7 +60,8 @@ - fixed: `controller.autoclear()` - fixed: `controller.proxy()` - fixed: `repeat` mode in `SCHEDULE()` -- fixed: `--inspect` argument for Workers +- fixed: `--inspect` argument for Workers by Tema Smirnov +- fixed: TLS in SMTP mail sender - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute From 7d8aa46f54c34ee3aea7f798a5f3ce2b502825d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 13 Nov 2018 13:29:42 +0100 Subject: [PATCH 185/217] Improved NoSQL joins. --- nosql.js | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/nosql.js b/nosql.js index 72eaa611e..cebc7b96e 100755 --- a/nosql.js +++ b/nosql.js @@ -2256,7 +2256,7 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { var join = self.$join[key]; var response = self.$response; - var unique = []; + var unique = new Set(); if (response instanceof Array && response.length) { for (var i = 0; i < response.length; i++) { @@ -2264,18 +2264,16 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { var val = item[join.b]; if (val !== undefined) { if (val instanceof Array) { - for (var j = 0; j < val.length; j++) { - if (unique.indexOf(val[j]) === -1) - unique.push(val[j]); - } - } else if (unique.indexOf(val) === -1) - unique.push(val); + for (var j = 0; j < val.length; j++) + unique.add(val[j]); + } else + unique.add(val); } } } else if (response) { var val = response[join.b]; - if (val !== undefined && unique.indexOf(val) === -1) - unique.push(val); + if (val !== undefined) + unique.add(val); } var isTable = self.db instanceof Table; @@ -2283,8 +2281,8 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { if (join.scalartype) { join.items = []; - join.count = unique.length; - for (var i = 0; i < unique.length; i++) { + join.count = unique.size; + for (var m of unique.values()) { (function(val) { var builder = db.scalar(join.scalartype, join.scalarfield).callback(function(err, response) { join.items.push({ id: val, response: response }); @@ -2309,20 +2307,20 @@ DatabaseBuilder.prototype.$callbackjoin = function(callback) { builder.$scope = join.builder.$scope; builder.where(join.a, val); - })(unique[i]); + })(m); } } else { - if (unique.length) { + if (unique.size) { join.builder.$options.fields && join.builder.$options.fields.push(join.a); join.builder.$callback = function(err, docs) { join.items = docs; next(); }; if (isTable) - db.find(join.builder).in(join.a, unique); + db.find(join.builder).in(join.a, Array.from(unique)); else - db.find(join.builder).in(join.a, unique); + db.find(join.builder).in(join.a, Array.from(unique)); } else { join.items = join.builder.$options.first ? null : []; next(); From c93c5b5be3379391ca9861564de7eba339c3128f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 13 Nov 2018 22:31:41 +0100 Subject: [PATCH 186/217] Updated `F.path.mkdir()` by adding a cache(). --- changes.txt | 1 + index.js | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/changes.txt b/changes.txt index 4c439f748..021908e1a 100755 --- a/changes.txt +++ b/changes.txt @@ -34,6 +34,7 @@ - updated: `REQUEST()` can return binary data if the content-type is not `text/*` or `application/*` - updated: NoSQL joins support array values - updated: `ROUTING(id:|search, [flags])` method +- updated: `F.path.mkdir()` currently caches a value - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator diff --git a/index.js b/index.js index 8dccfb8d7..0bb1b283f 100755 --- a/index.js +++ b/index.js @@ -9660,6 +9660,13 @@ FrameworkPath.prototype.verify = function(name) { FrameworkPath.prototype.mkdir = function(p) { + var key = '$directory-' + p; + + if (F.temporary.path[key]) + return F; + + F.temporary.path[key] = true; + var is = F.isWindows; var s = ''; @@ -9683,6 +9690,9 @@ FrameworkPath.prototype.mkdir = function(p) { p = p.substring(0, l); } + if (existsSync(p)) + return F; + var arr = is ? p.replace(/\//g, '\\').split('\\') : p.split('/'); var directory = s; From 58c1c82694254e8be98ec633487d39260b0c023e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 14 Nov 2018 17:52:37 +0100 Subject: [PATCH 187/217] Fixed `versions` regexp. --- changes.txt | 1 + index.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 021908e1a..f5c6a2d45 100755 --- a/changes.txt +++ b/changes.txt @@ -63,6 +63,7 @@ - fixed: `repeat` mode in `SCHEDULE()` - fixed: `--inspect` argument for Workers by Tema Smirnov - fixed: TLS in SMTP mail sender +- fixed: applying of versions - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute diff --git a/index.js b/index.js index 0bb1b283f..420007fc0 100755 --- a/index.js +++ b/index.js @@ -49,7 +49,7 @@ const COMPRESSIONSPECIAL = { 'js': 1, 'css': 1 }; const REG_TEMPORARY = /\//g; const REG_MOBILE = /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini|Mobile|Tablet/i; const REG_ROBOT = /search|agent|bot|crawler|spider/i; -const REG_VERSIONS = /(href|src)="[a-zA-Z0-9/:\-.]+\.(jpg|js|css|png|apng|gif|svg|html|ico|json|less|sass|scss|swf|txt|webp|heif|heic|jpeg|woff|woff2|xls|xlsx|xml|xsl|xslt|zip|rar|csv|doc|docx|eps|gzip|jpe|jpeg|manifest|mov|mp3|flac|mp4|ogg|package|pdf)"/gi; +const REG_VERSIONS = /(href|src)="[a-zA-Z0-9/:\-.\_]+\.(jpg|js|css|png|apng|gif|svg|html|ico|json|less|sass|scss|swf|txt|webp|heif|heic|jpeg|woff|woff2|xls|xlsx|xml|xsl|xslt|zip|rar|csv|doc|docx|eps|gzip|jpe|jpeg|manifest|mov|mp3|flac|mp4|ogg|package|pdf)"/gi; const REG_COMPILECSS = /url\(.*?\)/g; const REG_ROUTESTATIC = /^(\/\/|https:|http:)+/; const REG_NEWIMPL = /^(async\s)?function(\s)?([a-zA-Z$][a-zA-Z0-9$]+)?(\s)?\([a-zA-Z0-9$]+\)|^function anonymous\(\$/; From ef4c435e7b5cad0ebc64f76e9065c3ba527b4cbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 15 Nov 2018 12:57:52 +0100 Subject: [PATCH 188/217] Improved cluster. --- cluster.js | 30 +++++++++++++++++++----------- index.js | 2 +- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/cluster.js b/cluster.js index 95567a852..0c099d669 100644 --- a/cluster.js +++ b/cluster.js @@ -48,23 +48,31 @@ exports.on = function(name, callback) { return F; }; -exports.emit = function(name, data) { - if (Cluster.isMaster) { - CLUSTER_EMIT.name = name; - CLUSTER_EMIT.data = data; +exports.emit = function(name, a, b, c, d, e) { + + CLUSTER_EMIT.name = name; + CLUSTER_EMIT.a = a; + CLUSTER_EMIT.b = b; + CLUSTER_EMIT.c = c; + CLUSTER_EMIT.d = d; + CLUSTER_EMIT.e = e; + + if (Cluster.isMaster) message(CLUSTER_EMIT); - } else if (F.isCluster) { - CLUSTER_EMIT.name = name; - CLUSTER_EMIT.data = data; + else if (F.isCluster) process.send(CLUSTER_EMIT); - } + return F; }; -exports.master = function(name, data) { +exports.master = function(name, a, b, c, d, e) { if (F.isCluster) { CLUSTER_MASTER.name = name; - CLUSTER_MASTER.data = data; + CLUSTER_MASTER.a = a; + CLUSTER_MASTER.b = b; + CLUSTER_MASTER.c = c; + CLUSTER_MASTER.d = d; + CLUSTER_MASTER.e = e; process.send(CLUSTER_MASTER); } return F; @@ -233,7 +241,7 @@ function message(m) { if (m.TYPE === 'master') { if (MASTER && MASTER[m.name]) { for (var i = 0, length = MASTER[m.name].length; i < length; i++) - MASTER[m.name][i](m.data); + MASTER[m.name][i](m.a, m.b, m.c, m.d, m.e); } } else { if (m.target === 'master') { diff --git a/index.js b/index.js index 420007fc0..df2053386 100755 --- a/index.js +++ b/index.js @@ -16772,7 +16772,7 @@ process.on('message', function(msg, h) { msg.TYPE === 'cache-clear' && F.cache.clear(false); msg.TYPE === 'req' && F.cluster.req(msg); msg.TYPE === 'res' && msg.target === F.id && F.cluster.res(msg); - msg.TYPE === 'emit' && F.$events[msg.name] && EMIT(msg.name, msg.data); + msg.TYPE === 'emit' && F.$events[msg.name] && EMIT(msg.name, msg.a, msg.b, msg.c, msg.d, msg.e); msg.TYPE === 'nosql-meta' && NOSQL(msg.name).meta(msg.key, msg.value, true); msg.TYPE === 'table-meta' && TABLE(msg.name).meta(msg.key, msg.value, true); } From 00126637acfda5df5605f19835b5dd5aaef97951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 15 Nov 2018 16:04:04 +0100 Subject: [PATCH 189/217] Added build exceptions for `.file` or `file-bk.ext` --- changes.txt | 1 + index.js | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index f5c6a2d45..26d2083ac 100755 --- a/changes.txt +++ b/changes.txt @@ -26,6 +26,7 @@ - added: Bundles can be downloaded from URL addresses - added: `ONCE()` alias to `F.once()` - added: `image.define(value)` performs `convert -define 'value'` +- added: Total.js JS files (+ packages) tarted with `.` (dot) or ended with `-bk` won't be processed - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) diff --git a/index.js b/index.js index df2053386..433fb7cb2 100755 --- a/index.js +++ b/index.js @@ -3730,8 +3730,9 @@ F.$load = function(types, targetdirectory, callback, packageName) { var ext = U.getExtension(o); if (ext) ext = '.' + ext; - if (ext !== extension) + if (ext !== extension || o[0] === '.' || o.endsWith('-bk' + extension)) return; + var name = (level ? U.$normalize(directory).replace(dir, '') + '/' : '') + o.substring(0, o.length - ext.length); output.push({ name: name[0] === '/' ? name.substring(1) : name, filename: Path.join(dir, name) + extension }); }); From 6e1d809d8ad4073505c8499740421883cdd7325b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 17 Nov 2018 17:21:08 +0100 Subject: [PATCH 190/217] Added `ROUTE()` dynamic schemas. --- builders.js | 31 +++++++++++-- changes.txt | 1 + index.js | 83 +++++++++++++++++++++++++--------- package.json | 2 +- test/controllers/default.js | 6 +++ test/definitions/initialize.js | 4 -- test/schemas/user.js | 21 ++++++++- test/test-framework-debug.js | 73 ++++++++++++++++++++++++------ test/test-framework-release.js | 73 ++++++++++++++++++++++++------ 9 files changed, 237 insertions(+), 57 deletions(-) diff --git a/builders.js b/builders.js index 217f1253d..bdaad14d5 100755 --- a/builders.js +++ b/builders.js @@ -36,7 +36,9 @@ const hasOwnProperty = Object.prototype.hasOwnProperty; const Qs = require('querystring'); var schemas = {}; +var schemasall = {}; var operations = {}; +var schemacache = {}; var transforms = { pagination: {}, error: {}, restbuilder: {} }; function SchemaBuilder(name) { @@ -237,9 +239,10 @@ SchemaBuilder.prototype.remove = function(name) { var schema = this.collection[name]; schema && schema.destroy(); schema = null; + delete schemasall[name.toLowerCase()]; delete this.collection[name]; } else { - delete schemas[this.name]; + exports.remove(this.name); this.collection = null; } }; @@ -1665,6 +1668,7 @@ SchemaBuilderEntity.prototype.make = function(model, filter, callback, argument, } var builder = this.validate(output, undefined, undefined, undefined, filter); + if (builder.hasError()) { this.onError && this.onError(builder, model, 'make'); callback && callback(builder, null, argument); @@ -3169,7 +3173,7 @@ global.EACHSCHEMA = exports.eachschema = function(group, fn) { } }; -exports.getschema = function(group, name, fn, timeout) { +global.$$$ = global.GETSCHEMA = exports.getschema = function(group, name, fn, timeout) { if (!name || typeof(name) === 'function') { fn = name; @@ -3189,6 +3193,10 @@ exports.getschema = function(group, name, fn, timeout) { return g ? g.get(name) : undefined; }; +exports.findschema = function(groupname) { + return schemasall[groupname.toLowerCase()]; +}; + exports.newschema = function(group, name) { if (!group) @@ -3198,7 +3206,11 @@ exports.newschema = function(group, name) { schemas[group] = new SchemaBuilder(group); var o = schemas[group].create(name); + var key = group + '/' + name; + o.owner = F.$owner(); + schemasall[key.toLowerCase()] = o; + return o; }; @@ -3209,10 +3221,23 @@ exports.newschema = function(group, name) { */ exports.remove = function(group, name) { if (name) { + var g = schemas[group || DEFAULT_SCHEMA]; g && g.remove(name); - } else + var key = ((group || DEFAULT_SCHEMA) + '/' + name).toLowerCase(); + delete schemasall[key]; + + } else { + delete schemas[group]; + + var lower = group.toLowerCase(); + + Object.keys(schemasall).forEach(function(key) { + if (key.substring(0, group.length) === lower) + delete schemasall[key]; + }); + } }; global.EACHOPERATION = function(fn) { diff --git a/changes.txt b/changes.txt index 26d2083ac..eb36a952f 100755 --- a/changes.txt +++ b/changes.txt @@ -32,6 +32,7 @@ - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) - updated: routing, now it supports operations in the form `ROUTE('.. * --> @save_operation @load_operation (response)')` - updated: `ROUTE()` supports multiple HTTP method declaration `ROUTE('GET,POST,PUT /something/', action)` +- updated: `ROUTE()` supports dynamic schemas - updated: `REQUEST()` can return binary data if the content-type is not `text/*` or `application/*` - updated: NoSQL joins support array values - updated: `ROUTING(id:|search, [flags])` method diff --git a/index.js b/index.js index 433fb7cb2..42cbcefe6 100755 --- a/index.js +++ b/index.js @@ -306,7 +306,6 @@ global.RESOURCE = (name, key) => F.resource(name, key); global.TRANSLATE = (name, key) => F.translate(name, key); global.TRANSLATOR = (name, text) => F.translator(name, text); global.TRACE = (message, name, uri, ip) => F.trace(message, name, uri, ip); -global.$$$ = global.GETSCHEMA = (group, name, fn, timeout) => framework_builders.getschema(group, name, fn, timeout); global.CREATE = (group, name) => framework_builders.getschema(group, name).default(); global.SCRIPT = (body, value, callback, param) => F.script(body, value, callback, param); global.SINGLETON = (name, def) => SINGLETONS[name] || (SINGLETONS[name] = (new Function('return ' + (def || '{}')))()); @@ -1942,7 +1941,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu url = url.replace(/\t/g, ' '); - url = url.replace(/(^|\s?)\*([a-z0-9]|\s).*?$/i, function(text) { + url = url.replace(/(^|\s?)\*([{}a-z0-9}]|\s).*?$/i, function(text) { !flags && (flags = []); flags.push(text.trim()); return ''; @@ -2066,6 +2065,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu var corsflags = []; var membertype = 0; var isGENERATOR = false; + var isDYNAMICSCHEMA = false; var description; var id = null; var groups = []; @@ -2134,6 +2134,12 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu if (schema.length === 1) { schema[1] = schema[0]; schema[0] = 'default'; + + // Is dynamic schema? + if (schema[1][0] === '{') { + isDYNAMICSCHEMA = true; + schema[1] = schema[1].substring(1, schema[1].length - 1).trim(); + } } index = schema[1].indexOf('#'); @@ -2378,6 +2384,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu var params = []; var reg = null; var regIndex = null; + var dynamicidindex = -1; if (url.indexOf('{') !== -1) { routeURL.forEach(function(o, i) { @@ -2389,6 +2396,9 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu var sub = o.substring(1, o.length - 1); var name = o.substring(1, o.length - 1).trim(); + if (name === 'id') + dynamicidindex = i; + params.push(name); if (sub[0] !== '/') @@ -2492,6 +2502,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu r.urlraw = urlraw; r.url = routeURL; r.param = arr; + r.paramidindex = dynamicidindex; r.paramnames = params.length ? params : null; r.flags = flags || EMPTYARRAY; r.flags2 = flags_to_object(flags); @@ -2525,6 +2536,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu r.isCACHE = !url.startsWith('/#') && !CUSTOM && !arr.length && !isWILDCARD; r.isPARAM = arr.length > 0; r.isDELAY = isDELAY; + r.isDYNAMICSCHEMA = isDYNAMICSCHEMA; r.CUSTOM = CUSTOM; r.options = options; r.regexp = reg; @@ -5539,12 +5551,21 @@ F.$onParseQueryUrl = function(req) { * @param {String} name * @param {Function(err, body)} callback */ -F.onSchema = function(req, group, name, callback, filter, novalidate, workflow) { - var schema = GETSCHEMA(group, name); +F.onSchema = function(req, route, callback) { + + var schema; + + if (route.isDYNAMICSCHEMA) { + var index = route.param[route.paramnames.indexOf(route.schema[1])]; + req.$schemaname = route.schema[0] + '/' + req.split[index]; + schema = framework_builders.findschema(req.$schemaname); + } else + schema = GETSCHEMA(route.schema[0], route.schema[1]); + if (schema) - schema.make(req.body, filter, onSchema_callback, callback, novalidate, workflow ? workflow.meta : null); + schema.make(req.body, route.schema[2], onSchema_callback, callback, route.novalidate, route.workflow ? route.workflow.meta : null); else - callback(new Error('Schema "' + group + '/' + name + '" not found.')); + callback('Schema "' + (route.isDYNAMICSCHEMA ? req.$schemaname : (route.schema[0] + '/' + route.schema[1])) + '" not found.'); }; function onSchema_callback(err, res, callback) { @@ -10556,7 +10577,7 @@ Controller.prototype.getSchema = function() { var route = this.route; if (!route.schema || !route.schema[1]) throw new Error('The controller\'s route does not define any schema.'); - var schema = GETSCHEMA(route.schema[0], route.schema[1]); + var schema = route.isDYNAMICSCHEMA ? framework_builders.findschema(route.schema[0] + '/' + this.params[route.schema[1]]) : GETSCHEMA(route.schema[0], route.schema[1]); if (schema) return schema; throw new Error('Schema "{0}" does not exist.'.format(route.schema[1])); @@ -15009,10 +15030,12 @@ function extend_request(PROTO) { return next(self, code); } - F.onSchema(self, self.$total_route.schema[0], self.$total_route.schema[1], function(err, body) { - + F.onSchema(self, self.$total_route, function(err, body) { if (err) { - self.$total_400(err); + if (self.$total_route.isDYNAMICSCHEMA) + self.$total_404(err); + else + self.$total_400(err); next = null; } else { F.stats.request.schema++; @@ -15020,8 +15043,7 @@ function extend_request(PROTO) { self.$total_schema = true; next(self, code); } - - }, route.schema[2], route.novalidate, route.workflow); + }); }; PROTO.$total_authorize = function(isLogged, user, roles) { @@ -15193,6 +15215,12 @@ function extend_request(PROTO) { this.$total_execute(400, true); }; + PROTO.$total_404 = function(problem) { + this.$total_route = F.lookup(this, '#404', EMPTYARRAY, 0); + this.$total_exception = problem; + this.$total_execute(404, true); + }; + PROTO.$total_500 = function(problem) { this.$total_route = F.lookup(this, '#500', EMPTYARRAY, 0); this.$total_exception = problem; @@ -17046,19 +17074,21 @@ function parseComponent(body, filename) { return response; } -function getSchemaName(schema) { - return schema[0] === 'default' ? schema[1] : schema[0] + '/' + schema[1]; +function getSchemaName(schema, params) { + return schema[0] === 'default' ? (params ? params[schema[1]] : schema[1]) : schema[0] + '/' + schema[1]; } // Default action for workflow routing function controller_json_workflow(id) { var self = this; - self.id = id; var w = self.route.workflow; + self.id = self.paramidindex === -1 ? id : self.req.split[self.route.paramidindex]; + CONF.logger && (self.req.$logger = []); if (w instanceof Object) { + if (!w.type) { // IS IT AN OPERATION? @@ -17067,10 +17097,13 @@ function controller_json_workflow(id) { return; } - var schema = GETSCHEMA(self.route.schema[0], self.route.schema[1]); - + var schema = self.route.isDYNAMICSCHEMA ? framework_builders.findschema(self.req.$schemaname || (self.route.schema[0] + '/' + self.params[self.route.schema[1]])) : GETSCHEMA(self.route.schema[0], self.route.schema[1]); if (!schema) { - self.throw500('Schema "{0}" not found.'.format(getSchemaName(self.route.schema))); + var err = 'Schema "{0}" not found.'.format(getSchemaName(self.route.schema, self.route.isDYNAMICSCHEMA ? self.params : null)); + if (self.route.isDYNAMICSCHEMA) + self.throw404(err); + else + self.throw500(err); return; } @@ -17090,6 +17123,7 @@ function controller_json_workflow(id) { w.name = w.id; } } + if (w.name) self[w.type](w.name, self.callback()); else { @@ -17098,16 +17132,22 @@ function controller_json_workflow(id) { else self.throw500('Operation @' + w.id + ' not found.'); } + + if (self.route.isDYNAMICSCHEMA) + w.type = ''; + } else self.$exec(w, null, self.callback()); } // Default action for workflow routing function controller_json_workflow_multiple(id) { + var self = this; - self.id = id; var w = self.route.workflow; + self.id = self.paramidindex === -1 ? id : self.req.split[self.route.paramidindex]; + CONF.logger && (self.req.$logger = []); if (w instanceof Object) { @@ -17119,10 +17159,9 @@ function controller_json_workflow_multiple(id) { return; } - var schema = GETSCHEMA(self.route.schema[0], self.route.schema[1]); - + var schema = self.route.isDYNAMICSCHEMA ? framework_builders.findschema(self.route.schema[0] + '/' + self.params[self.route.schema[1]]) : GETSCHEMA(self.route.schema[0], self.route.schema[1]); if (!schema) { - self.throw500('Schema "{0}" not found.'.format(getSchemaName(self.route.schema))); + self.throw500('Schema "{0}" not found.'.format(getSchemaName(self.route.schema, self.isDYNAMICSCHEMA ? self.params : null))); return; } diff --git a/package.json b/package.json index b8873567e..3d3763e44 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-29", + "version": "3.0.1-30", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", diff --git a/test/controllers/default.js b/test/controllers/default.js index c535d0328..a258d92aa 100755 --- a/test/controllers/default.js +++ b/test/controllers/default.js @@ -101,6 +101,12 @@ exports.install = function() { F.route('/live/', viewLive); F.route('/live/incoming/', viewLiveIncoming, ['mixed']); + ROUTE('GET /api/static/orders/ *Orders --> @query'); + ROUTE('GET /api/static/users/ *Users --> @query'); + ROUTE('POST /api/static/orders/ *Orders --> @save'); + ROUTE('GET /api/dynamic/{schema}/ *{schema} --> @query'); + ROUTE('POST /api/dynamic/{schema}/ *{schema} --> @save'); + F.redirect('http://www.google.sk', 'http://www.petersirka.sk'); F.route('#408', function DEFER() { diff --git a/test/definitions/initialize.js b/test/definitions/initialize.js index 65c2b35a1..8e9aa5a80 100644 --- a/test/definitions/initialize.js +++ b/test/definitions/initialize.js @@ -71,10 +71,6 @@ framework.onPictureUrl = function(dimension, id, width, height, alt) { return dimension + '-' + id + '.jpg'; }; -framework.onValidate = function(name, value) { - return name + value; -}; - // Is read from http://www.totaljs.com/framework/include.js //framework.precompile('precompile.layout', 'http://www.totaljs.com/framework/_layout.html'); //framework.precompile('precompile.homepage', 'http://www.totaljs.com/framework/homepage.html'); \ No newline at end of file diff --git a/test/schemas/user.js b/test/schemas/user.js index b5f66fa17..15a631945 100644 --- a/test/schemas/user.js +++ b/test/schemas/user.js @@ -1 +1,20 @@ -F.global.schemas = 1; \ No newline at end of file +F.global.schemas = 1; + +NEWSCHEMA('Orders', function(schema) { + + schema.define('name', String, true); + + schema.setQuery(function($) { + $.success('orders'); + }); + + schema.setSave(function($) { + $.success('orders'); + }); +}); + +NEWSCHEMA('Users', function(schema) { + schema.setQuery(function($) { + $.success('users'); + }); +}); \ No newline at end of file diff --git a/test/test-framework-debug.js b/test/test-framework-debug.js index 7b814a460..616d946bb 100755 --- a/test/test-framework-debug.js +++ b/test/test-framework-debug.js @@ -5,13 +5,6 @@ var url = 'http://127.0.0.1:8001/'; var errorStatus = 0; var max = 100; -//F.snapshot('/templates/localization.html', '/users/petersirka/desktop/localization.html'); - -// INSTALL('module', 'https://www.totaljs.com/framework/include.js', { test: true }); - -//framework.map('/minify/', '@testpackage', ['.html', 'js']); -//framework.map('/minify/', 'models'); -//framework.map('/minify/', F.path.models()); framework.onCompileView = function(name, html) { return html + 'COMPILED'; }; @@ -822,7 +815,7 @@ function test_routing(next) { }); async.await('theme-green', function(complete) { - utils.request(url + '/green/js/default.js', [], function(error, data, code, headers) { + utils.request(url + 'green/js/default.js', [], function(error, data, code, headers) { if (error) throw error; assert(data === 'var a=1+1;', 'Themes: problem with static files.'); @@ -831,7 +824,7 @@ function test_routing(next) { }); async.await('theme-green-merge', function(complete) { - utils.request(url + '/merge-theme.js', [], function(error, data, code, headers) { + utils.request(url + 'merge-theme.js', [], function(error, data, code, headers) { if (error) throw error; assert(data.indexOf('var a=1+1;') !== -1 && data.indexOf('var b=2+2;'), 'Themes: problem with merging static files'); @@ -840,7 +833,7 @@ function test_routing(next) { }); async.await('theme-green-map', function(complete) { - utils.request(url + '/map-theme.js', [], function(error, data, code, headers) { + utils.request(url + 'map-theme.js', [], function(error, data, code, headers) { if (error) throw error; assert(data === 'var a=1+1;', 'Themes: problem with mapping static files.'); @@ -848,11 +841,65 @@ function test_routing(next) { }); }); - async.await('theme-green', function(complete) { - utils.request(url + 'theme-green/', ['get'], function(error, data, code, headers) { + async.await('dynamic-schema-1', function(complete) { + utils.request(url + 'api/dynamic/orders/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"orders"}', 'Dynamic schemas 1'); + complete(); + }); + }); + + async.await('dynamic-schema-2', function(complete) { + utils.request(url + 'api/dynamic/users/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"users"}', 'Dynamic schemas 2'); + complete(); + }); + }); + + async.await('dynamic-schema-3', function(complete) { + utils.request(url + 'api/dynamic/products/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 404, 'Dynamic schemas 3'); + complete(); + }); + }); + + async.await('dynamic-schema-4', function(complete) { + utils.request(url + 'api/dynamic/orders/', ['post', 'json'], { name: 'Total.js' }, function(error, data, code, headers) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"orders"}', 'Dynamic schemas 4'); + complete(); + }); + }); + + async.await('static-schema-1', function(complete) { + utils.request(url + 'api/static/orders/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"orders"}', 'static schemas 1'); + complete(); + }); + }); + + async.await('static-schema-2', function(complete) { + utils.request(url + 'api/static/users/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"users"}', 'static schemas 2'); + complete(); + }); + }); + + async.await('static-schema-3', function(complete) { + utils.request(url + 'api/static/orders/', ['post', 'json'], { name: 'Total.js' }, function(error, data, code, headers) { if (error) throw error; - console.log('--->', data); + assert(code === 200 && data === '{"success":true,"value":"orders"}', 'Dynamic schemas 3'); complete(); }); }); diff --git a/test/test-framework-release.js b/test/test-framework-release.js index e5a2efb7d..ee19598de 100755 --- a/test/test-framework-release.js +++ b/test/test-framework-release.js @@ -5,13 +5,6 @@ var url = 'http://127.0.0.1:8001/'; var errorStatus = 0; var max = 100; -//F.snapshot('/templates/localization.html', '/users/petersirka/desktop/localization.html'); - -// INSTALL('module', 'https://www.totaljs.com/framework/include.js', { test: true }); - -//framework.map('/minify/', '@testpackage', ['.html', 'js']); -//framework.map('/minify/', 'models'); -//framework.map('/minify/', F.path.models()); framework.onCompileView = function(name, html) { return html + 'COMPILED'; }; @@ -822,7 +815,7 @@ function test_routing(next) { }); async.await('theme-green', function(complete) { - utils.request(url + '/green/js/default.js', [], function(error, data, code, headers) { + utils.request(url + 'green/js/default.js', [], function(error, data, code, headers) { if (error) throw error; assert(data === 'var a=1+1;', 'Themes: problem with static files.'); @@ -831,7 +824,7 @@ function test_routing(next) { }); async.await('theme-green-merge', function(complete) { - utils.request(url + '/merge-theme.js', [], function(error, data, code, headers) { + utils.request(url + 'merge-theme.js', [], function(error, data, code, headers) { if (error) throw error; assert(data.indexOf('var a=1+1;') !== -1 && data.indexOf('var b=2+2;'), 'Themes: problem with merging static files'); @@ -840,7 +833,7 @@ function test_routing(next) { }); async.await('theme-green-map', function(complete) { - utils.request(url + '/map-theme.js', [], function(error, data, code, headers) { + utils.request(url + 'map-theme.js', [], function(error, data, code, headers) { if (error) throw error; assert(data === 'var a=1+1;', 'Themes: problem with mapping static files.'); @@ -848,11 +841,65 @@ function test_routing(next) { }); }); - async.await('theme-green', function(complete) { - utils.request(url + 'theme-green/', ['get'], function(error, data, code, headers) { + async.await('dynamic-schema-1', function(complete) { + utils.request(url + 'api/dynamic/orders/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"orders"}', 'Dynamic schemas 1'); + complete(); + }); + }); + + async.await('dynamic-schema-2', function(complete) { + utils.request(url + 'api/dynamic/users/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"users"}', 'Dynamic schemas 2'); + complete(); + }); + }); + + async.await('dynamic-schema-3', function(complete) { + utils.request(url + 'api/dynamic/products/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 404, 'Dynamic schemas 3'); + complete(); + }); + }); + + async.await('dynamic-schema-4', function(complete) { + utils.request(url + 'api/dynamic/orders/', ['post', 'json'], { name: 'Total.js' }, function(error, data, code, headers) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"orders"}', 'Dynamic schemas 4'); + complete(); + }); + }); + + async.await('static-schema-1', function(complete) { + utils.request(url + 'api/static/orders/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"orders"}', 'static schemas 1'); + complete(); + }); + }); + + async.await('static-schema-2', function(complete) { + utils.request(url + 'api/static/users/', [], function(error, data, code) { + if (error) + throw error; + assert(code === 200 && data === '{"success":true,"value":"users"}', 'static schemas 2'); + complete(); + }); + }); + + async.await('static-schema-3', function(complete) { + utils.request(url + 'api/static/orders/', ['post', 'json'], { name: 'Total.js' }, function(error, data, code, headers) { if (error) throw error; - console.log('--->', data); + assert(code === 200 && data === '{"success":true,"value":"orders"}', 'Dynamic schemas 3'); complete(); }); }); From 3d4d852bf28423b7a37dbb829e5df0870c45cfb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 18 Nov 2018 20:11:28 +0100 Subject: [PATCH 191/217] Fixed routing. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 42cbcefe6..5f568ca57 100755 --- a/index.js +++ b/index.js @@ -2502,7 +2502,7 @@ global.ROUTE = F.web = F.route = function(url, funcExecute, flags, length, langu r.urlraw = urlraw; r.url = routeURL; r.param = arr; - r.paramidindex = dynamicidindex; + r.paramidindex = isDYNAMICSCHEMA ? dynamicidindex : -1; r.paramnames = params.length ? params : null; r.flags = flags || EMPTYARRAY; r.flags2 = flags_to_object(flags); From d17683fb9a1fec2fc130390ef479a5c140f316e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 18 Nov 2018 20:58:57 +0100 Subject: [PATCH 192/217] Fixed routing. --- index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 5f568ca57..b6645b9e1 100755 --- a/index.js +++ b/index.js @@ -17083,7 +17083,7 @@ function controller_json_workflow(id) { var self = this; var w = self.route.workflow; - self.id = self.paramidindex === -1 ? id : self.req.split[self.route.paramidindex]; + self.id = self.route.paramidindex === -1 ? id : self.req.split[self.route.paramidindex]; CONF.logger && (self.req.$logger = []); @@ -17146,7 +17146,7 @@ function controller_json_workflow_multiple(id) { var self = this; var w = self.route.workflow; - self.id = self.paramidindex === -1 ? id : self.req.split[self.route.paramidindex]; + self.id = self.route.paramidindex === -1 ? id : self.req.split[self.route.paramidindex]; CONF.logger && (self.req.$logger = []); From a9fa0c3c95cb955d907c1af16688815f3e23798c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Mon, 19 Nov 2018 08:35:11 +0100 Subject: [PATCH 193/217] Improved code. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index b6645b9e1..b925c2fdd 100755 --- a/index.js +++ b/index.js @@ -49,7 +49,7 @@ const COMPRESSIONSPECIAL = { 'js': 1, 'css': 1 }; const REG_TEMPORARY = /\//g; const REG_MOBILE = /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini|Mobile|Tablet/i; const REG_ROBOT = /search|agent|bot|crawler|spider/i; -const REG_VERSIONS = /(href|src)="[a-zA-Z0-9/:\-.\_]+\.(jpg|js|css|png|apng|gif|svg|html|ico|json|less|sass|scss|swf|txt|webp|heif|heic|jpeg|woff|woff2|xls|xlsx|xml|xsl|xslt|zip|rar|csv|doc|docx|eps|gzip|jpe|jpeg|manifest|mov|mp3|flac|mp4|ogg|package|pdf)"/gi; +const REG_VERSIONS = /(href|src)="[a-zA-Z0-9/:\-._]+\.(jpg|js|css|png|apng|gif|svg|html|ico|json|less|sass|scss|swf|txt|webp|heif|heic|jpeg|woff|woff2|xls|xlsx|xml|xsl|xslt|zip|rar|csv|doc|docx|eps|gzip|jpe|jpeg|manifest|mov|mp3|flac|mp4|ogg|package|pdf)"/gi; const REG_COMPILECSS = /url\(.*?\)/g; const REG_ROUTESTATIC = /^(\/\/|https:|http:)+/; const REG_NEWIMPL = /^(async\s)?function(\s)?([a-zA-Z$][a-zA-Z0-9$]+)?(\s)?\([a-zA-Z0-9$]+\)|^function anonymous\(\$/; From 9faad8b5a908fdd592a6ce4cb8555bbc0bd23358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 23 Nov 2018 09:59:24 +0100 Subject: [PATCH 194/217] Fixed test. --- test.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test.js b/test.js index d4abd2114..a3e9c1fe5 100644 --- a/test.js +++ b/test.js @@ -178,6 +178,9 @@ exports.load = function() { return next(); } + if (U.getExtension(filename) !== 'js') + return next(); + T.current = { filename: filename, items: [] }; var m = require(filename); T.current.module = m; From fef98c9e3590555b8230b4e6f8ec4f83cab44f56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 23 Nov 2018 10:01:06 +0100 Subject: [PATCH 195/217] New changes. --- changes.txt | 1 + package.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index eb36a952f..ce2627ecc 100755 --- a/changes.txt +++ b/changes.txt @@ -66,6 +66,7 @@ - fixed: `--inspect` argument for Workers by Tema Smirnov - fixed: TLS in SMTP mail sender - fixed: applying of versions +- fixed: unit-tests reads only `javascript` files - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute diff --git a/package.json b/package.json index 3d3763e44..bb82596fb 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-30", + "version": "3.0.1-31", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From ba9646533d086151c0804b8091fb37e852787af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 24 Nov 2018 12:42:18 +0100 Subject: [PATCH 196/217] Improved TaskBuilder. --- builders.js | 89 +++++++++++++++++++++++++++++++++++++++++++++++------ changes.txt | 1 + index.js | 6 ++++ 3 files changed, 87 insertions(+), 9 deletions(-) diff --git a/builders.js b/builders.js index bdaad14d5..63c31d994 100755 --- a/builders.js +++ b/builders.js @@ -38,7 +38,6 @@ const Qs = require('querystring'); var schemas = {}; var schemasall = {}; var operations = {}; -var schemacache = {}; var transforms = { pagination: {}, error: {}, restbuilder: {} }; function SchemaBuilder(name) { @@ -54,7 +53,22 @@ function SchemaOptions(error, model, options, callback, controller) { this.controller = controller; } -SchemaOptions.prototype = { +function TaskBuilder($) { + var t = this; + t.value = t.model = {}; + t.tasks = {}; + if ($ instanceof SchemaOptions || $ instanceof OperationOptions) { + t.error = $.error; + t.controller = $.controller; + } else { + if ($ instanceof Controller) + t.controller = $; + else if ($ instanceof ErrorBuilder) + t.error = $; + } +} + +SchemaOptions.prototype = TaskBuilder.prototype = { get user() { return this.controller ? this.controller.user : null; @@ -4953,6 +4967,10 @@ OperationOptions.prototype = { } }; +SchemaOptions.prototype.tasks = OperationOptions.prototype.tasks = function() { + return new TaskBuilder(this); +}; + OperationOptions.prototype.cancel = function() { var self = this; self.callback = null; @@ -5202,6 +5220,7 @@ global.Pagination = Pagination; global.Page = Page; global.UrlBuilder = global.URLBuilder = UrlBuilder; global.SchemaBuilder = SchemaBuilder; +global.TaskBuilder = TaskBuilder; // Uninstall owners exports.uninstall = function(owner) { @@ -5219,11 +5238,63 @@ exports.uninstall = function(owner) { }); }; -exports.restart = function() { - schemas = {}; - operations = {}; - Object.keys(transforms).forEach(function(key) { - if (key.indexOf('_') === -1) - transforms[key] = {}; - }); +TaskBuilder.prototype.invalid = function(error) { + var self = this; + if (!self.$done) { + !self.error && (self.error = new ErrorBuilder()); + self.error.push(self.current, error); + self.done(); + } + return self; +}; + +TaskBuilder.prototype.push = function(name, fn) { + var self = this; + self.tasks[name] = fn; + return self; +}; + +TaskBuilder.prototype.next = function() { + var self = this; + if (!self.$done) { + self.prev = self.current; + for (var i = 0; i < arguments.length; i++) { + var task = self.tasks[self.current = arguments[i]]; + if (task == null) + continue; + else { + task.call(self, self); + return self; + } + } + self.done(); + } + return self; +}; + +TaskBuilder.prototype.done = function(data) { + var self = this; + self.$callback && self.$callback(self.error && self.error.is ? self.error : null, data || self.value); + self.$done = true; + return self; +}; + +TaskBuilder.prototype.success = function(data) { + return this.done(SUCCESS(true, data)); }; + +TaskBuilder.prototype.callback = function(fn) { + var self = this; + self.$callback = fn; + return self; +}; + +TaskBuilder.prototype.exec = function(name, callback) { + var self = this; + + if (callback) + self.$callback = callback; + + self.next(name); + return self; +}; \ No newline at end of file diff --git a/changes.txt b/changes.txt index ce2627ecc..b4fcac9c0 100755 --- a/changes.txt +++ b/changes.txt @@ -27,6 +27,7 @@ - added: `ONCE()` alias to `F.once()` - added: `image.define(value)` performs `convert -define 'value'` - added: Total.js JS files (+ packages) tarted with `.` (dot) or ended with `-bk` won't be processed +- added: A new builder called `TaskBuilder` for creating custom tasks in Schemas or Operations - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) diff --git a/index.js b/index.js index b925c2fdd..22f5c4c27 100755 --- a/index.js +++ b/index.js @@ -10525,6 +10525,12 @@ Controller.prototype.operation = function(name, value, callback, options) { return this; }; +Controller.prototype.tasks = function() { + var tb = new TaskBuilder(this); + // tb.callback(this.callback()); + return tb; +}; + Controller.prototype.$operation2 = function(name, helper, callback) { if (callback == null && typeof(helper) === 'function') { From 039c51720ba15cb0dbe03fe7a3290e21325018b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 24 Nov 2018 12:51:54 +0100 Subject: [PATCH 197/217] Fixed prototype. --- builders.js | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/builders.js b/builders.js index 63c31d994..49d350e06 100755 --- a/builders.js +++ b/builders.js @@ -68,7 +68,46 @@ function TaskBuilder($) { } } -SchemaOptions.prototype = TaskBuilder.prototype = { +TaskBuilder.prototype = { + + get user() { + return this.controller ? this.controller.user : null; + }, + + get session() { + return this.controller ? this.controller.session : null; + }, + + get language() { + return (this.controller ? this.controller.language : '') || ''; + }, + + get ip() { + return this.controller ? this.controller.ip : null; + }, + + get id() { + return this.controller ? this.controller.id : null; + }, + + get params() { + return this.controller ? this.controller.params : null; + }, + + get files() { + return this.controller ? this.controller.files : null; + }, + + get body() { + return this.controller ? this.controller.body : null; + }, + + get query() { + return this.controller ? this.controller.query : null; + } +}; + +SchemaOptions.prototype = { get user() { return this.controller ? this.controller.user : null; From 09b3565040876d2918ae63d4d7037306628a65a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 24 Nov 2018 13:01:34 +0100 Subject: [PATCH 198/217] Added `logger` for `TaskBuiler`. --- builders.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/builders.js b/builders.js index 49d350e06..a4e187727 100755 --- a/builders.js +++ b/builders.js @@ -5296,6 +5296,7 @@ TaskBuilder.prototype.push = function(name, fn) { TaskBuilder.prototype.next = function() { var self = this; if (!self.$done) { + self.current && self.controller && CONF.logger && F.ilogger((self.name || 'tasks') + '.' + self.current, self.controller, self.$now); self.prev = self.current; for (var i = 0; i < arguments.length; i++) { var task = self.tasks[self.current = arguments[i]]; @@ -5334,6 +5335,9 @@ TaskBuilder.prototype.exec = function(name, callback) { if (callback) self.$callback = callback; + if (CONF.logger) + self.$now = Date.now(); + self.next(name); return self; }; \ No newline at end of file From 57afb58ba8bee9fd633f2971a80ee7ffa2da8fa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Nov 2018 10:24:39 +0100 Subject: [PATCH 199/217] Updated `F.path.mkdir()`. --- changes.txt | 2 +- index.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/changes.txt b/changes.txt index b4fcac9c0..87dab9e5b 100755 --- a/changes.txt +++ b/changes.txt @@ -37,7 +37,7 @@ - updated: `REQUEST()` can return binary data if the content-type is not `text/*` or `application/*` - updated: NoSQL joins support array values - updated: `ROUTING(id:|search, [flags])` method -- updated: `F.path.mkdir()` currently caches a value +- updated: `F.path.mkdir(path, [cache])` can cache a current satte (default: false) - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator diff --git a/index.js b/index.js index 22f5c4c27..1bba46dff 100755 --- a/index.js +++ b/index.js @@ -9680,11 +9680,11 @@ FrameworkPath.prototype.verify = function(name) { return F; }; -FrameworkPath.prototype.mkdir = function(p) { +FrameworkPath.prototype.mkdir = function(p, cache) { var key = '$directory-' + p; - if (F.temporary.path[key]) + if (cache && F.temporary.path[key]) return F; F.temporary.path[key] = true; From 7d6b155cd2c75c59a052cfb82594b0ae54656c1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Nov 2018 15:33:50 +0100 Subject: [PATCH 200/217] Fixed GZIP compression. --- index.js | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/index.js b/index.js index 1bba46dff..e119c1564 100755 --- a/index.js +++ b/index.js @@ -6095,13 +6095,15 @@ function compile_check(res) { if (!res.noCompress && COMPRESSIONSPECIAL[req.extension] && CONF.allow_compile && !REG_NOCOMPRESS.test(res.options.filename)) return compile_file(res); - var tmp = F.temporary.path[req.$key] = [res.options.filename, size, stats.mtime.toUTCString()]; + var tmp = [res.options.filename, size, stats.mtime.toUTCString()]; if (CONF.allow_gzip && COMPRESSION[U.getContentType(req.extension)]) { compile_gzip(tmp, function() { + F.temporary.path[req.$key] = tmp; res.$file(); delete F.temporary.processing[req.$key]; }); } else { + F.temporary.path[req.$key] = tmp; res.$file(); delete F.temporary.processing[req.$key]; } @@ -13839,7 +13841,7 @@ WebSocket.prototype.destroy = function(problem) { return self; self.close(); - self.$events.destroy && selEMIT('destroy'); + self.$events.destroy && self.emit('destroy'); setTimeout(function() { @@ -15197,14 +15199,6 @@ function extend_request(PROTO) { file.execute(req, res, false); return; - - /* - } catch (err) { - F.error(err, file.controller, req.uri); - res.throw500(); - return; - } - */ } res.continue(); @@ -16004,6 +15998,8 @@ function extend_response(PROTO) { if (name[1] && !compress) headers[HEADER_LENGTH] = name[1]; + else if (compress && name[4]) + headers[HEADER_LENGTH] = name[4]; else if (headers[HEADER_LENGTH]) delete headers[HEADER_LENGTH]; From 5f0d0c965e17c7538eadfab91c6bf0ae17b36f02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Nov 2018 15:41:42 +0100 Subject: [PATCH 201/217] Fixed GZIP compression. --- changes.txt | 1 + index.js | 25 +++++++++++++++---------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/changes.txt b/changes.txt index 87dab9e5b..b22dff4eb 100755 --- a/changes.txt +++ b/changes.txt @@ -43,6 +43,7 @@ - fixed: a critical bug with JavaScript minificator - fixed: a critical bug with NoSQL counter and freezing app - fixed: a critical bug with rendering of multiple async components +- fixed: a critical bug with GZIP compression - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function - fixed: `F.wait()` in the test mode - fixed: `LOCALIZE()` for nested directories diff --git a/index.js b/index.js index e119c1564..e670564ad 100755 --- a/index.js +++ b/index.js @@ -5887,8 +5887,9 @@ function compile_file(res) { F.path.verify('temp'); Fs.writeFileSync(file, compile_content(req.extension, framework_internal.parseBlock(F.routes.blocks[uri.pathname], buffer.toString(ENCODING)), res.options.filename), ENCODING); var stats = Fs.statSync(file); - var tmp = F.temporary.path[req.$key] = [file, stats.size, stats.mtime.toUTCString()]; - compile_gzip(tmp, function() { + var tmp = [file, stats.size, stats.mtime.toUTCString()]; + compile_gzip(tmp, function(tmp) { + F.temporary.path[req.$key] = tmp; delete F.temporary.processing[req.$key]; res.$file(); }); @@ -5905,9 +5906,10 @@ function compile_merge(res) { if (!CONF.debug && existsSync(filename)) { var stats = Fs.statSync(filename); - var tmp = F.temporary.path[req.$key] = [filename, stats.size, stats.mtime.toUTCString()]; - compile_gzip(tmp, function() { + var tmp = [filename, stats.size, stats.mtime.toUTCString()]; + compile_gzip(tmp, function(tmp) { delete F.temporary.processing[req.$key]; + F.temporary.path[req.$key] = tmp; res.$file(); }); return; @@ -5917,9 +5919,10 @@ function compile_merge(res) { writer.on('finish', function() { var stats = Fs.statSync(filename); - var tmp = F.temporary.path[req.$key] = [filename, stats.size, stats.mtime.toUTCString()]; + var tmp = [filename, stats.size, stats.mtime.toUTCString()]; this.destroy && this.destroy(); - compile_gzip(tmp, function() { + compile_gzip(tmp, function(tmp) { + F.temporary.path[req.$key] = tmp; delete F.temporary.processing[req.$key]; res.$file(); }); @@ -6063,13 +6066,15 @@ F.compile_virtual = function(res) { return compile_file(res); } - var tmp = F.temporary.path[req.$key] = [tmpname, size, stats.mtime.toUTCString()]; + var tmp = [tmpname, size, stats.mtime.toUTCString()]; if (CONF.allow_gzip && COMPRESSION[U.getContentType(req.extension)]) { - compile_gzip(tmp, function() { + compile_gzip(tmp, function(tmp) { + F.temporary.path[req.$key] = tmp; delete F.temporary.processing[req.$key]; res.$file(); }); } else { + F.temporary.path[req.$key] = tmp; delete F.temporary.processing[req.$key]; res.$file(); } @@ -6097,7 +6102,7 @@ function compile_check(res) { var tmp = [res.options.filename, size, stats.mtime.toUTCString()]; if (CONF.allow_gzip && COMPRESSION[U.getContentType(req.extension)]) { - compile_gzip(tmp, function() { + compile_gzip(tmp, function(tmp) { F.temporary.path[req.$key] = tmp; res.$file(); delete F.temporary.processing[req.$key]; @@ -6131,7 +6136,7 @@ function compile_gzip(arr, callback) { CLEANUP(writer, function() { fsFileExists(filename, function(e, size) { arr.push(size); - callback(); + callback(arr); }); }); From 81615fddb88b48423f5e73a2f5bf95eb5da291a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Nov 2018 15:42:40 +0100 Subject: [PATCH 202/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bb82596fb..0f956763a 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-31", + "version": "3.0.1-32", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 784367cf2900263586ba69f1c31bb9cb862a22c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sun, 25 Nov 2018 15:43:27 +0100 Subject: [PATCH 203/217] Updated changelog. --- changes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index b22dff4eb..78ad0edc7 100755 --- a/changes.txt +++ b/changes.txt @@ -43,7 +43,7 @@ - fixed: a critical bug with JavaScript minificator - fixed: a critical bug with NoSQL counter and freezing app - fixed: a critical bug with rendering of multiple async components -- fixed: a critical bug with GZIP compression +- fixed: a critical bug with GZIP compression (sometimes appeared in Safari) - fixed: `nosql.update()` and `nosql.modify()` methods if the first argument is a function - fixed: `F.wait()` in the test mode - fixed: `LOCALIZE()` for nested directories From e2f376ded9e5ba70d8e9e579544c865fcac4a701 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 28 Nov 2018 09:23:00 +0100 Subject: [PATCH 204/217] Improved `websocketclient.send()`. --- index.js | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/index.js b/index.js index e670564ad..81f31bc74 100755 --- a/index.js +++ b/index.js @@ -14501,27 +14501,29 @@ WebSocketClient.prototype.$onclose = function() { */ WebSocketClient.prototype.send = function(message, raw, replacer) { - if (this.isClosed) - return this; + var self = this; - if (this.type !== 1) { - var data = this.type === 3 ? (raw ? message : JSON.stringify(message, replacer)) : (message || '').toString(); + if (self.isClosed) + return self; + + if (self.type !== 1) { + var data = self.type === 3 ? (raw ? message : JSON.stringify(message, replacer)) : typeof(message) === 'object' ? JSON.stringify(message, replacer) : message.toString(); if (CONF.default_websocket_encodedecode === true && data) data = encodeURIComponent(data); - if (this.deflate) { - this.deflatepending.push(U.createBuffer(data)); - this.sendDeflate(); + if (self.deflate) { + self.deflatepending.push(U.createBuffer(data)); + self.sendDeflate(); } else - this.socket.write(U.getWebSocketFrame(0, data, 0x01)); + self.socket.write(U.getWebSocketFrame(0, data, 0x01)); } else if (message) { - if (this.deflate) { - this.deflatepending.push(U.createBuffer(message)); - this.sendDeflate(); + if (self.deflate) { + self.deflatepending.push(U.createBuffer(message)); + self.sendDeflate(); } else - this.socket.write(U.getWebSocketFrame(0, new Int8Array(message), 0x02)); + self.socket.write(U.getWebSocketFrame(0, new Int8Array(message), 0x02)); } - return this; + return self; }; WebSocketClient.prototype.sendDeflate = function() { From 7813128ba8596ec5022a705632614fffd5ab7d2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 29 Nov 2018 09:16:12 +0100 Subject: [PATCH 205/217] Updated readme. --- readme.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/readme.md b/readme.md index 8f620d01e..2302f17c3 100755 --- a/readme.md +++ b/readme.md @@ -13,8 +13,9 @@ $ npm install -g total.js [![Support](https://www.totaljs.com/img/button-support.png?v=3)](https://www.totaljs.com/support/) -- [__NEW__ Total.js CMS](http://www.totaljs.com/cms/) -- [__NEW__ Total.js Eshop](http://www.totaljs.com/eshop/) +- [__NEW__: Total.js Code editor](http://www.totaljs.com/code/) +- [Total.js CMS](http://www.totaljs.com/cms/) +- [Total.js Eshop](http://www.totaljs.com/eshop/) - [Total.js Wiki](https://www.totaljs.com/wiki/) - [Total.js Flow](https://www.totaljs.com/flow/) - [Total.js Flowboard](https://www.totaljs.com/flowboard/) @@ -29,7 +30,8 @@ $ npm install -g total.js - [Total.js modules and packages](https://modules.totaljs.com) - [Total.js +100 examples](https://github.com/totaljs/examples) - [NoSQL embedded database explorer](https://www.totaljs.com/nosql/) -- [Download +100 client-side components (jComponent) for free](https://componentator.com) +- [Download +100 UI components for free](https://componentator.com) +- [Download +100 CMS widgets for free](https://componentator.com/widgets/) --- From 7f5b00821d2fd265d433841668af90bef55b32ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 29 Nov 2018 09:35:25 +0100 Subject: [PATCH 206/217] Removed useless code. --- changes.txt | 3 +++ index.js | 30 ++++++++++++------------------ 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/changes.txt b/changes.txt index 78ad0edc7..5f147260e 100755 --- a/changes.txt +++ b/changes.txt @@ -70,6 +70,9 @@ - fixed: applying of versions - fixed: unit-tests reads only `javascript` files +- removed: `F.config.debug` +- removed: `controller.isDebug` + - improved: NoSQL reader - improved: `UID()` -> now it changes a random hash each minute - improved: CORS diff --git a/index.js b/index.js index 81f31bc74..60cc6e199 100755 --- a/index.js +++ b/index.js @@ -4217,8 +4217,8 @@ F.install = function(type, name, declaration, options, callback, internal, useRe content = parseComponent(internal ? declaration : Fs.readFileSync(declaration).toString(ENCODING), name); if (F.$bundling) { - content.js && Fs.appendFileSync(F.path.temp(temporary + '.js'), hash + (CONF.debug ? component_debug(name, content.js, 'js') : content.js) + hash.substring(0, hash.length - 1)); - content.css && Fs.appendFileSync(F.path.temp(temporary + '.css'), hash + (CONF.debug ? component_debug(name, content.css, 'css') : content.css) + hash.substring(0, hash.length - 1)); + content.js && Fs.appendFileSync(F.path.temp(temporary + '.js'), hash + (DEBUG ? component_debug(name, content.js, 'js') : content.js) + hash.substring(0, hash.length - 1)); + content.css && Fs.appendFileSync(F.path.temp(temporary + '.css'), hash + (DEBUG ? component_debug(name, content.css, 'css') : content.css) + hash.substring(0, hash.length - 1)); } if (content.js) @@ -4295,12 +4295,12 @@ F.install = function(type, name, declaration, options, callback, internal, useRe tmp = F.components.groups[obj.group] = {}; if (content.js) { - Fs.appendFileSync(F.path.temp(temporary + '.js'), hash + (CONF.debug ? component_debug(name, content.js, 'js') : content.js) + hash.substring(0, hash.length - 1)); + Fs.appendFileSync(F.path.temp(temporary + '.js'), hash + (DEBUG ? component_debug(name, content.js, 'js') : content.js) + hash.substring(0, hash.length - 1)); tmp.js = true; } if (content.css) { - Fs.appendFileSync(F.path.temp(temporary + '.css'), hash + (CONF.debug ? component_debug(name, content.css, 'css') : content.css) + hash.substring(0, hash.length - 1)); + Fs.appendFileSync(F.path.temp(temporary + '.css'), hash + (DEBUG ? component_debug(name, content.css, 'css') : content.css) + hash.substring(0, hash.length - 1)); tmp.css = true; } @@ -4735,7 +4735,7 @@ F.install = function(type, name, declaration, options, callback, internal, useRe } else { F.$configure_configs('@' + name + '/config'); - if (CONF.debug) + if (DEBUG) F.$configure_configs('@' + name + '/config-debug'); else F.$configure_configs('@' + name + '/config-release'); @@ -5747,7 +5747,7 @@ F.usage = function(detailed) { memoryTotal: (memory.heapTotal / 1024 / 1024).floor(2), memoryUsage: (memory.heapUsed / 1024 / 1024).floor(2), memoryRss: (memory.rss / 1024 / 1024).floor(2), - mode: CONF.debug ? 'debug' : 'release', + mode: DEBUG, port: F.port, ip: F.ip, directory: process.cwd() @@ -5904,7 +5904,7 @@ function compile_merge(res) { var merge = F.routes.merge[uri.pathname]; var filename = merge.filename; - if (!CONF.debug && existsSync(filename)) { + if (!DEBUG && existsSync(filename)) { var stats = Fs.statSync(filename); var tmp = [filename, stats.size, stats.mtime.toUTCString()]; compile_gzip(tmp, function(tmp) { @@ -6612,7 +6612,6 @@ global.LOAD = F.load = function(debug, types, pwd) { } F.isWorker = true; - CONF.debug = debug; F.isDebug = debug; global.DEBUG = debug; @@ -6709,7 +6708,6 @@ F.initialize = function(http, debug, options) { if (options.id) F.id = options.id; - CONF.debug = debug; F.isDebug = debug; if (options.bundling != null) @@ -6922,7 +6920,7 @@ F.mode = function(http, name, options) { debug = true; break; } - CONF.debug = debug; + DEBUG = debug; CONF.trace = debug; F.isDebug = debug; global.DEBUG = debug; @@ -7035,7 +7033,7 @@ F.console = function() { console.log('Version : ' + CONF.version); console.log('Author : ' + CONF.author); console.log('Date : ' + NOW.format('yyyy-MM-dd HH:mm:ss')); - console.log('Mode : ' + (CONF.debug ? 'debug' : 'release')); + console.log('Mode : ' + (DEBUG ? 'debug' : 'release')); console.log('===================================================='); console.log('Directory : ' + process.cwd()); console.log('node_modules : ' + PATHMODULES); @@ -7437,7 +7435,7 @@ F.$requestcontinue = function(req, res, headers) { flags.push('sse'); } - if (CONF.debug) { + if (DEBUG) { req.$flags += 'h'; flags.push('debug'); } @@ -8793,7 +8791,7 @@ F.$configure_configs = function(arr, rewrite) { if (!arr) { var filenameA = U.combine('/', 'config'); - var filenameB = U.combine('/', 'config-' + (CONF.debug ? 'debug' : 'release')); + var filenameB = U.combine('/', 'config-' + (DEBUG ? 'debug' : 'release')); arr = []; @@ -10258,10 +10256,6 @@ Controller.prototype = { return F.controllers; }, - get isDebug() { - return CONF.debug; - }, - get isTest() { return this.req.headers['x-assertion-testing'] === '1'; }, @@ -13526,7 +13520,7 @@ WebSocket.prototype = { }, get isDebug() { - return CONF.debug; + return DEBUG; }, get path() { From 62321eb586bf1628d3f669093cc1abcd8d6ed94d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 29 Nov 2018 11:02:08 +0100 Subject: [PATCH 207/217] Updated `controller.all()`. --- changes.txt | 1 + index.js | 14 ++++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/changes.txt b/changes.txt index 5f147260e..65d7df0f2 100755 --- a/changes.txt +++ b/changes.txt @@ -38,6 +38,7 @@ - updated: NoSQL joins support array values - updated: `ROUTING(id:|search, [flags])` method - updated: `F.path.mkdir(path, [cache])` can cache a current satte (default: false) +- updated: `controller.all()` can return `Array` of all WebSocketClient - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator diff --git a/index.js b/index.js index 60cc6e199..206d413c9 100755 --- a/index.js +++ b/index.js @@ -13795,11 +13795,17 @@ WebSocket.prototype.change = function(message) { * @return {WebSocket} */ WebSocket.prototype.all = function(fn) { - if (this._keys) { - for (var i = 0, length = this._keys.length; i < length; i++) - fn(this.connections[this._keys[i]], i); + var arr = fn == null || fn == true ? [] : null; + var self = this; + if (self._keys) { + for (var i = 0, length = self._keys.length; i < length; i++) { + if (arr) + arr.push(self.connections[self._keys[i]]); + else + fn(self.connections[self._keys[i]], i); + } } - return this; + return arr ? arr : self; }; /** From 61477c6c78f11fb748ab8ef1e00caf12dbce1bcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Thu, 29 Nov 2018 19:36:37 +0100 Subject: [PATCH 208/217] Added error handling for unpacking. --- index.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/index.js b/index.js index 206d413c9..0d04bbc78 100755 --- a/index.js +++ b/index.js @@ -6248,6 +6248,16 @@ F.restore = function(filename, target, callback, filter) { output.count++; + tmp.zlib.on('error', function(e) { + pending--; + var tmp = this.$self; + tmp.writer.end(); + tmp.writer = null; + tmp.zlib = null; + delete open[tmp.name]; + F.error(e, 'bundling', path); + }); + tmp.zlib.on('data', function(chunk) { this.$self.writer.write(chunk); }); From 1a365ffa068fb0fff673f4cc89c3fc5ec1f931c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 30 Nov 2018 08:08:01 +0100 Subject: [PATCH 209/217] Fixed `controller.invalid()`. --- changes.txt | 1 + index.js | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/changes.txt b/changes.txt index 65d7df0f2..5894594e0 100755 --- a/changes.txt +++ b/changes.txt @@ -70,6 +70,7 @@ - fixed: TLS in SMTP mail sender - fixed: applying of versions - fixed: unit-tests reads only `javascript` files +- fixed: `controller.invalid()` a problem with ErrorBuilder as a argument - removed: `F.config.debug` - removed: `controller.isDebug` diff --git a/index.js b/index.js index 0d04bbc78..1776f4ebd 100755 --- a/index.js +++ b/index.js @@ -10804,6 +10804,12 @@ Controller.prototype.error = function(err) { Controller.prototype.invalid = function(status) { var self = this; + + if (status instanceof ErrorBuilder) { + setImmediate(next_controller_invalid, self, status); + return status; + } + var type = typeof(status); if (type === 'number') From 1aa7ed1820d93416a90c7974b11b02d7c2870058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 30 Nov 2018 08:41:01 +0100 Subject: [PATCH 210/217] New improvements. --- changes.txt | 2 ++ index.js | 67 +++++++++++++++++++++++++++++-------- test/controllers/default.js | 10 +++--- 3 files changed, 60 insertions(+), 19 deletions(-) diff --git a/changes.txt b/changes.txt index 5894594e0..22cd9476d 100755 --- a/changes.txt +++ b/changes.txt @@ -28,6 +28,8 @@ - added: `image.define(value)` performs `convert -define 'value'` - added: Total.js JS files (+ packages) tarted with `.` (dot) or ended with `-bk` won't be processed - added: A new builder called `TaskBuilder` for creating custom tasks in Schemas or Operations +- added: `WebSocket.send2(message, [comparer(client, message)], [replacer])` a new method for better sending frames +- addde: `PATH` as a global alias for `F.path` - updated: `debug` mode creates a `start_name_script.pid` instead of `debug.pid` - updated: `NEWOPERATION()` supports `repeat`, `stop` and `binderror` arguments (more in docs) diff --git a/index.js b/index.js index 1776f4ebd..d6e95da78 100755 --- a/index.js +++ b/index.js @@ -967,7 +967,7 @@ function Framework() { // intialize cache self.cache = new FrameworkCache(); - self.path = new FrameworkPath(); + self.path = global.PATH = new FrameworkPath(); self._request_check_redirect = false; self._request_check_referer = false; @@ -10219,6 +10219,7 @@ Controller.prototype = { }, get path() { + OBSOLETE('controller.path', 'Use: PATH'); return F.path; }, @@ -10255,10 +10256,12 @@ Controller.prototype = { }, get cache() { + OBSOLETE('controller.cache', 'Use: F.cache or CACHE()'); return F.cache; }, get config() { + OBSOLETE('controller.config', 'Use: CONF'); return CONF; }, @@ -10271,6 +10274,7 @@ Controller.prototype = { }, get isSecure() { + OBSOLETE('controller.isSecure', 'Use: controller.secured'); return this.req.isSecure; }, @@ -13524,30 +13528,32 @@ WebSocket.prototype = { }, get global() { + OBSOLETE('controller.global', 'Use: G'); return F.global; }, get config() { + OBSOLETE('controller.config', 'Use: CONF'); return CONF; }, get cache() { + OBSOLETE('controller.cache', 'Use: F.cache or CACHE()'); return F.cache; }, get isDebug() { + OBSOLETE('controller.isDebug', 'Use: DEBUG'); return DEBUG; }, get path() { + OBSOLETE('controller.path', 'Use: PATH'); return F.path; }, - get fs() { - return F.fs; - }, - get isSecure() { + OBSOLETE('controller.isSecure', 'Use: controller.secured'); return this.req.isSecure; }, @@ -13643,23 +13649,25 @@ WebSocket.prototype.removeAllListeners = function(name) { */ WebSocket.prototype.send = function(message, id, blacklist, replacer) { - var keys = this._keys; - if (!keys || !keys.length) - return this; + var self = this; + var keys = self._keys; + + if (!keys || !keys.length || message === undefined) + return self; var data; var raw = false; for (var i = 0, length = keys.length; i < length; i++) { - var conn = this.connections[keys[i]]; + var conn = self.connections[keys[i]]; if (id) { if (id instanceof Array) { if (!websocket_valid_array(conn.id, id)) continue; } else if (id instanceof Function) { - if (!websocket_valid_fn(conn.id, conn, id)) + if (!websocket_valid_fn(conn.id, conn, id, message)) continue; } else throw new Error('Invalid "id" argument.'); @@ -13670,7 +13678,7 @@ WebSocket.prototype.send = function(message, id, blacklist, replacer) { if (websocket_valid_array(conn.id, blacklist)) continue; } else if (blacklist instanceof Function) { - if (websocket_valid_fn(conn.id, conn, blacklist)) + if (websocket_valid_fn(conn.id, conn, blacklist, message)) continue; } else throw new Error('Invalid "blacklist" argument.'); @@ -13688,15 +13696,46 @@ WebSocket.prototype.send = function(message, id, blacklist, replacer) { F.stats.response.websocket++; } - return this; + return self; +}; + +WebSocket.prototype.send2 = function(message, comparer, replacer) { + + var self = this; + var keys = self._keys; + if (!keys || !keys.length || message === undefined) + return self; + + var data; + var raw = false; + + for (var i = 0, length = keys.length; i < length; i++) { + + var conn = self.connections[keys[i]]; + + if (data === undefined) { + if (conn.type === 3) { + raw = true; + data = JSON.stringify(message, replacer); + } else + data = message; + } + + if (comparer && comparer(conn, message)) { + conn.send(data, raw); + F.stats.response.websocket++; + } + } + + return self; }; function websocket_valid_array(id, arr) { return arr.indexOf(id) !== -1; } -function websocket_valid_fn(id, client, fn) { - return fn && fn(id, client) ? true : false; +function websocket_valid_fn(id, client, fn, msg) { + return fn && fn(id, client, msg) ? true : false; } /** diff --git a/test/controllers/default.js b/test/controllers/default.js index a258d92aa..fa70af164 100755 --- a/test/controllers/default.js +++ b/test/controllers/default.js @@ -156,7 +156,7 @@ function plain_options() { function *synchronize() { var self = this; - var content = (yield sync(require('fs').readFile)(self.path.public('file.txt'))).toString('utf8'); + var content = (yield sync(require('fs').readFile)(PATH.public('file.txt'))).toString('utf8'); self.plain(content); } @@ -439,9 +439,9 @@ function viewIndex() { var self = this; var name = 'controller: '; - assert.ok(self.path.public('file.txt').endsWith('/public/file.txt'), name + 'path.public'); - assert.ok(self.path.logs('file.txt').endsWith('/file.txt'), name + 'path.logs'); - assert.ok(self.path.temp('file.txt').endsWith('/file.txt'), name + 'path.temp'); + assert.ok(PATH.public('file.txt').endsWith('/public/file.txt'), name + 'path.public'); + assert.ok(PATH.logs('file.txt').endsWith('/file.txt'), name + 'path.logs'); + assert.ok(PATH.temp('file.txt').endsWith('/file.txt'), name + 'path.temp'); self.meta('A', 'B'); assert.ok(self.repository['$title'] === 'A' && self.repository['$description'] === 'B', name + 'meta() - write'); @@ -458,7 +458,7 @@ function viewIndex() { assert.ok(framework.model('other/products').ok === 2, 'framework: model() - 2'); assert.ok(self.secured === false, 'controller.secured'); - assert.ok(self.config.isDefinition === true, 'definitions()'); + assert.ok(CONF.isDefinition === true, 'definitions()'); assert.ok(!self.xhr, name + 'xhr'); assert.ok(self.flags.indexOf('get') !== -1, name + 'flags') From 2b17995cee4e24e1f8a0c8a4b26cdd0d0b5e0a5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 30 Nov 2018 09:00:42 +0100 Subject: [PATCH 211/217] Fixed cache. --- index.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index d6e95da78..c8dd04258 100755 --- a/index.js +++ b/index.js @@ -11355,7 +11355,7 @@ Controller.prototype.$view = function(name, model, expire, key) { if (expire) { cache = '$view.' + name + '.' + (key || ''); - var output = self.cache.read2(cache); + var output = F.cache.read2(cache); if (output) return output.body; } @@ -11365,7 +11365,7 @@ Controller.prototype.$view = function(name, model, expire, key) { if (!value) return ''; - expire && self.cache.add(cache, { components: value instanceof Function, body: value instanceof Function ? '' : value }, expire, false); + expire && F.cache.add(cache, { components: value instanceof Function, body: value instanceof Function ? '' : value }, expire, false); return value; }; @@ -13382,7 +13382,7 @@ Controller.prototype.memorize = function(key, expires, disabled, fnTo, fnFrom) { self.themeName && (key += '#' + self.themeName); - var output = self.cache.read2(key); + var output = F.cache.read2(key); if (!output) return self.$memorize_prepare(key, expires, disabled, fnTo, fnFrom); @@ -13473,7 +13473,7 @@ Controller.prototype.$memorize_prepare = function(key, expires, disabled, fnTo, } } - self.cache.add(key, options, expires, false); + F.cache.add(key, options, expires, false); self.precache = null; delete F.temporary.processing[pk]; }; From 032b45b680415dc861d9c5b1dc56b52b284febe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 30 Nov 2018 20:07:47 +0100 Subject: [PATCH 212/217] Updated beta version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0f956763a..f25f66df8 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-32", + "version": "3.0.1-33", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues", From 541f93014ec2fcc64b8ac65d424550e674f12820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Sat, 1 Dec 2018 10:51:45 +0100 Subject: [PATCH 213/217] Fixed `controller.send2()`. --- index.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index c8dd04258..dada5d503 100755 --- a/index.js +++ b/index.js @@ -13721,10 +13721,11 @@ WebSocket.prototype.send2 = function(message, comparer, replacer) { data = message; } - if (comparer && comparer(conn, message)) { - conn.send(data, raw); - F.stats.response.websocket++; - } + if (comparer && !comparer(conn, message)) + continue; + + conn.send(data, raw); + F.stats.response.websocket++; } return self; From 0ee9046606d1dc27e615fe89d23b6e0548a8f288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 4 Dec 2018 08:12:54 +0100 Subject: [PATCH 214/217] Updated startup info. --- changes.txt | 1 + index.js | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 22cd9476d..9135b9d66 100755 --- a/changes.txt +++ b/changes.txt @@ -41,6 +41,7 @@ - updated: `ROUTING(id:|search, [flags])` method - updated: `F.path.mkdir(path, [cache])` can cache a current satte (default: false) - updated: `controller.all()` can return `Array` of all WebSocketClient +- updated: startup info by adding user name - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator diff --git a/index.js b/index.js index dada5d503..74488abae 100755 --- a/index.js +++ b/index.js @@ -7036,12 +7036,13 @@ F.console = function() { console.log('Node.js : ' + process.version); console.log('Total.js : v' + F.version_header); console.log('OS : ' + Os.platform() + ' ' + Os.release()); + console.log('User : ' + Os.userInfo().username); CONF.nosql_worker && console.log('NoSQL PID : ' + framework_nosql.pid()); console.log('Memory : ' + memory.heapUsed.filesize(2) + ' / ' + memory.heapTotal.filesize(2)); console.log('===================================================='); console.log('Name : ' + CONF.name); console.log('Version : ' + CONF.version); - console.log('Author : ' + CONF.author); + CONF.author && console.log('Author : ' + CONF.author); console.log('Date : ' + NOW.format('yyyy-MM-dd HH:mm:ss')); console.log('Mode : ' + (DEBUG ? 'debug' : 'release')); console.log('===================================================='); From 49bcf4dcc864298cb894d4e056ae633fbfcdf0f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Tue, 4 Dec 2018 08:31:31 +0100 Subject: [PATCH 215/217] Updated ordering. --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 74488abae..50bf2e2d3 100755 --- a/index.js +++ b/index.js @@ -7036,9 +7036,9 @@ F.console = function() { console.log('Node.js : ' + process.version); console.log('Total.js : v' + F.version_header); console.log('OS : ' + Os.platform() + ' ' + Os.release()); - console.log('User : ' + Os.userInfo().username); CONF.nosql_worker && console.log('NoSQL PID : ' + framework_nosql.pid()); console.log('Memory : ' + memory.heapUsed.filesize(2) + ' / ' + memory.heapTotal.filesize(2)); + console.log('User : ' + Os.userInfo().username); console.log('===================================================='); console.log('Name : ' + CONF.name); console.log('Version : ' + CONF.version); From 1110d9e961f6d8fed8102cedbcbeaa1969d16e5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Wed, 5 Dec 2018 08:57:32 +0100 Subject: [PATCH 216/217] Updated `LOCALIZE()`. --- changes.txt | 1 + index.js | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/changes.txt b/changes.txt index 9135b9d66..52f44a486 100755 --- a/changes.txt +++ b/changes.txt @@ -42,6 +42,7 @@ - updated: `F.path.mkdir(path, [cache])` can cache a current satte (default: false) - updated: `controller.all()` can return `Array` of all WebSocketClient - updated: startup info by adding user name +- updated: `LOCALIZE()` now `url` arg can be a function which replaces `F.onLocale` - fixed: a critical bug with storing uploaded files via `httpfile.fs()` or `httpfile.nosql()` - fixed: a critical bug with JavaScript minificator diff --git a/index.js b/index.js index 50bf2e2d3..4850ed7cf 100755 --- a/index.js +++ b/index.js @@ -3393,6 +3393,11 @@ function sitemapurl(url) { global.LOCALIZE = F.localize = function(url, flags, minify) { + if (typeof(url) === 'function') { + F.onLocale = url; + return; + } + if (url[0] === '#') url = sitemapurl(url.substring(1)); @@ -3482,7 +3487,6 @@ global.LOCALIZE = F.localize = function(url, flags, minify) { }); }, flags); - return F; }; F.$notModified = function(req, res, date) { From b90209d921e471338b220e64acd6a577fd044d20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Cirka?= Date: Fri, 7 Dec 2018 11:20:54 +0100 Subject: [PATCH 217/217] Updated version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f25f66df8..ce28533b2 100755 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "name": "Sarp Aykent", "email": "shackhers@gmail.com" }], - "version": "3.0.1-33", + "version": "3.0.1", "homepage": "http://www.totaljs.com", "bugs": { "url": "https://github.com/totaljs/framework/issues",