diff --git a/.eslintrc.json b/.eslintrc.json index 70e7644..f82bef2 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -19,7 +19,10 @@ ], "quotes": [ "error", - "single" + "single", + { + "allowTemplateLiterals": true + } ], "semi": [ "error", diff --git a/.travis.yml b/.travis.yml index dee6a06..43720f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,8 @@ node_js: services: - postgresql before_script: - - psql -f db/install.sql -U postgres - - PGPASSWORD=marcus psql -d application -f db/structure.sql -U marcus - - PGPASSWORD=marcus psql -d application -f db/data.sql -U marcus + - psql -f application/db/install.sql -U postgres + - PGPASSWORD=marcus psql -d application -f application/db/structure.sql -U marcus + - PGPASSWORD=marcus psql -d application -f application/db/data.sql -U marcus script: - npm test diff --git a/api/introspection.js b/api/introspection.js deleted file mode 100644 index 5d14b01..0000000 --- a/api/introspection.js +++ /dev/null @@ -1 +0,0 @@ -(application.introspect); diff --git a/api/resmon.js b/api/resmon.js deleted file mode 100644 index 08bfea0..0000000 --- a/api/resmon.js +++ /dev/null @@ -1,9 +0,0 @@ -async () => { - const loadavg = api.os.loadavg(); - const stats = application.resmon.getStatistics(); - const { heapTotal, heapUsed, external, contexts } = stats; - const total = application.utils.bytesToSize(heapTotal); - const used = application.utils.bytesToSize(heapUsed); - const ext = application.utils.bytesToSize(external); - return { total, used, ext, contexts, loadavg }; -}; diff --git a/application/.eslintrc.json b/application/.eslintrc.json new file mode 100644 index 0000000..38d9208 --- /dev/null +++ b/application/.eslintrc.json @@ -0,0 +1,17 @@ +{ + "parserOptions": { + "sourceType": "module" + }, + "rules": { + "id-denylist": [ + 2, + "application", + "node", + "npm", + "api", + "lib", + "domain", + "config" + ] + } +} diff --git a/application/api/.eslintrc.json b/application/api/.eslintrc.json new file mode 100644 index 0000000..0cfae79 --- /dev/null +++ b/application/api/.eslintrc.json @@ -0,0 +1,11 @@ +{ + "globals": { + "application": "readonly", + "config": "readonly", + "context": "readonly", + "node": "readonly", + "npm": "readonly", + "lib": "readonly", + "domain": "readonly" + } +} diff --git a/api/registerUser.js b/application/api/auth.1/register.js similarity index 100% rename from api/registerUser.js rename to application/api/auth.1/register.js diff --git a/api/signIn.js b/application/api/auth.1/signIn.js similarity index 100% rename from api/signIn.js rename to application/api/auth.1/signIn.js diff --git a/api/status.js b/application/api/auth.1/status.js similarity index 100% rename from api/status.js rename to application/api/auth.1/status.js diff --git a/api/.eslintrc.json b/application/api/cms.1/.eslintrc.json similarity index 100% rename from api/.eslintrc.json rename to application/api/cms.1/.eslintrc.json diff --git a/api/about.js b/application/api/cms.1/about.js similarity index 100% rename from api/about.js rename to application/api/cms.1/about.js diff --git a/api/citiesByCountry.js b/application/api/example.1/citiesByCountry.js similarity index 63% rename from api/citiesByCountry.js rename to application/api/example.1/citiesByCountry.js index a146819..cb88c02 100644 --- a/api/citiesByCountry.js +++ b/application/api/example.1/citiesByCountry.js @@ -1,6 +1,6 @@ async ({ countryId }) => { const fields = ['Id', 'Name']; const where = { countryId }; - const data = await application.db.select('City', fields, where); + const data = await domain.database.example.select('City', fields, where); return { result: 'success', data }; }; diff --git a/api/counter.js b/application/api/example.1/counter.js similarity index 100% rename from api/counter.js rename to application/api/example.1/counter.js diff --git a/api/countries.js b/application/api/example.1/countries.js similarity index 55% rename from api/countries.js rename to application/api/example.1/countries.js index 1b926e2..670cbaa 100644 --- a/api/countries.js +++ b/application/api/example.1/countries.js @@ -1,5 +1,5 @@ async () => { const fields = ['Id', 'Name']; - const data = await application.db.select('Country', fields); + const data = await domain.database.example.select('Country', fields); return { result: 'success', data }; }; diff --git a/application/api/example.1/error.js b/application/api/example.1/error.js new file mode 100644 index 0000000..5286309 --- /dev/null +++ b/application/api/example.1/error.js @@ -0,0 +1 @@ +async () => new Error('Hello!', 54321); diff --git a/application/api/example.1/exception.js b/application/api/example.1/exception.js new file mode 100644 index 0000000..ee3fd4d --- /dev/null +++ b/application/api/example.1/exception.js @@ -0,0 +1,3 @@ +async () => { + throw new Error('Hello', 12345); +}; diff --git a/application/api/example.1/getUndefined.js b/application/api/example.1/getUndefined.js new file mode 100644 index 0000000..5443cfd --- /dev/null +++ b/application/api/example.1/getUndefined.js @@ -0,0 +1 @@ +async () => undefined; diff --git a/application/api/example.1/remoteMethod.js b/application/api/example.1/remoteMethod.js new file mode 100644 index 0000000..0ea0279 --- /dev/null +++ b/application/api/example.1/remoteMethod.js @@ -0,0 +1,7 @@ +({ + access: 'public', + method: async ({ ...args }) => { + console.debug({ remoteMethod: args }); + return { result: 'success' }; + } +}); diff --git a/application/api/example.1/resources.js b/application/api/example.1/resources.js new file mode 100644 index 0000000..81e3f90 --- /dev/null +++ b/application/api/example.1/resources.js @@ -0,0 +1,9 @@ +async () => { + const loadavg = node.os.loadavg(); + const stats = lib.resmon.getStatistics(); + const { heapTotal, heapUsed, external, contexts } = stats; + const total = lib.utils.bytesToSize(heapTotal); + const used = lib.utils.bytesToSize(heapUsed); + const ext = lib.utils.bytesToSize(external); + return { total, used, ext, contexts, loadavg }; +}; diff --git a/application/api/example.1/subscribe.js b/application/api/example.1/subscribe.js new file mode 100644 index 0000000..45b4573 --- /dev/null +++ b/application/api/example.1/subscribe.js @@ -0,0 +1,7 @@ +async () => { + setInterval(() => { + const stats = lib.resmon.getStatistics(); + context.client.emit('example/resmon', stats); + }, config.resmon.interval); + return { subscribed: 'resmon' }; +}; diff --git a/application/api/example.1/uploadFile.js b/application/api/example.1/uploadFile.js new file mode 100644 index 0000000..e2eb89b --- /dev/null +++ b/application/api/example.1/uploadFile.js @@ -0,0 +1,9 @@ +async ({ name, data }) => { + const buffer = Buffer.from(data, 'base64'); + const tmpPath = 'application/tmp'; + const filePath = node.path.join(tmpPath, name); + if (filePath.startsWith(tmpPath)) { + await node.fsp.writeFile(filePath, buffer); + } + return { uploaded: data.length }; +}; diff --git a/application/api/example.1/wait.js b/application/api/example.1/wait.js new file mode 100644 index 0000000..132277e --- /dev/null +++ b/application/api/example.1/wait.js @@ -0,0 +1,3 @@ +async ({ delay }) => new Promise(resolve => { + setTimeout(resolve, delay, 'done'); +}); diff --git a/application/api/example.1/webHook.js b/application/api/example.1/webHook.js new file mode 100644 index 0000000..9af71e7 --- /dev/null +++ b/application/api/example.1/webHook.js @@ -0,0 +1,7 @@ +({ + access: 'public', + method: async ({ ...args }) => { + console.debug({ webHook: args }); + return { result: 'success' }; + } +}); diff --git a/application/api/system.1/introspect.js b/application/api/system.1/introspect.js new file mode 100644 index 0000000..c307e61 --- /dev/null +++ b/application/api/system.1/introspect.js @@ -0,0 +1,4 @@ +({ + access: 'public', + method: application.introspect +}); diff --git a/cert/generate.ext b/application/cert/generate.ext similarity index 100% rename from cert/generate.ext rename to application/cert/generate.ext diff --git a/application/cert/generate.sh b/application/cert/generate.sh new file mode 100755 index 0000000..69593e2 --- /dev/null +++ b/application/cert/generate.sh @@ -0,0 +1,5 @@ +cd "$(dirname "$0")" +openssl genrsa -out key.pem 3072 +openssl req -new -out self.pem -key key.pem -subj '/CN=localhost' +openssl req -text -noout -in self.pem +openssl x509 -req -days 1024 -in self.pem -signkey key.pem -out cert.pem -extfile generate.ext diff --git a/config/.eslintrc.json b/application/config/.eslintrc.json similarity index 100% rename from config/.eslintrc.json rename to application/config/.eslintrc.json diff --git a/config/database.js b/application/config/database.js similarity index 100% rename from config/database.js rename to application/config/database.js diff --git a/config/resmon.js b/application/config/resmon.js similarity index 100% rename from config/resmon.js rename to application/config/resmon.js diff --git a/config/server.js b/application/config/server.js similarity index 51% rename from config/server.js rename to application/config/server.js index b033da9..0e7d13d 100644 --- a/config/server.js +++ b/application/config/server.js @@ -1,10 +1,16 @@ ({ host: '127.0.0.1', - ports: [8000, 8001, 8002, 8003], + balancer: 8000, + protocol: 'http', + ports: [8001, 8002], timeout: 5000, concurrency: 1000, queue: { size: 2000, timeout: 3000, }, + workers: { + pool: 2, + timeout: 3000, + }, }); diff --git a/db/data.sql b/application/db/data.sql similarity index 100% rename from db/data.sql rename to application/db/data.sql diff --git a/db/install.sql b/application/db/install.sql similarity index 100% rename from db/install.sql rename to application/db/install.sql diff --git a/db/structure.sql b/application/db/structure.sql similarity index 100% rename from db/structure.sql rename to application/db/structure.sql diff --git a/doc/README.md b/application/doc/README.md similarity index 100% rename from doc/README.md rename to application/doc/README.md diff --git a/domain/.eslintrc.json b/application/domain/.eslintrc.json similarity index 50% rename from domain/.eslintrc.json rename to application/domain/.eslintrc.json index d1e6a44..4cb85e1 100644 --- a/domain/.eslintrc.json +++ b/application/domain/.eslintrc.json @@ -7,7 +7,10 @@ }, "globals": { "application": "readonly", - "context": "readonly", - "api": "readonly" + "config": "readonly", + "node": "readonly", + "npm": "readonly", + "lib": "readonly", + "domain": "readonly" } } diff --git a/application/domain/database/start.js b/application/domain/database/start.js new file mode 100644 index 0000000..80a2b6e --- /dev/null +++ b/application/domain/database/start.js @@ -0,0 +1,4 @@ +(async () => { + console.debug('Connect to pg'); + domain.database.example = new lib.pg.Database(config.database); +}); diff --git a/application/lib/.eslintrc.json b/application/lib/.eslintrc.json new file mode 100644 index 0000000..4cb85e1 --- /dev/null +++ b/application/lib/.eslintrc.json @@ -0,0 +1,16 @@ +{ + "rules": { + "strict": [ + "error", + "never" + ] + }, + "globals": { + "application": "readonly", + "config": "readonly", + "node": "readonly", + "npm": "readonly", + "lib": "readonly", + "domain": "readonly" + } +} diff --git a/application/lib/example/add.js b/application/lib/example/add.js new file mode 100644 index 0000000..3cb282f --- /dev/null +++ b/application/lib/example/add.js @@ -0,0 +1,13 @@ +({ + parameters: { + a: 'number', + b: 'number', + }, + + method({ a, b }) { + console.log({ a, b }); + return a + b; + }, + + returns: 'number', +}); diff --git a/application/lib/example/cache.js b/application/lib/example/cache.js new file mode 100644 index 0000000..644ce3b --- /dev/null +++ b/application/lib/example/cache.js @@ -0,0 +1,15 @@ +({ + values: new Map(), + + set({ key, val }) { + console.log({ set: { key, val } }); + return this.values.set(key, val); + }, + + get({ key }) { + console.log({ get: key }); + const res = this.values.get(key); + console.log({ return: res }); + return res; + } +}); diff --git a/application/lib/example/doSomething.js b/application/lib/example/doSomething.js new file mode 100644 index 0000000..69710ab --- /dev/null +++ b/application/lib/example/doSomething.js @@ -0,0 +1,3 @@ +(() => { + console.debug('Call method: example.doSomething'); +}); diff --git a/application/lib/example/start.js b/application/lib/example/start.js new file mode 100644 index 0000000..ed5eac9 --- /dev/null +++ b/application/lib/example/start.js @@ -0,0 +1,10 @@ +({ + privateField: 100, + + async method() { + console.log('Start example plugin'); + this.parent.cache.set({ key: 'keyName', val: this.privateField }); + const res = lib.example.cache.get({ key: 'keyName' }); + console.log({ res, cache: this.parent.cache.values }); + } +}); diff --git a/application/lib/example/stop.js b/application/lib/example/stop.js new file mode 100644 index 0000000..a386621 --- /dev/null +++ b/application/lib/example/stop.js @@ -0,0 +1,3 @@ +(async () => { + console.debug('Stop example plugin'); +}); diff --git a/application/lib/example/storage/set.js b/application/lib/example/storage/set.js new file mode 100644 index 0000000..b66f5ad --- /dev/null +++ b/application/lib/example/storage/set.js @@ -0,0 +1,13 @@ +({ + values: new Map(), + + method({ key, val }) { + console.log({ key, val }); + if (val) { + return this.values.set(key, val); + } + const res = this.values.get(key); + console.log({ return: { res } }); + return res; + } +}); diff --git a/application/lib/example/submodule1/method1.js b/application/lib/example/submodule1/method1.js new file mode 100644 index 0000000..82bda04 --- /dev/null +++ b/application/lib/example/submodule1/method1.js @@ -0,0 +1,3 @@ +(() => { + console.debug('Call method: example.submodule1.method1'); +}); diff --git a/application/lib/example/submodule1/method2.js b/application/lib/example/submodule1/method2.js new file mode 100644 index 0000000..e16524d --- /dev/null +++ b/application/lib/example/submodule1/method2.js @@ -0,0 +1,3 @@ +(() => { + console.debug('Call method: example.submodule1.method2'); +}); diff --git a/application/lib/example/submodule2/method1.js b/application/lib/example/submodule2/method1.js new file mode 100644 index 0000000..cb99f42 --- /dev/null +++ b/application/lib/example/submodule2/method1.js @@ -0,0 +1,3 @@ +(() => { + console.debug('Call method: example.submodule2.method1'); +}); diff --git a/application/lib/example/submodule2/method2.js b/application/lib/example/submodule2/method2.js new file mode 100644 index 0000000..58e0df7 --- /dev/null +++ b/application/lib/example/submodule2/method2.js @@ -0,0 +1,3 @@ +(() => { + console.debug('Call method: example.submodule2.method2'); +}); diff --git a/application/lib/example/submodule2/nested1/method1.js b/application/lib/example/submodule2/nested1/method1.js new file mode 100644 index 0000000..19036d1 --- /dev/null +++ b/application/lib/example/submodule2/nested1/method1.js @@ -0,0 +1,3 @@ +(() => { + console.debug('Call method: example.submodule2.nested1.method1'); +}); diff --git a/application/lib/example/submodule3/nested2/method1.js b/application/lib/example/submodule3/nested2/method1.js new file mode 100644 index 0000000..c640472 --- /dev/null +++ b/application/lib/example/submodule3/nested2/method1.js @@ -0,0 +1,3 @@ +(() => { + console.debug('Call method: example.submodule3.nested2.method1'); +}); diff --git a/application/lib/pg/Database.js b/application/lib/pg/Database.js new file mode 100644 index 0000000..071422a --- /dev/null +++ b/application/lib/pg/Database.js @@ -0,0 +1,58 @@ +(class Database { + constructor(options) { + this.pool = new npm.pg.Pool(options); + } + + query(sql, values) { + const data = values ? values.join(',') : ''; + console.debug(`${sql}\t[${data}]`); + return this.pool.query(sql, values); + } + + insert(table, record) { + const keys = Object.keys(record); + const nums = new Array(keys.length); + const data = new Array(keys.length); + let i = 0; + for (const key of keys) { + data[i] = record[key]; + nums[i] = `$${++i}`; + } + const fields = keys.join(', '); + const params = nums.join(', '); + const sql = `INSERT INTO ${table} (${fields}) VALUES (${params})`; + return this.query(sql, data); + } + + async select(table, fields = ['*'], conditions = null) { + const keys = fields.join(', '); + const sql = `SELECT ${keys} FROM ${table}`; + let whereClause = ''; + let args = []; + if (conditions) { + const whereData = lib.pg.utils.where(conditions); + whereClause = ' WHERE ' + whereData.clause; + args = whereData.args; + } + const res = await this.query(sql + whereClause, args); + return res.rows; + } + + delete(table, conditions = null) { + const { clause, args } = lib.pg.utils.where(conditions); + const sql = `DELETE FROM ${table} WHERE ${clause}`; + return this.query(sql, args); + } + + update(table, delta = null, conditions = null) { + const upd = lib.pg.utils.updates(delta); + const cond = lib.pg.utils.where(conditions, upd.args.length + 1); + const sql = `UPDATE ${table} SET ${upd.clause} WHERE ${cond.clause}`; + const args = [...upd.args, ...cond.args]; + return this.query(sql, args); + } + + close() { + this.pool.end(); + } +}); diff --git a/application/lib/pg/constants.js b/application/lib/pg/constants.js new file mode 100644 index 0000000..362dd39 --- /dev/null +++ b/application/lib/pg/constants.js @@ -0,0 +1,3 @@ +({ + OPERATORS: ['>=', '<=', '<>', '>', '<'] +}); diff --git a/application/lib/pg/updates.js b/application/lib/pg/updates.js new file mode 100644 index 0000000..2d6492a --- /dev/null +++ b/application/lib/pg/updates.js @@ -0,0 +1,12 @@ +((delta, firstArgIndex = 1) => { + const clause = []; + const args = []; + let i = firstArgIndex; + const keys = Object.keys(delta); + for (const key of keys) { + const value = delta[key].toString(); + clause.push(`${key} = $${i++}`); + args.push(value); + } + return { clause: clause.join(', '), args }; +}); diff --git a/application/lib/pg/where.js b/application/lib/pg/where.js new file mode 100644 index 0000000..8a9c606 --- /dev/null +++ b/application/lib/pg/where.js @@ -0,0 +1,26 @@ +((conditions, firstArgIndex = 1) => { + const clause = []; + const args = []; + let i = firstArgIndex; + const keys = Object.keys(conditions); + for (const key of keys) { + let operator = '='; + let value = conditions[key]; + if (typeof value === 'string') { + for (const op of lib.pg.constants.OPERATORS) { + const len = op.length; + if (value.startsWith(op)) { + operator = op; + value = value.substring(len); + } + } + if (value.includes('*') || value.includes('?')) { + operator = 'LIKE'; + value = value.replace(/\*/g, '%').replace(/\?/g, '_'); + } + } + clause.push(`${key} ${operator} $${i++}`); + args.push(value); + } + return { clause: clause.join(' AND '), args }; +}); diff --git a/application/lib/resmon/getStatistics.js b/application/lib/resmon/getStatistics.js new file mode 100644 index 0000000..447c7db --- /dev/null +++ b/application/lib/resmon/getStatistics.js @@ -0,0 +1,7 @@ +(() => { + const { heapTotal, heapUsed, external } = node.process.memoryUsage(); + const hs = node.v8.getHeapStatistics(); + const contexts = hs.number_of_native_contexts; + const detached = hs.number_of_detached_contexts; + return { heapTotal, heapUsed, external, contexts, detached }; +}); diff --git a/application/lib/resmon/start.js b/application/lib/resmon/start.js new file mode 100644 index 0000000..b0fc39e --- /dev/null +++ b/application/lib/resmon/start.js @@ -0,0 +1,13 @@ +(async () => { + if (config.resmon.active) { + setInterval(() => { + const stats = lib.resmon.getStatistics(); + const { heapTotal, heapUsed, external, contexts, detached } = stats; + const total = lib.utils.bytesToSize(heapTotal); + const used = lib.utils.bytesToSize(heapUsed); + const ext = lib.utils.bytesToSize(external); + console.debug(`Heap: ${used} of ${total}, ext: ${ext}`); + console.debug(`Contexts: ${contexts}, detached: ${detached}`); + }, config.resmon.interval); + } +}); diff --git a/application/lib/utils/bytesToSize.js b/application/lib/utils/bytesToSize.js new file mode 100644 index 0000000..5ecc9f7 --- /dev/null +++ b/application/lib/utils/bytesToSize.js @@ -0,0 +1,9 @@ +(bytes => { + const UNITS = ['', ' Kb', ' Mb', ' Gb', ' Tb', ' Pb', ' Eb', ' Zb', ' Yb']; + if (bytes === 0) return '0'; + const exp = Math.floor(Math.log(bytes) / Math.log(1000)); + const size = bytes / 1000 ** exp; + const short = Math.round(size, 2); + const unit = UNITS[exp]; + return short + unit; +}); diff --git a/tmp/.gitkeep b/application/schemas/.gitkeep similarity index 100% rename from tmp/.gitkeep rename to application/schemas/.gitkeep diff --git a/static/.eslintrc.json b/application/static/.eslintrc.json similarity index 51% rename from static/.eslintrc.json rename to application/static/.eslintrc.json index bd14a19..2f25734 100644 --- a/static/.eslintrc.json +++ b/application/static/.eslintrc.json @@ -1,5 +1,10 @@ { "parserOptions": { "sourceType": "module" + }, + "rules": { + "id-denylist": [ + 0 + ] } } diff --git a/static/console.css b/application/static/console.css similarity index 100% rename from static/console.css rename to application/static/console.css diff --git a/static/console.js b/application/static/console.js similarity index 81% rename from static/console.js rename to application/static/console.js index 11cc39a..90ee62c 100644 --- a/static/console.js +++ b/application/static/console.js @@ -1,6 +1,7 @@ import { Metacom } from './metacom.js'; -const metacom = new Metacom(location.host); +const protocol = location.protocol === 'http:' ? 'ws' : 'wss'; +const metacom = new Metacom(`${protocol}://${location.host}`); const { api } = metacom; window.api = api; @@ -228,11 +229,60 @@ document.onkeypress = event => { } }; +const blobToBase64 = blob => { + const reader = new FileReader(); + reader.readAsDataURL(blob); + return new Promise(resolve => { + reader.onloadend = () => { + resolve(reader.result); + }; + }); +}; + +const uploadFile = (file, done) => { + blobToBase64(file) + .then(url => { + const data = url.substring(url.indexOf(',') + 1); + api.example.uploadFile({ name: file.name, data }).then(done); + }); +}; + +const upload = () => { + const element = document.createElement('form'); + element.style.visibility = 'hidden'; + element.innerHTML = ''; + document.body.appendChild(element); + const fileSelect = document.getElementById('fileSelect'); + fileSelect.click(); + fileSelect.onchange = () => { + const files = Array.from(fileSelect.files); + print('Uploading ' + files.length + ' file(s)'); + files.sort((a, b) => a.size - b.size); + let i = 0; + const uploadNext = () => { + const file = files[i]; + uploadFile(file, () => { + print(`name: ${file.name}, size: ${file.size} done`); + i++; + if (i < files.length) { + return uploadNext(); + } + document.body.removeChild(element); + commandLoop(); + }); + }; + uploadNext(); + }; +}; + const exec = async line => { const args = line.split(' '); - const cmd = args.shift(); - const data = await api[cmd](args); - print(data); + if (args[0] === 'upload') { + upload(); + } else { + const data = await api.cms.content(args); + print(data); + } commandLoop(); }; @@ -245,12 +295,14 @@ function commandLoop() { const signIn = async () => { try { - await metacom.load('status', 'signIn', 'introspection'); - await api.status(); + await metacom.load('auth'); + await api.auth.status(); } catch (err) { - await api.signIn({ login: 'marcus', password: 'marcus' }); + await api.auth.signIn({ login: 'marcus', password: 'marcus' }); } await metacom.load('example'); + api.example.on('resmon', data => print(JSON.stringify(data))); + api.example.subscribe(); }; window.addEventListener('load', () => { diff --git a/static/favicon.ico b/application/static/favicon.ico similarity index 100% rename from static/favicon.ico rename to application/static/favicon.ico diff --git a/static/favicon.png b/application/static/favicon.png similarity index 100% rename from static/favicon.png rename to application/static/favicon.png diff --git a/static/index.html b/application/static/index.html similarity index 100% rename from static/index.html rename to application/static/index.html diff --git a/static/manifest.json b/application/static/manifest.json similarity index 100% rename from static/manifest.json rename to application/static/manifest.json diff --git a/application/static/metacom.js b/application/static/metacom.js new file mode 100644 index 0000000..9b5fede --- /dev/null +++ b/application/static/metacom.js @@ -0,0 +1,129 @@ +class MetacomError extends Error { + constructor(message, code) { + super(message); + this.code = code; + } +} + +class MetacomInterface { + constructor() { + this._events = new Map(); + } + + on(name, fn) { + const event = this._events.get(name); + if (event) event.add(fn); + else this._events.set(name, new Set([fn])); + } + + emit(name, ...args) { + const event = this._events.get(name); + if (!event) return; + for (const fn of event.values()) fn(...args); + } +} + +export class Metacom { + constructor(url) { + this.url = url; + this.socket = new WebSocket(url); + this.api = {}; + this.callId = 0; + this.calls = new Map(); + this.socket.addEventListener('message', ({ data }) => { + this.message(data); + }); + } + + message(data) { + let packet; + try { + packet = JSON.parse(data); + } catch (err) { + console.error(err); + return; + } + const [callType, target] = Object.keys(packet); + const callId = packet[callType]; + const args = packet[target]; + if (callId && args) { + if (callType === 'callback') { + const promised = this.calls.get(callId); + if (!promised) return; + const [resolve, reject] = promised; + if (packet.error) { + const { message, code } = packet.error; + const error = new MetacomError(message, code); + reject(error); + return; + } + resolve(args); + return; + } + if (callType === 'event') { + const [interfaceName, eventName] = target.split('/'); + const metacomInterface = this.api[interfaceName]; + metacomInterface.emit(eventName, args); + } + } + } + + ready() { + return new Promise(resolve => { + if (this.socket.readyState === WebSocket.OPEN) resolve(); + else this.socket.addEventListener('open', resolve); + }); + } + + async load(...interfaces) { + const introspect = this.httpCall('system')('introspect'); + const introspection = await introspect(interfaces); + const available = Object.keys(introspection); + for (const interfaceName of interfaces) { + if (!available.includes(interfaceName)) continue; + const methods = new MetacomInterface(); + const iface = introspection[interfaceName]; + const request = this.socketCall(interfaceName); + const methodNames = Object.keys(iface); + for (const methodName of methodNames) { + methods[methodName] = request(methodName); + } + this.api[interfaceName] = methods; + } + } + + httpCall(iname, ver) { + return methodName => (args = {}) => { + const callId = ++this.callId; + const interfaceName = ver ? `${iname}.${ver}` : iname; + const target = interfaceName + '/' + methodName; + const packet = { call: callId, [target]: args }; + const dest = new URL(this.url); + const protocol = dest.protocol === 'ws:' ? 'http' : 'https'; + const url = `${protocol}://${dest.host}/api`; + return fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(packet), + }).then(res => { + const { status } = res; + if (status === 200) return res.json().then(({ result }) => result); + throw new Error(`Status Code: ${status}`); + }); + }; + } + + socketCall(iname, ver) { + return methodName => async (args = {}) => { + const callId = ++this.callId; + const interfaceName = ver ? `${iname}.${ver}` : iname; + const target = interfaceName + '/' + methodName; + await this.ready(); + return new Promise((resolve, reject) => { + this.calls.set(callId, [resolve, reject]); + const packet = { call: callId, [target]: args }; + this.socket.send(JSON.stringify(packet)); + }); + }; + } +} diff --git a/static/metarhia.png b/application/static/metarhia.png similarity index 100% rename from static/metarhia.png rename to application/static/metarhia.png diff --git a/static/metarhia.svg b/application/static/metarhia.svg similarity index 100% rename from static/metarhia.svg rename to application/static/metarhia.svg diff --git a/application/tasks/.gitkeep b/application/tasks/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/application/test/.gitkeep b/application/test/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/application/tmp/.gitkeep b/application/tmp/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/cert/generate.sh b/cert/generate.sh deleted file mode 100755 index a02da77..0000000 --- a/cert/generate.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh - -cd "$(dirname "$0")" - -KEY_FILE=key.pem -if [ -f "$KEY_FILE" ]; then - read -e -p "Are you sure you want to replace existing key? [y/N] " YES_NO - if [ "$YES_NO" != "y" ] && [ "$YES_NO" != "Y" ]; then - exit 0 - fi -fi - -if [ "$1" == "secure" ]; then - echo "Generating private ed25519 key" - openssl genpkey -algorithm ed25519 -out $KEY_FILE -else - echo "Generating private RSA3072 key" - openssl genrsa -out key.pem 3072 -fi - -set -e - -echo "Generating certificate signing request" -openssl req -new -out self.pem -key $KEY_FILE -subj '/CN=localhost' - -openssl req -text -noout -in self.pem - -echo "Generating certificate" -openssl x509 -req -days 1024 -in self.pem -signkey $KEY_FILE -out cert.pem -extfile generate.ext diff --git a/domain/resmon.js b/domain/resmon.js deleted file mode 100644 index f1ea0c1..0000000 --- a/domain/resmon.js +++ /dev/null @@ -1,21 +0,0 @@ -({ - getStatistics() { - const { heapTotal, heapUsed, external } = api.process.memoryUsage(); - const hs = api.v8.getHeapStatistics(); - const contexts = hs.number_of_native_contexts; - const detached = hs.number_of_detached_contexts; - return { heapTotal, heapUsed, external, contexts, detached }; - }, - - start() { - const { interval } = application.resmon.config; - setTimeout(() => { - const stats = application.resmon.getStatistics(); - const { heapTotal, heapUsed, external } = stats; - const total = application.utils.bytesToSize(heapTotal); - const used = application.utils.bytesToSize(heapUsed); - const ext = application.utils.bytesToSize(external); - console.log(`Heap: ${used} of ${total}, ext: ${ext}`); - }, interval); - } -}); diff --git a/domain/utils.js b/domain/utils.js deleted file mode 100644 index d007ccf..0000000 --- a/domain/utils.js +++ /dev/null @@ -1,11 +0,0 @@ -({ - bytesToSize(bytes) { - const UNITS = ['', ' Kb', ' Mb', ' Gb', ' Tb', ' Pb', ' Eb', ' Zb', ' Yb']; - if (bytes === 0) return '0'; - const exp = Math.floor(Math.log(bytes) / Math.log(1000)); - const size = bytes / 1000 ** exp; - const short = Math.round(size, 2); - const unit = UNITS[exp]; - return short + unit; - } -}); diff --git a/lib/application.js b/lib/application.js index 629429c..531ac46 100644 --- a/lib/application.js +++ b/lib/application.js @@ -1,42 +1,76 @@ 'use strict'; -const api = require('./dependencies.js'); -const { path, events, vm, fs, fsp } = api; +const { node, npm } = require('./dependencies.js'); +const { path, events, vm, fs, fsp } = node; +const { common } = npm; + const security = require('./security.js'); const SCRIPT_OPTIONS = { timeout: 5000 }; const EMPTY_CONTEXT = Object.freeze({}); +const MODULE = 2; class Application extends events.EventEmitter { constructor() { super(); + this.initialization = true; this.finalization = false; this.namespaces = ['db']; - this.path = process.cwd(); - this.staticPath = path.join(this.path, 'static'); - this.api = new Map(); - this.domain = new Map(); + this.api = {}; this.static = new Map(); + this.root = process.cwd(); + this.path = path.join(this.root, 'application'); + this.apiPath = path.join(this.path, 'api'); + this.libPath = path.join(this.path, 'lib'); + this.domainPath = path.join(this.path, 'domain'); + this.staticPath = path.join(this.path, 'static'); + this.starts = []; } async init() { this.createSandbox(); - await this.loadPlace('api', path.join(this.path, 'api')); - await this.loadPlace('domain', path.join(this.path, 'domain')); - await this.loadPlace('static', path.join(this.path, 'static')); + await Promise.allSettled([ + this.loadPlace('static', this.staticPath), + this.loadPlace('api', this.apiPath), + (async () => { + await this.loadPlace('lib', this.libPath); + await this.loadPlace('domain', this.domainPath); + })(), + ]); + await Promise.allSettled(this.starts.map(fn => fn())); + this.starts = null; + this.initialization = true; } async shutdown() { this.finalization = true; - await this.server.close(); + await this.stopPlace('domain'); + await this.stopPlace('lib'); + if (this.server) await this.server.close(); + await this.logger.close(); + } + + async stopPlace(name) { + const place = this.sandbox[name]; + for (const moduleName of Object.keys(place)) { + const module = place[moduleName]; + if (module.stop) await this.execute(module.stop); + } } createSandbox() { - const introspect = async () => [...this.api.keys()]; - const application = { security, introspect }; - for (const name of this.namespaces) application[name] = this[name]; + const { config, namespaces, server: { host, port, protocol } = {} } = this; + const introspect = async interfaces => this.introspect(interfaces); + const worker = { id: 'W' + node.worker.threadId.toString() }; + const server = { host, port, protocol }; + const application = { security, introspect, worker, server }; + const api = {}; + const lib = {}; + const domain = {}; + for (const name of namespaces) application[name] = this[name]; const sandbox = { - console: this.logger, Buffer, application, api, + Buffer, URL, URLSearchParams, Error: this.Error, console: this.logger, + application, node, npm, api, lib, domain, config, setTimeout, setImmediate, setInterval, clearTimeout, clearImmediate, clearInterval, }; @@ -57,16 +91,95 @@ class Application extends events.EventEmitter { const script = new vm.Script(src, options); return script.runInContext(this.sandbox, SCRIPT_OPTIONS); } catch (err) { - if (err.code !== 'ENOENT') this.logger.error(err.stack); + if (err.code !== 'ENOENT') { + this.logger.error(err.stack); + } return null; } } - runMethod(methodName, session) { - const script = this.api.get(methodName); - if (!script) return null; - const exp = script(session ? session.context : EMPTY_CONTEXT); - return typeof exp !== 'object' ? { access: 'logged', method: exp } : exp; + getMethod(iname, ver, methodName, context) { + const iface = this.api[iname]; + if (!iface) return null; + const version = ver === '*' ? iface.default : parseInt(ver); + const methods = iface[version.toString()]; + if (!methods) return null; + const method = methods[methodName]; + if (!method) return null; + const exp = method(context); + return typeof exp === 'object' ? exp : { access: 'logged', method: exp }; + } + + async loadMethod(fileName) { + const rel = fileName.substring(this.apiPath.length + 1); + if (!rel.includes('/')) return; + const [interfaceName, methodFile] = rel.split('/'); + if (!methodFile.endsWith('.js')) return; + const name = path.basename(methodFile, '.js'); + const [iname, ver] = interfaceName.split('.'); + const version = parseInt(ver, 10); + const script = await this.createScript(fileName); + if (!script) return; + let iface = this.api[iname]; + const { api } = this.sandbox; + let internalInterface = api[iname]; + if (!iface) { + this.api[iname] = iface = { default: version }; + api[iname] = internalInterface = {}; + } + let methods = iface[ver]; + if (!methods) iface[ver] = methods = {}; + methods[name] = script; + internalInterface[name] = script(EMPTY_CONTEXT); + if (version > iface.default) iface.default = version; + } + + async loadModule(fileName) { + const rel = fileName.substring(this.path.length + 1); + if (!rel.endsWith('.js')) return; + const script = await this.createScript(fileName); + const name = path.basename(rel, '.js'); + const namespaces = rel.split(path.sep); + namespaces[namespaces.length - 1] = name; + const exp = script ? script(EMPTY_CONTEXT) : null; + const container = typeof exp === 'function' ? { method: exp } : exp; + const iface = {}; + if (container !== null) { + const methods = Object.keys(container); + for (const method of methods) { + const fn = container[method]; + if (typeof fn === 'function') { + container[method] = iface[method] = fn.bind(container); + } + } + } + let level = this.sandbox; + const last = namespaces.length - 1; + for (let depth = 0; depth <= last; depth++) { + const namespace = namespaces[depth]; + let next = level[namespace]; + if (next) { + if (depth === MODULE && namespace === 'stop') { + if (exp === null && level.stop) await this.execute(level.stop); + } + } else { + next = depth === last ? iface : {}; + level[namespace] = iface.method || iface; + container.parent = level; + if (depth === MODULE && namespace === 'start') { + this.starts.push(iface.method); + } + } + level = next; + } + } + + async execute(fn) { + try { + await fn(); + } catch (err) { + this.logger.error(err.stack); + } } async loadFile(filePath) { @@ -75,47 +188,86 @@ class Application extends events.EventEmitter { const data = await fsp.readFile(filePath); this.static.set(key, data); } catch (err) { - if (err.code !== 'ENOENT') this.logger.error(err.stack); - } - } - - async loadScript(place, fileName) { - const { name, ext } = path.parse(fileName); - if (ext !== '.js' || name.startsWith('.')) return; - const script = await this.createScript(fileName); - const scripts = this[place]; - if (!script) { - scripts.delete(name); - return; - } - if (place === 'domain') { - const config = this.config.sections[name]; - this.sandbox.application[name] = { config }; - const exp = script(EMPTY_CONTEXT); - if (config) exp.config = config; - this.sandbox.application[name] = exp; - this.sandboxInject(name, exp); - if (exp.start) exp.start(); - } else { - scripts.set(name, script); + if (err.code !== 'ENOENT') { + this.logger.error(err.stack); + } } } async loadPlace(place, placePath) { const files = await fsp.readdir(placePath, { withFileTypes: true }); - const isStatic = place === 'static'; for (const file of files) { + if (file.name.startsWith('.')) continue; const filePath = path.join(placePath, file.name); - if (!isStatic) await this.loadScript(place, filePath); - else if (file.isDirectory()) await this.loadPlace(place, filePath); - else await this.loadFile(filePath); + if (file.isDirectory()) await this.loadPlace(place, filePath); + else if (place === 'api') await this.loadMethod(filePath); + else if (place === 'static') await this.loadFile(filePath); + else await this.loadModule(filePath); } - fs.watch(placePath, (event, fileName) => { + this.watch(place, placePath); + } + + watch(place, placePath) { + fs.watch(placePath, async (event, fileName) => { + if (fileName.startsWith('.')) return; const filePath = path.join(placePath, fileName); - if (isStatic) this.loadFile(filePath); - else this.loadScript(place, filePath); + try { + const stat = await node.fsp.stat(filePath); + if (stat.isDirectory()) { + this.loadPlace(place, filePath); + return; + } + } catch { + return; + } + if (node.worker.threadId === 1) { + const relPath = filePath.substring(this.path.length); + this.logger.debug('Reload: ' + relPath); + } + if (place === 'api') this.loadMethod(filePath); + else if (place === 'static') this.loadFile(filePath); + else this.loadModule(filePath); }); } + + introspect(interfaces) { + const intro = {}; + for (const interfaceName of interfaces) { + const [iname, ver = '*'] = interfaceName.split('.'); + const iface = this.api[iname]; + if (!iface) continue; + const version = ver === '*' ? iface.default : parseInt(ver); + const methods = iface[version.toString()]; + const methodNames = Object.keys(methods); + const interfaceMethods = intro[iname] = {}; + for (const methodName of methodNames) { + const exp = methods[methodName](EMPTY_CONTEXT); + const fn = typeof exp === 'object' ? exp.method : exp; + const src = fn.toString(); + const signature = common.between(src, '({', '})'); + if (signature === '') { + interfaceMethods[methodName] = []; + continue; + } + const args = signature.split(',').map(s => s.trim()); + interfaceMethods[methodName] = args; + } + } + return intro; + } + + getStaticFile(fileName) { + return this.static.get(fileName); + } } -module.exports = new Application(); +const application = new Application(); + +application.Error = class extends Error { + constructor(message, code) { + super(message); + this.code = code; + } +}; + +module.exports = application; diff --git a/lib/auth.js b/lib/auth.js index 0faa28d..c5d3d73 100644 --- a/lib/auth.js +++ b/lib/auth.js @@ -1,6 +1,8 @@ 'use strict'; -const { crypto, common } = require('./dependencies.js'); +const { node, npm } = require('./dependencies.js'); +const { crypto } = node; +const { common } = npm; const application = require('./application.js'); const BYTE = 256; diff --git a/lib/channel.js b/lib/channel.js new file mode 100644 index 0000000..c5f3b7c --- /dev/null +++ b/lib/channel.js @@ -0,0 +1,176 @@ +'use strict'; + +const { http, path } = require('./dependencies.js').node; + +const MIME_TYPES = { + html: 'text/html; charset=UTF-8', + json: 'application/json; charset=UTF-8', + js: 'application/javascript; charset=UTF-8', + css: 'text/css', + png: 'image/png', + ico: 'image/x-icon', + svg: 'image/svg+xml', +}; + +const HEADERS = { + 'X-XSS-Protection': '1; mode=block', + 'X-Content-Type-Options': 'nosniff', + 'Strict-Transport-Security': 'max-age=31536000; includeSubdomains; preload', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'POST, GET, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type', + 'Content-Security-Policy': [ + `default-src 'self' ws:`, + `style-src 'self' https://fonts.googleapis.com`, + `font-src 'self' https://fonts.gstatic.com`, + ].join('; '), +}; + +class Client { + constructor(connection) { + this.callId = 0; + this.connection = connection; + } + + emit(name, data) { + const packet = { event: --this.callId, [name]: data }; + this.connection.send(JSON.stringify(packet)); + } +} + +class Channel { + constructor(req, res, connection, application) { + this.req = req; + this.res = res; + this.ip = req.socket.remoteAddress; + this.connection = connection; + this.application = application; + this.client = new Client(connection); + this.session = null; + return this.init(); + } + + async init() { + this.session = await this.application.auth.restore(this); + return this; + } + + static() { + const { + req: { url, method }, + res, + ip, + application, + } = this; + const filePath = url === '/' ? '/index.html' : url; + const fileExt = path.extname(filePath).substring(1); + const mimeType = MIME_TYPES[fileExt] || MIME_TYPES.html; + res.writeHead(200, { ...HEADERS, 'Content-Type': mimeType }); + if (res.writableEnded) return; + const data = application.getStaticFile(filePath); + if (data) { + res.end(data); + application.logger.access(`${ip}\t${method}\t${url}`); + return; + } + this.error(404); + } + + redirect(location) { + const { res } = this; + if (res.headersSent) return; + res.writeHead(302, { Location: location, ...HEADERS }); + res.end(); + } + + options() { + const { res } = this; + if (res.headersSent) return; + res.writeHead(200, HEADERS); + res.end(); + } + + error(code, err, callId = err) { + const { + req: { url, method }, + res, + connection, + ip, + application, + } = this; + const status = http.STATUS_CODES[code]; + if (typeof err === 'number') err = undefined; + const reason = err ? err.stack : status; + application.logger.error(`${ip}\t${method}\t${url}\t${code}\t${reason}`); + const { Error } = this.application; + const message = err instanceof Error ? err.message : status; + const error = { message, code }; + if (connection) { + connection.send(JSON.stringify({ callback: callId, error })); + return; + } + if (res.writableEnded) return; + res.writeHead(code, { 'Content-Type': MIME_TYPES.json, ...HEADERS }); + res.end(JSON.stringify({ error })); + } + + message(data) { + let packet; + try { + packet = JSON.parse(data); + } catch (err) { + this.error(500, new Error('JSON parsing error')); + return; + } + const [callType, target] = Object.keys(packet); + const callId = packet[callType]; + const args = packet[target]; + if (callId && args) { + const [interfaceName, methodName] = target.split('/'); + this.rpc(callId, interfaceName, methodName, args); + return; + } + this.error(500, new Error('Packet structure error')); + } + + async rpc(callId, interfaceName, methodName, args) { + const { res, connection, ip, application, session, client } = this; + const [iname, ver = '*'] = interfaceName.split('.'); + try { + const context = session ? session.context : { client }; + const proc = application.getMethod(iname, ver, methodName, context); + if (!proc) { + this.error(404, callId); + return; + } + if (!this.session && proc.access !== 'public') { + this.error(403, callId); + return; + } + const result = await proc.method(args); + if (result instanceof Error) { + this.error(result.code, result, callId); + return; + } + const userId = result ? result.userId : undefined; + if (!this.session && userId && proc.access === 'public') { + this.session = application.auth.start(this, userId); + result.token = this.session.token; + } + const data = JSON.stringify({ callback: callId, result }); + if (connection) { + connection.send(data); + } else { + res.writeHead(200, { 'Content-Type': MIME_TYPES.json, ...HEADERS }); + res.end(data); + } + const token = this.session ? this.session.token : 'anonymous'; + const record = `${ip}\t${token}\t${interfaceName}/${methodName}`; + application.logger.access(record); + } catch (err) { + this.error(500, err, callId); + } + } +} + +module.exports = Channel; diff --git a/lib/client.js b/lib/client.js deleted file mode 100644 index ae354b6..0000000 --- a/lib/client.js +++ /dev/null @@ -1,133 +0,0 @@ -'use strict'; - -const { http, path } = require('./dependencies.js'); -const application = require('./application.js'); - -const MIME_TYPES = { - html: 'text/html; charset=UTF-8', - json: 'application/json; charset=UTF-8', - js: 'application/javascript; charset=UTF-8', - css: 'text/css', - png: 'image/png', - ico: 'image/x-icon', - svg: 'image/svg+xml', -}; - -const HEADERS = { - 'X-XSS-Protection': '1; mode=block', - 'X-Content-Type-Options': 'nosniff', - 'Strict-Transport-Security': 'max-age=31536000; includeSubdomains; preload', - 'Access-Control-Allow-Origin': '*', - 'Content-Security-Policy': [ - 'default-src \'self\'', - 'style-src \'self\' https://fonts.googleapis.com', - 'font-src \'self\' https://fonts.gstatic.com', - ].join('; '), -}; - -class Client { - constructor(req, res, connection) { - this.req = req; - this.res = res; - this.ip = req.socket.remoteAddress; - this.connection = connection; - } - - static() { - const { req: { url, method }, res, ip } = this; - const filePath = url === '/' ? '/index.html' : url; - const fileExt = path.extname(filePath).substring(1); - const mimeType = MIME_TYPES[fileExt] || MIME_TYPES.html; - res.writeHead(200, { ...HEADERS, 'Content-Type': mimeType }); - if (res.writableEnded) return; - const data = application.static.get(filePath); - if (data) { - res.end(data); - application.logger.log(`${ip}\t${method}\t${url}`); - return; - } - this.error(404); - } - - redirect(location) { - const { res } = this; - if (res.headersSent) return; - res.writeHead(302, { 'Location': location }); - res.end(); - } - - error(code, err, callId = err) { - const { req: { url, method }, res, connection, ip } = this; - const status = http.STATUS_CODES[code]; - if (typeof err === 'number') err = undefined; - const reason = err ? err.stack : status; - application.logger.error(`${ip}\t${method}\t${url}\t${code}\t${reason}`); - if (connection) { - const packet = { callback: callId, error: { code, message: status } }; - connection.send(JSON.stringify(packet)); - return; - } - if (res.writableEnded) return; - res.writeHead(code, { 'Content-Type': MIME_TYPES.json }); - const packet = { code, error: status }; - res.end(JSON.stringify(packet)); - } - - message(data) { - let packet; - try { - packet = JSON.parse(data); - } catch (err) { - this.error(500, new Error('JSON parsing error')); - return; - } - const [callType, methodName] = Object.keys(packet); - const callId = packet[callType]; - const args = packet[methodName]; - if (callId && args) { - this.rpc(callId, methodName, args); - return; - } - this.error(500, new Error('Packet structure error')); - } - - async rpc(callId, method, args) { - const { res, connection, ip } = this; - const { semaphore } = application.server; - try { - await semaphore.enter(); - } catch { - this.error(504, callId); - return; - } - try { - let session = await application.auth.restore(this); - const proc = application.runMethod(method, session); - if (!proc) { - this.error(404, callId); - return; - } - if (!session && proc.access !== 'public') { - this.error(403, callId); - return; - } - const result = await proc.method(args); - if (!session && result && result.userId && proc.access === 'public') { - session = application.auth.start(this, result.userId); - result.token = session.token; - } - const packet = { callback: callId, result }; - const data = JSON.stringify(packet); - if (connection) connection.send(data); - else res.end(data); - const token = session ? session.token : 'anonymous'; - application.logger.log(`${ip}\t${token}\t${method}`); - } catch (err) { - this.error(500, err, callId); - } finally { - semaphore.leave(); - } - } -} - -module.exports = Client; diff --git a/lib/common.js b/lib/common.js index 4eb6f45..06c9301 100644 --- a/lib/common.js +++ b/lib/common.js @@ -6,10 +6,23 @@ const parseHost = host => { return host; }; -const timeout = msec => new Promise(resolve => { - setTimeout(resolve, msec); -}); +const timeout = msec => + new Promise(resolve => { + setTimeout(resolve, msec); + }); const sample = arr => arr[Math.floor(Math.random() * arr.length)]; -module.exports = { parseHost, timeout, sample }; +const between = (s, prefix, suffix) => { + let i = s.indexOf(prefix); + if (i === -1) return ''; + s = s.substring(i + prefix.length); + if (suffix) { + i = s.indexOf(suffix); + if (i === -1) return ''; + s = s.substring(0, i); + } + return s; +}; + +module.exports = { parseHost, timeout, sample, between }; diff --git a/lib/config.js b/lib/config.js index b223b97..4ca9bd7 100644 --- a/lib/config.js +++ b/lib/config.js @@ -1,6 +1,6 @@ 'use strict'; -const { path, fsp, vm } = require('./dependencies.js'); +const { path, fsp, vm } = require('./dependencies.js').node; const SCRIPT_OPTIONS = { timeout: 5000 }; @@ -17,7 +17,7 @@ class Config { for (const fileName of files) { await this.loadFile(fileName); } - return this; + return this.sections; } async loadFile(fileName) { diff --git a/lib/dependencies.js b/lib/dependencies.js index 49363f5..c21f380 100644 --- a/lib/dependencies.js +++ b/lib/dependencies.js @@ -1,20 +1,59 @@ 'use strict'; -const api = { common: require('./common.js') }; +const node = {}; const internals = [ - 'util', 'child_process', 'worker_threads', 'os', 'v8', 'vm', 'path', 'url', - 'assert', 'querystring', 'string_decoder', 'perf_hooks', 'async_hooks', - 'timers', 'events', 'stream', 'fs', 'crypto', 'zlib', - 'dns', 'net', 'tls', 'http', 'https', 'http2', 'dgram', + 'util', + 'child_process', + 'worker_threads', + 'os', + 'v8', + 'vm', + 'path', + 'url', + 'assert', + 'querystring', + 'string_decoder', + 'perf_hooks', + 'async_hooks', + 'timers', + 'events', + 'stream', + 'fs', + 'crypto', + 'zlib', + 'readline', + 'dns', + 'net', + 'tls', + 'http', + 'https', + 'http2', + 'dgram', ]; -for (const name of internals) api[name] = Object.freeze(require(name)); -api.process = process; -api.childProcess = api['child_process']; -api.StringDecoder = api['string_decoder']; -api.perfHooks = api['perf_hooks']; -api.asyncHooks = api['async_hooks']; -api.worker = api['worker_threads']; -api.fsp = api.fs.promises; +for (const name of internals) node[name] = require(name); +node.process = process; +node.childProcess = node['child_process']; +node.StringDecoder = node['string_decoder']; +node.perfHooks = node['perf_hooks']; +node.asyncHooks = node['async_hooks']; +node.worker = node['worker_threads']; +node.fsp = node.fs.promises; +Object.freeze(node); -module.exports = Object.freeze(api); +const npm = { + common: require('./common.js'), + ws: require('ws'), +}; + +const pkgPath = node.path.join(process.cwd(), 'package.json'); +const pkg = require(pkgPath); + +if (pkg.dependencies) { + for (const dependency of Object.keys(pkg.dependencies)) { + if (dependency !== 'impress') npm[dependency] = require(dependency); + } +} +Object.freeze(npm); + +module.exports = { node, npm }; diff --git a/lib/logger.js b/lib/logger.js index 3deef7a..1684b42 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -1,11 +1,13 @@ 'use strict'; -const { fs, util, path } = require('./dependencies.js'); +const { fs, util, path } = require('./dependencies.js').node; const COLORS = { info: '\x1b[1;37m', debug: '\x1b[1;33m', error: '\x1b[0;31m', + system: '\x1b[1;34m', + access: '\x1b[1;38m', }; const DATETIME_LENGTH = 19; @@ -19,6 +21,10 @@ class Logger { this.regexp = new RegExp(path.dirname(this.path), 'g'); } + close() { + return new Promise(resolve => this.stream.end(resolve)); + } + write(level = 'info', s) { const now = new Date().toISOString(); const date = now.substring(0, DATETIME_LENGTH); @@ -48,6 +54,16 @@ class Logger { const msg = util.format(...args).replace(/[\n\r]{2,}/g, '\n'); this.write('error', msg.replace(this.regexp, '')); } + + system(...args) { + const msg = util.format(...args); + this.write('system', msg); + } + + access(...args) { + const msg = util.format(...args); + this.write('access', msg); + } } module.exports = Logger; diff --git a/lib/security.js b/lib/security.js index 96d82d2..d3b184e 100644 --- a/lib/security.js +++ b/lib/security.js @@ -1,6 +1,6 @@ 'use strict'; -const { crypto } = require('./dependencies.js'); +const { crypto } = require('./dependencies.js').node; const serializeHash = (hash, salt, params) => { const paramString = Object.entries(params) diff --git a/lib/server.js b/lib/server.js index 99a122c..8cda6c7 100644 --- a/lib/server.js +++ b/lib/server.js @@ -1,17 +1,15 @@ 'use strict'; -const { http, https, worker, common } = require('./dependencies.js'); -const application = require('./application.js'); +const { node, npm } = require('./dependencies.js'); +const { http, https, worker } = node; +const { common, ws } = npm; -const WebSocket = require('ws'); const Semaphore = require('./semaphore.js'); -const Client = require('./client.js'); const SHUTDOWN_TIMEOUT = 5000; +const SHORT_TIMEOUT = 500; const LONG_RESPONSE = 30000; -const clients = new Map(); - const receiveBody = async req => { const buffers = []; for await (const chunk of req) { @@ -20,81 +18,113 @@ const receiveBody = async req => { return Buffer.concat(buffers).toString(); }; - -const closeClients = () => { - for (const [connection, client] of clients.entries()) { - clients.delete(connection); - client.error(503); - connection.destroy(); +class Server { + constructor(config, { application, Channel }) { + this.config = config; + this.application = application; + this.Channel = Channel; + this.channels = new Map(); + const { host, balancer, protocol, ports, concurrency, queue } = config; + this.semaphore = new Semaphore(concurrency, queue.size, queue.timeout); + const { threadId } = worker; + this.balancer = balancer && threadId === 1; + const skipBalancer = balancer ? 1 : 0; + this.port = this.balancer ? balancer : ports[threadId - skipBalancer - 1]; + const transport = protocol === 'http' || this.balancer ? http : https; + const listener = this.listener.bind(this); + this.server = transport.createServer({ ...application.cert }, listener); + this.ws = new ws.Server({ server: this.server }); + this.ws.on('connection', async (connection, req) => { + const channel = await new Channel(req, null, connection, application); + connection.on('message', data => { + channel.message(data); + }); + }); + this.protocol = protocol; + this.host = host; + this.server.listen(this.port, host); } -}; - -const listener = (req, res) => { - let finished = false; - const { method, url, connection } = req; - const client = new Client(req, res); - clients.set(connection, client); - const timer = setTimeout(() => { - if (finished) return; - finished = true; - clients.delete(connection); - client.error(504); - }, LONG_RESPONSE); + async listener(req, res) { + const { channels, Channel } = this; + let finished = false; + const { url, connection } = req; + const channel = await new Channel(req, res, null, this.application); + channels.set(connection, channel); - res.on('close', () => { - if (finished) return; - finished = true; - clearTimeout(timer); - clients.delete(connection); - }); + const timer = setTimeout(() => { + if (finished) return; + finished = true; + channels.delete(connection); + channel.error(504); + }, LONG_RESPONSE); - if (url === '/api') { - if (method !== 'POST') { - client.error(403); - return; - } - receiveBody(req).then(data => { - client.message(data); - }, err => { - client.error(500, err); + res.on('close', () => { + if (finished) return; + finished = true; + clearTimeout(timer); + channels.delete(connection); }); - } else { - if (url === '/' && !req.connection.encrypted) { + + if (this.balancer) { const host = common.parseHost(req.headers.host); - const port = common.sample(application.server.ports); - client.redirect(`https://${host}:${port}/`); + const port = common.sample(this.config.ports); + const { protocol } = this.config; + channel.redirect(`${protocol}://${host}:${port}/`); + return; } - client.static(); + + if (url.startsWith('/api')) this.request(channel); + else channel.static(); } -}; -class Server { - constructor(config) { - this.config = config; - const { ports, host, concurrency, queue } = config; - this.semaphore = new Semaphore(concurrency, queue.size, queue.timeout); - const { threadId } = worker; - const port = ports[threadId - 1]; - this.ports = config.ports.slice(1); - const transport = threadId === 1 ? http : https; - this.instance = transport.createServer({ ...application.cert }, listener); - this.ws = new WebSocket.Server({ server: this.instance }); - this.ws.on('connection', (connection, req) => { - const client = new Client(req, null, connection); - connection.on('message', data => { - client.message(data); + request(channel) { + const { req } = channel; + if (req.method === 'OPTIONS') { + channel.options(); + return; + } + if (req.method !== 'POST') { + channel.error(403); + return; + } + const body = receiveBody(req); + if (req.url === '/api') { + body.then(data => { + channel.message(data); + }); + } else { + body.then(data => { + const { pathname, searchParams } = new URL('http://' + req.url); + const [, interfaceName, methodName] = pathname.split('/'); + const args = data ? JSON.parse(data) : Object.fromEntries(searchParams); + channel.rpc(-1, interfaceName, methodName, args); }); + } + body.catch(err => { + channel.error(500, err); }); - this.instance.listen(port, host); + } + + closeChannels() { + const { channels } = this; + for (const [connection, channel] of channels.entries()) { + channels.delete(connection); + channel.error(503); + connection.destroy(); + } } async close() { - this.instance.close(err => { - if (err) application.logger.error(err.stack); + this.server.close(err => { + if (err) this.application.logger.error(err.stack); }); + if (this.channels.size === 0) { + await common.timeout(SHORT_TIMEOUT); + return; + } await common.timeout(SHUTDOWN_TIMEOUT); - closeClients(); + this.closeChannels(); } } diff --git a/lib/worker.js b/lib/worker.js index 59ff23c..006243a 100644 --- a/lib/worker.js +++ b/lib/worker.js @@ -1,49 +1,64 @@ 'use strict'; -const { worker, fsp, path } = require('./dependencies.js'); +const { node } = require('./dependencies.js'); +const { worker, fsp, path } = node; const application = require('./application.js'); const Config = require('./config.js'); const Logger = require('./logger.js'); const Database = require('./database.js'); const Server = require('./server.js'); +const Channel = require('./channel.js'); const initAuth = require('./auth.js'); (async () => { const configPath = path.join(application.path, 'config'); const config = await new Config(configPath); - const logPath = path.join(application.path, 'log'); + const logPath = path.join(application.root, 'log'); + const { threadId } = worker; const logger = await new Logger(logPath, worker.threadId); - const certPath = path.join(application.path, 'cert'); Object.assign(application, { config, logger }); + + const logError = err => { + logger.error(err ? err.stack : 'No exception stack available'); + }; + + process.on('uncaughtException', logError); + process.on('warning', logError); + process.on('unhandledRejection', logError); + + const certPath = path.join(application.path, 'cert'); try { const key = await fsp.readFile(path.join(certPath, 'key.pem')); const cert = await fsp.readFile(path.join(certPath, 'cert.pem')); application.cert = { key, cert }; } catch { - if (worker.threadId === 1) logger.log('Can not load TLS certificates'); + if (threadId === 1) logger.error('Can not load TLS certificates'); } - application.db = new Database(config.sections.database); - application.server = new Server(config.sections.server); + + application.db = new Database(config.database); application.auth = initAuth(); application.sandboxInject('auth', application.auth); + + const { balancer, ports = [] } = config.server; + const servingThreads = ports.length + (balancer ? 1 : 0); + if (threadId <= servingThreads) { + const options = { application, Channel }; + application.server = new Server(config.server, options); + const { port } = application.server; + logger.system(`Listen port ${port} in worker ${threadId}`); + } + await application.init(); - logger.log(`Application started in worker ${worker.threadId}`); + logger.system(`Application started in worker ${threadId}`); worker.parentPort.on('message', async message => { if (message.name === 'stop') { if (application.finalization) return; - logger.log(`Graceful shutdown in worker ${worker.threadId}`); + logger.system(`Graceful shutdown in worker ${threadId}`); await application.shutdown(); process.exit(0); } }); - const logError = err => { - logger.error(err.stack); - }; - - process.on('uncaughtException', logError); - process.on('warning', logError); - process.on('unhandledRejection', logError); })(); diff --git a/package.json b/package.json index a4cd9f7..43434c7 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "scripts": { "test": "npm run -s lint && node test/all.js", "lint": "eslint .", - "install": "cert/generate.sh" + "install": "application/cert/generate.sh" }, "repository": { "type": "git", diff --git a/server.js b/server.js index 4c3e3da..09da7d9 100644 --- a/server.js +++ b/server.js @@ -6,20 +6,23 @@ const path = require('path'); const Config = require('./lib/config.js'); const PATH = process.cwd(); -const CFG_PATH = path.join(PATH, 'config'); +const CFG_PATH = path.join(PATH, 'application/config'); const options = { trackUnmanagedFds: true }; (async () => { const config = await new Config(CFG_PATH); - const count = config.sections.server.ports.length; - const workers = new Array(count); + const { balancer, ports = [], workers = {} } = config.server; + const count = ports.length + (balancer ? 1 : 0) + (workers.pool || 0); + let active = count; + const threads = new Array(count); const start = id => { const worker = new Worker('./lib/worker.js', options); - workers[id] = worker; + threads[id] = worker; worker.on('exit', code => { if (code !== 0) start(id); + else if (--active === 0) process.exit(0); }); }; @@ -27,7 +30,7 @@ const options = { trackUnmanagedFds: true }; const stop = async () => { console.log(); - for (const worker of workers) { + for (const worker of threads) { worker.postMessage({ name: 'stop' }); } }; diff --git a/static/metacom.js b/static/metacom.js deleted file mode 100644 index b08e63b..0000000 --- a/static/metacom.js +++ /dev/null @@ -1,68 +0,0 @@ -export class Metacom { - constructor(host) { - this.socket = new WebSocket('wss://' + host); - this.api = {}; - this.callId = 0; - this.calls = new Map(); - this.socket.addEventListener('message', ({ data }) => { - try { - const packet = JSON.parse(data); - const { callback, event } = packet; - const callId = callback || event; - const promised = this.calls.get(callId); - if (!promised) return; - const [resolve, reject] = promised; - if (packet.error) { - const { code, message } = packet.error; - const error = new Error(message); - error.code = code; - reject(error); - return; - } - resolve(packet.result); - } catch (err) { - console.error(err); - } - }); - } - - ready() { - return new Promise(resolve => { - if (this.socket.readyState === WebSocket.OPEN) resolve(); - else this.socket.addEventListener('open', resolve); - }); - } - - async load(...methods) { - for (const methodName of methods) { - this.api[methodName] = this.socketCall(methodName); - } - } - - httpCall(methodName) { - return (args = {}) => { - const callId = ++this.callId; - const packet = { call: callId, [methodName]: args }; - return fetch('/api', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(packet), - }).then(res => { - if (res.status === 200) return res.json().then(({ result }) => result); - throw new Error(`Status Code: ${res.status}`); - }); - }; - } - - socketCall(methodName) { - return async (args = {}) => { - const callId = ++this.callId; - await this.ready(); - return new Promise((resolve, reject) => { - this.calls.set(callId, [resolve, reject]); - const packet = { call: callId, [methodName]: args }; - this.socket.send(JSON.stringify(packet)); - }); - }; - } -} diff --git a/test/system.js b/test/system.js index dca0b1b..b0dfdac 100644 --- a/test/system.js +++ b/test/system.js @@ -1,33 +1,30 @@ 'use strict'; +require('../server.js'); + const http = require('http'); const assert = require('assert').strict; -const { Worker } = require('worker_threads'); - -const worker = new Worker('./lib/worker.js'); const HOST = '127.0.0.1'; -const PORT = 8000; -const START_TIMEOUT = 1000; +const PORT = 8001; +const START_DELAY = 2000; +const TEST_DELAY = 100; const TEST_TIMEOUT = 3000; let callId = 0; console.log('System test started'); setTimeout(async () => { - worker.postMessage({ name: 'stop' }); + console.log('System test finished'); + process.exit(0); }, TEST_TIMEOUT); -worker.on('exit', code => { - console.log(`System test finished with code ${code}`); -}); - const tasks = [ - { get: '/', status: 302 }, + { get: '/', port: 8000, status: 302 }, { get: '/console.js' }, { post: '/api', - method: 'signIn', + method: 'auth/signIn', args: { login: 'marcus', password: 'marcus' } } ]; @@ -35,7 +32,7 @@ const tasks = [ const getRequest = task => { const request = { host: HOST, - port: PORT, + port: task.port || PORT, agent: false }; if (task.get) { @@ -66,7 +63,7 @@ setTimeout(() => { const expectedStatus = task.status || 200; setTimeout(() => { assert.equal(res.statusCode, expectedStatus); - }, TEST_TIMEOUT); + }, TEST_DELAY); }); req.on('error', err => { console.log(err.stack); @@ -74,4 +71,4 @@ setTimeout(() => { if (task.data) req.write(task.data); req.end(); }); -}, START_TIMEOUT); +}, START_DELAY); diff --git a/test/unit.config.js b/test/unit.config.js index 698f52d..563e061 100644 --- a/test/unit.config.js +++ b/test/unit.config.js @@ -8,11 +8,11 @@ const Config = require('../lib/config.js'); assert(Config); const PATH = process.cwd(); -const configPath = path.join(PATH, 'config'); +const configPath = path.join(PATH, 'application/config'); (async () => { const config = await new Config(configPath); - assert(config.sections); - assert(config.sections.database); - assert(config.sections.server); + assert(config); + assert(config.database); + assert(config.server); })(); diff --git a/test/unit.database.js b/test/unit.database.js index 40f2e37..cb34bab 100644 --- a/test/unit.database.js +++ b/test/unit.database.js @@ -15,9 +15,9 @@ assert(Config); const PATH = process.cwd(); (async () => { - const configPath = path.join(PATH, 'config'); + const configPath = path.join(PATH, 'application/config'); const config = await new Config(configPath); - const databaseConfig = config.sections.database; + const databaseConfig = config.database; const database = new Database(databaseConfig); const empty = 'empty';