From 9d99ba59056e7ce1109c2b1f8d013dabdb8b49b1 Mon Sep 17 00:00:00 2001 From: techcoderx Date: Sat, 15 Jul 2023 19:29:03 +0800 Subject: [PATCH 1/9] replace vm2 with isolate-vm --- package.json | 2 +- src/services/chainBridge.ts | 1 - src/services/contractEngine.ts | 68 +++++++++++++++------------------ src/services/transactionPool.ts | 1 - 4 files changed, 32 insertions(+), 40 deletions(-) diff --git a/package.json b/package.json index 77bca74..ae23cff 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "hamt-sharding": "^3.0.2", "hive-tx": "^4.3.0", "ipfs-http-client": "^60.0.0", + "isolated-vm": "^4.5.0", "it-pushable": "^1.4.2", "k-bucket": "^5.1.0", "key-did-provider-ed25519": "^1.1.0", @@ -37,7 +38,6 @@ "peer-id": "^0.16.0", "shuffle-seed": "^1.1.6", "uuid": "^8.3.2", - "vm2": "^3.9.19", "winston": "^3.8.2", "xor-distance": "^2.0.0" }, diff --git a/src/services/chainBridge.ts b/src/services/chainBridge.ts index 6f63cac..af27d0e 100644 --- a/src/services/chainBridge.ts +++ b/src/services/chainBridge.ts @@ -8,7 +8,6 @@ import { Collection } from 'mongodb' import networks from './networks' import { WitnessService } from './witness' import type PQueue from 'p-queue' -import { VMScript } from 'vm2' import * as vm from 'vm'; import Pushable from 'it-pushable' import { CommitmentStatus, Contract, ContractCommitment } from '../types/contracts' diff --git a/src/services/contractEngine.ts b/src/services/contractEngine.ts index 41bfc65..402ff0d 100644 --- a/src/services/contractEngine.ts +++ b/src/services/contractEngine.ts @@ -1,5 +1,5 @@ import { Collection } from 'mongodb' -import { NodeVM, VM, VMScript } from 'vm2' +import { Isolate } from 'isolated-vm' import { CID } from 'multiformats' import jsonpatch from 'fast-json-patch' import SHA256 from 'crypto-js/sha256' @@ -297,6 +297,7 @@ export class ContractEngine { } } + /* async executeContract(id, call, args) { const contractInfo = await this.contractDb.findOne({ id, @@ -327,6 +328,7 @@ export class ContractEngine { }) await vm.run(script, 'vm.js') } + */ /** * Executes a list of operations @@ -356,8 +358,6 @@ export class ContractEngine { } let code = codeTemplate.replace('###ACTIONS###', codeRaw) - - const script = new VMScript(code).compile() let stateMerkle let startMerkle @@ -376,41 +376,35 @@ export class ContractEngine { if (!startMerkle) { startMerkle = state.startMerkle.toString() } - - const executeOutput = (await new Promise((resolve, reject) => { - const vm = new NodeVM({ - sandbox: { - Date: MockDate, - utils: { - SHA256: (payloadToHash) => { - if (typeof payloadToHash === 'string') { - return SHA256(payloadToHash).toString(enchex); - } - - return SHA256(JSON.stringify(payloadToHash)).toString(enchex); - }, - }, - api: { - action: opData.action, - payload: opData.payload, - input: { - sender: { - type: "DID", - id: op.account_auth - }, - tx_id: op.id, - included_in: op.included_in - } - }, - done: () => { - return resolve(state.finish()) - }, - // console: "redirect", - state: state.client, + + const isolate = new Isolate({ memoryLimit: 128 }) // fixed 128MB memory limit for now, maybe should be part of tx fee calculation + const context = await isolate.createContext() + context.global.setSync('Date', MockDate) + context.global.setSync('utils', { + SHA256: (payloadToHash) => { + if (typeof payloadToHash === 'string') { + return SHA256(payloadToHash).toString(enchex); + } + + return SHA256(JSON.stringify(payloadToHash)).toString(enchex); + }, + }) + context.global.setSync('api', { + action: opData.action, + payload: opData.payload, + input: { + sender: { + type: "DID", + id: op.account_auth }, - }) - vm.run(script, 'vm.js') - })) as { stateMerkle: string } + tx_id: op.id, + included_in: op.included_in + } + }) + context.global.setSync('state', state.client) + context.global.setSync('done', state.finish) + const compiled = await isolate.compileScript(code) + const executeOutput = await compiled.run(context) as { stateMerkle: string } stateMerkle = executeOutput.stateMerkle } diff --git a/src/services/transactionPool.ts b/src/services/transactionPool.ts index fc13bd2..e3bf79f 100644 --- a/src/services/transactionPool.ts +++ b/src/services/transactionPool.ts @@ -8,7 +8,6 @@ import { sha256 as hasher } from 'multiformats/hashes/sha2' import BloomFilters from 'bloom-filters' import { CoreService } from '.' import { BlockHeader, TransactionContainer, TransactionDbRecord, TransactionDbStatus, TransactionDbType, TransactionRaw } from '../types' -import { VM, NodeVM, VMScript } from 'vm2' import fs from 'fs/promises' import { isNamedType } from 'graphql/type/definition.js' import * as vm from 'vm'; From 0fcbf049c6049c09c20e7ba1818f4a04b0eeacc7 Mon Sep 17 00:00:00 2001 From: techcoderx Date: Sat, 15 Jul 2023 19:30:28 +0800 Subject: [PATCH 2/9] fix import --- src/services/contractEngine.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/contractEngine.ts b/src/services/contractEngine.ts index 402ff0d..f35f336 100644 --- a/src/services/contractEngine.ts +++ b/src/services/contractEngine.ts @@ -1,5 +1,5 @@ import { Collection } from 'mongodb' -import { Isolate } from 'isolated-vm' +import ivm from 'isolated-vm' import { CID } from 'multiformats' import jsonpatch from 'fast-json-patch' import SHA256 from 'crypto-js/sha256' @@ -377,7 +377,7 @@ export class ContractEngine { startMerkle = state.startMerkle.toString() } - const isolate = new Isolate({ memoryLimit: 128 }) // fixed 128MB memory limit for now, maybe should be part of tx fee calculation + const isolate = new ivm.Isolate({ memoryLimit: 128 }) // fixed 128MB memory limit for now, maybe should be part of tx fee calculation const context = await isolate.createContext() context.global.setSync('Date', MockDate) context.global.setSync('utils', { From cc01de117e146e2165b08052374e1c30462a87a6 Mon Sep 17 00:00:00 2001 From: techcoderx Date: Sat, 15 Jul 2023 22:15:16 +0800 Subject: [PATCH 3/9] deref global into isolate --- src/services/contractEngine.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/services/contractEngine.ts b/src/services/contractEngine.ts index f35f336..75a08d2 100644 --- a/src/services/contractEngine.ts +++ b/src/services/contractEngine.ts @@ -379,6 +379,7 @@ export class ContractEngine { const isolate = new ivm.Isolate({ memoryLimit: 128 }) // fixed 128MB memory limit for now, maybe should be part of tx fee calculation const context = await isolate.createContext() + context.global.setSync('global', context.global.derefInto()) context.global.setSync('Date', MockDate) context.global.setSync('utils', { SHA256: (payloadToHash) => { From 8c701d5483205f95f4798a1352eef2200dac90d4 Mon Sep 17 00:00:00 2001 From: techcoderx Date: Thu, 5 Oct 2023 10:42:48 +0800 Subject: [PATCH 4/9] update isolated-vm --- package.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index d87052a..cd54abe 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "graphql-yoga": "^3.6.0", "hamt-sharding": "^3.0.2", "hive-tx": "^4.3.0", - "isolated-vm": "^4.5.0", + "isolated-vm": "^4.6.0", "it-pushable": "^1.4.2", "k-bucket": "^5.1.0", "key-did-provider-ed25519": "^1.1.0", @@ -74,10 +74,10 @@ "build:clear": "rimraf & tsc", "contracts:deploy": "node --experimental-specifier-resolution=node --max-old-space-size=3072 --loader ts-node/esm src/transactions/scriptDeployContract.ts", "contracts:compile": "cd contracts && live-tsc --src . --dest ./ -p ../live-tsc.json", - "ts-node": "node --experimental-specifier-resolution=node --max-old-space-size=3072 --loader ts-node/esm", - "start": "node --experimental-specifier-resolution=node --max-old-space-size=3072 dist/index.js", - "start:benchmark": "nodemon --exec \"node --experimental-specifier-resolution=node --max-old-space-size=3072 --loader ts-node/esm\" src/benchmarkVm.ts", - "dev": "nodemon --exec \"node --experimental-specifier-resolution=node --max-old-space-size=3072 --loader ts-node/esm\" src/index.ts", + "ts-node": "node --experimental-specifier-resolution=node --max-old-space-size=3072 --no-node-snapshot --loader ts-node/esm", + "start": "node --experimental-specifier-resolution=node --max-old-space-size=3072 --no-node-snapshot dist/index.js", + "start:benchmark": "nodemon --exec \"node --experimental-specifier-resolution=node --max-old-space-size=3072 --no-node-snapshot --loader ts-node/esm\" src/benchmarkVm.ts", + "dev": "nodemon --exec \"node --experimental-specifier-resolution=node --max-old-space-size=3072 --no-node-snapshot --loader ts-node/esm\" src/index.ts", "lint:fix": "eslint --fix \"src/**/*.{ts,tsx}\"", "lint": "eslint \"src/**/*.{ts,tsx}\"", "test:report": "sensible-browser ./coverage/lcov-report/index.html", From f0bcbdb7075ebc56643d04bfd5067a8ae9570279 Mon Sep 17 00:00:00 2001 From: techcoderx Date: Thu, 5 Oct 2023 15:28:10 +0800 Subject: [PATCH 5/9] move MockDate and OutputActions into isolate --- src/services/contractEngine.ts | 102 +++++++++++++++++---------------- 1 file changed, 54 insertions(+), 48 deletions(-) diff --git a/src/services/contractEngine.ts b/src/services/contractEngine.ts index 519564c..1f91997 100644 --- a/src/services/contractEngine.ts +++ b/src/services/contractEngine.ts @@ -28,71 +28,79 @@ export class OutputActions { return this.opStack.push(input) } } -class MockDate extends Date { - - constructor(val) { +let codeTemplate = ` +function wrapper () { + RegExp.prototype.constructor = function () { }; + RegExp.prototype.exec = function () { }; + RegExp.prototype.test = function () { }; + Math.random = function () { }; + class MockDate extends Date { + constructor(val) { if(val) { - if(typeof val === 'string') { - if(val.endsWith('Z')) { - super(val) - } else { - super(val + "Z") - } + if(typeof val === 'string') { + if(val.endsWith('Z')) { + super(val) } else { - super(val) + super(val + "Z") } + } else { + super(val) + } } else { - super(0); + super(0); } - // console.log(this) - // this = new Date(0) - } + } - getTimezoneOffset() { + getTimezoneOffset() { return 0; - } + } - toLocaleString() { + toLocaleString() { return this.toUTCString() - } + } - static now() { + static now() { return 0; + } + } + class OutputActions { + constructor() { + this.opStack = [] + } + + addHiveOp(input) { + return this.opStack.push(input) + } } -} -let codeTemplate = ` -function wrapper () { - RegExp.prototype.constructor = function () { }; - RegExp.prototype.exec = function () { }; - RegExp.prototype.test = function () { }; - Math.random = function () { }; - let actions = {}; + Date = MockDate; - ###ACTIONS### + let actions = {}; - const execute = async function () { - try { - if (api.action && typeof api.action === 'string' && typeof actions[api.action] === 'function') { - if (api.action !== 'init') { - actions.init = null; - } - await actions[api.action](api.payload); - if(api.payload) { - done(api.payload) - } - done(null); - } else { - done('invalid action'); + ###ACTIONS### + + const execute = async function () { + try { + if (api.action && typeof api.action === 'string' && typeof actions[api.action] === 'function') { + if (api.action !== 'init') { + actions.init = null; } - } catch (error) { - done(error); + await actions[api.action](api.payload); + if(api.payload) { + done(api.payload) + } + done(null); + } else { + done('invalid action'); } + } catch (error) { + done(error); } - - execute(); } - wrapper(); + + execute(); +} +wrapper(); ` export class ContractEngine { @@ -433,8 +441,6 @@ export class ContractEngine { const isolate = new ivm.Isolate({ memoryLimit: 128 }) // fixed 128MB memory limit for now, maybe should be part of tx fee calculation const context = await isolate.createContext() context.global.setSync('global', context.global.derefInto()) - context.global.setSync('Date', MockDate) - context.global.setSync('OutputActions', OutputActions) context.global.setSync('utils', { SHA256: (payloadToHash) => { if (typeof payloadToHash === 'string') { From 22c77ded85c4dd4a3ece4c66f6e7ec81a9366953 Mon Sep 17 00:00:00 2001 From: techcoderx Date: Thu, 5 Oct 2023 18:38:57 +0800 Subject: [PATCH 6/9] use ivm reference this is required for async functions to work --- src/services/contractEngine.ts | 153 ++++++++++++++++++++++++++------- 1 file changed, 124 insertions(+), 29 deletions(-) diff --git a/src/services/contractEngine.ts b/src/services/contractEngine.ts index 1f91997..d989085 100644 --- a/src/services/contractEngine.ts +++ b/src/services/contractEngine.ts @@ -74,6 +74,67 @@ function wrapper () { } Date = MockDate; + let utils = { + SHA256: sha256, + bitcoin: { + ValidateSPV: { + validateHeaderChain: btc_validate_spv_header_chain, + validateProof: btc_validate_spv_proof + } + ser: { + deserializeSPVProof: btc_ser_deserialize_spv_proof + }, + parseTxHex: btc_parse_tx_hex, + reverseBytes: btc_reverse_bytes, + BTCUtils: { + extractPrevBlockLE: btc_utils_extract_prev_block_le + extractTimestamp: btc_utils_extract_ts + extractTimestampLE: btc_utils_extract_ts_le + extractMerkleRootLE: btc_utils_extract_merkleroot_le + hash256: btc_utils_hash256 + extractOutputAtIndex: btc_utils_extract_output_at_idx + extractValue: btc_utils_extract_value + }, + SPVUtils: { + deserializeHex: btc_spv_utils_deserialize_hex + } + } + } + let output = { + setChainActions: set_chain_actions + } + api = api.copy(); + api.transferFunds = async (to, amount) => { + return await (await transfer_funds.applySyncPromise(undefined, [to, amount])).copy() + } + api.withdrawFunds = async (amount) => { + return await (await withdraw_funds.applySyncPromise(undefined, [amount])).copy() + } + let state = { + remoteState: async (id) => { + let result = await (await state_remote.applySyncPromise(undefined, [id])).copy() + return { + pull: async (key) => { + return await (await result.pull.applySyncPromise(undefined, [key])).copy() + }, + ls: async (key) => { + return await (await result.ls.applySyncPromise(undefined, [key])).copy() + } + } + }, + pull: async (key) => { + return await (await state_pull.applySyncPromise(undefined, [key])).copy() + }, + update: async (key, value) => { + await state_update.applySyncPromise(undefined, [key, value]) + }, + ls: async (key) => { + return await (await state_ls.applySyncPromise(undefined, [key])).copy() + }, + del: async (key) => { + await state_del.applySyncPromise(undefined, [key]) + } + } let actions = {}; @@ -441,29 +502,34 @@ export class ContractEngine { const isolate = new ivm.Isolate({ memoryLimit: 128 }) // fixed 128MB memory limit for now, maybe should be part of tx fee calculation const context = await isolate.createContext() context.global.setSync('global', context.global.derefInto()) - context.global.setSync('utils', { - SHA256: (payloadToHash) => { - if (typeof payloadToHash === 'string') { - return SHA256(payloadToHash).toString(enchex); - } - - return SHA256(JSON.stringify(payloadToHash)).toString(enchex); - }, - bitcoin: { - ValidateSPV, - ser: ser, - parseTxHex: parseTxHex, - reverseBytes: reverse, - BTCUtils, - SPVUtils: utils + context.global.setSync('sha256', (payloadToHash: string | object) => { + if (typeof payloadToHash === 'string') { + return SHA256(payloadToHash).toString(enchex); } + + return SHA256(JSON.stringify(payloadToHash)).toString(enchex); }) - context.global.setSync('output', { - setChainActions: (actions) => { - chainActions = (actions.opStack as Array).map(e => ({tx: e})) - } + + // btc functions + context.global.setSync('btc_validate_spv_header_chain', ValidateSPV.validateHeaderChain) + context.global.setSync('btc_validate_spv_proof', ValidateSPV.validateProof) + context.global.setSync('btc_ser_deserialize_spv_proof', ser.deserializeSPVProof) + context.global.setSync('btc_parse_tx_hex', parseTxHex) + context.global.setSync('btc_reverse_bytes', reverse) + context.global.setSync('btc_utils_extract_prev_block_le', BTCUtils.extractPrevBlockLE) + context.global.setSync('btc_utils_extract_ts', BTCUtils.extractTimestamp) + context.global.setSync('btc_utils_extract_ts_le', BTCUtils.extractTimestampLE) + context.global.setSync('btc_utils_extract_merkleroot_le', BTCUtils.extractMerkleRootLE) + context.global.setSync('btc_utils_hash256', BTCUtils.hash256) + context.global.setSync('btc_utils_extract_output_at_idx', BTCUtils.extractOutputAtIndex) + context.global.setSync('btc_utils_extract_value', BTCUtils.extractValue) + context.global.setSync('btc_spv_utils_deserialize_hex', utils.deserializeHex) + + // apis + context.global.setSync('set_chain_actions', (actions: OutputActions) => { + chainActions = (actions.opStack as Array).map(e => ({tx: e})) }) - context.global.setSync('api', { + context.global.setSync('api', new ivm.ExternalCopy({ action: opData.action, payload: opData.payload, input: { @@ -475,16 +541,45 @@ export class ContractEngine { included_in: op.included_in, included_block: includedRecord.hive_ref_block, included_date: includedRecord.hive_ref_date - }, - transferFunds: this.transferFunds, - withdrawFunds: this.withdrawFunds, - getBalance: (accountId: string) => { - return this.self.chainBridge.calculateBalanceSum(accountId, { - // pla: TODO, NEED TO SUPPLY CURRENT BLOCK INFORMATION IN ORDER TO CALC THE BALANCE - } as BlockRef, id) } - }) - context.global.setSync('state', state.client) + })) + context.global.setSync('transfer_funds', new ivm.Reference(async (to: DID, amount: number) => { + let result = await this.transferFunds(to, amount) + return new ivm.ExternalCopy(result) + })) + context.global.setSync('withdraw_funds', new ivm.Reference(async (amount: number) => { + let result = await this.withdrawFunds(amount) + return new ivm.ExternalCopy(result) + })) + context.global.setSync('get_balance', new ivm.Reference(async (accountId: string) => { + let result = await this.self.chainBridge.calculateBalanceSum(accountId, { + // pla: TODO, NEED TO SUPPLY CURRENT BLOCK INFORMATION IN ORDER TO CALC THE BALANCE + } as BlockRef, id) + return new ivm.ExternalCopy(result) + })) + + // state + context.global.setSync('state_remote', new ivm.Reference(async (id: string) => { + let result = await state.client.remoteState(id) + return new ivm.ExternalCopy({ + pull: new ivm.Reference(async (key: string) => new ivm.ExternalCopy(await result.pull(key))), + ls: new ivm.Reference(async (key: string) => new ivm.ExternalCopy(await result.ls(key))), + }) + })) + context.global.setSync('state_pull', new ivm.Reference(async (key: string) => { + let result = await state.client.pull(key) + return new ivm.ExternalCopy(result) + })) + context.global.setSync('state_update', new ivm.Reference(async (key: string, value: any) => { + await state.client.update(key, value) + })) + context.global.setSync('state_ls', new ivm.Reference(async (key: string) => { + let result = await state.client.ls(key) + return new ivm.ExternalCopy(result) + })) + context.global.setSync('state_del', new ivm.Reference(async (key: string) => { + await state.client.del(key) + })) context.global.setSync('done', state.finish) const compiled = await isolate.compileScript(code) const executeOutput = await compiled.run(context) as { stateMerkle: string } From 7338cf588cf922c5e24c2bbf790127cadc15342e Mon Sep 17 00:00:00 2001 From: techcoderx Date: Thu, 5 Oct 2023 19:28:59 +0800 Subject: [PATCH 7/9] assign new stateMerkle immediately on done() --- src/services/contractEngine.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/services/contractEngine.ts b/src/services/contractEngine.ts index d989085..a60ff29 100644 --- a/src/services/contractEngine.ts +++ b/src/services/contractEngine.ts @@ -580,10 +580,11 @@ export class ContractEngine { context.global.setSync('state_del', new ivm.Reference(async (key: string) => { await state.client.del(key) })) - context.global.setSync('done', state.finish) + context.global.setSync('done', () => { + stateMerkle = state.finish().stateMerkle + }) const compiled = await isolate.compileScript(code) - const executeOutput = await compiled.run(context) as { stateMerkle: string } - stateMerkle = executeOutput.stateMerkle + await compiled.run(context) } this.self.logger.info('new state merkle of executed contract', stateMerkle) From 5952d60710618b464060d795789141da34b61503 Mon Sep 17 00:00:00 2001 From: techcoderx Date: Wed, 18 Oct 2023 10:08:47 +0800 Subject: [PATCH 8/9] fix missing commas --- src/services/contractEngine.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/services/contractEngine.ts b/src/services/contractEngine.ts index a60ff29..6b205ec 100644 --- a/src/services/contractEngine.ts +++ b/src/services/contractEngine.ts @@ -80,19 +80,19 @@ function wrapper () { ValidateSPV: { validateHeaderChain: btc_validate_spv_header_chain, validateProof: btc_validate_spv_proof - } + }, ser: { deserializeSPVProof: btc_ser_deserialize_spv_proof }, parseTxHex: btc_parse_tx_hex, reverseBytes: btc_reverse_bytes, BTCUtils: { - extractPrevBlockLE: btc_utils_extract_prev_block_le - extractTimestamp: btc_utils_extract_ts - extractTimestampLE: btc_utils_extract_ts_le - extractMerkleRootLE: btc_utils_extract_merkleroot_le - hash256: btc_utils_hash256 - extractOutputAtIndex: btc_utils_extract_output_at_idx + extractPrevBlockLE: btc_utils_extract_prev_block_le, + extractTimestamp: btc_utils_extract_ts, + extractTimestampLE: btc_utils_extract_ts_le, + extractMerkleRootLE: btc_utils_extract_merkleroot_le, + hash256: btc_utils_hash256, + extractOutputAtIndex: btc_utils_extract_output_at_idx, extractValue: btc_utils_extract_value }, SPVUtils: { From 25f928b9b88090c4d8ef2d29b6ed9ae03920668a Mon Sep 17 00:00:00 2001 From: techcoderx Date: Wed, 18 Oct 2023 10:59:55 +0800 Subject: [PATCH 9/9] create ExternalCopy for objects passed in --- src/services/contractEngine.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/services/contractEngine.ts b/src/services/contractEngine.ts index 6b205ec..163ac00 100644 --- a/src/services/contractEngine.ts +++ b/src/services/contractEngine.ts @@ -126,6 +126,8 @@ function wrapper () { return await (await state_pull.applySyncPromise(undefined, [key])).copy() }, update: async (key, value) => { + if (typeof value === 'object') + value = make_external_copy(value).copyInto() await state_update.applySyncPromise(undefined, [key, value]) }, ls: async (key) => { @@ -502,6 +504,13 @@ export class ContractEngine { const isolate = new ivm.Isolate({ memoryLimit: 128 }) // fixed 128MB memory limit for now, maybe should be part of tx fee calculation const context = await isolate.createContext() context.global.setSync('global', context.global.derefInto()) + + // Expose a new function to the isolate which will create ExternalCopy of the passed argument + context.global.setSync('make_external_copy', function(arg: any) { + return new ivm.ExternalCopy(arg) + }) + + // sha256 context.global.setSync('sha256', (payloadToHash: string | object) => { if (typeof payloadToHash === 'string') { return SHA256(payloadToHash).toString(enchex);