From 16556357bdaf944f1315a05a7be8b92e7d64d4f5 Mon Sep 17 00:00:00 2001 From: Joseph Reeve Date: Tue, 21 Aug 2018 18:47:13 +0100 Subject: [PATCH] Added AudioWorklet with WASM support --- dist/app.js | 179 ++++++++- dist/style.css | 9 + files/.gitignore | 2 + index.html | 1 + package-lock.json | 536 ++++++++++++++++++++++++++- package.json | 7 +- server.js | 53 +++ src/nodes/modifiers/customWorklet.js | 165 +++++++++ src/nodes/modifiers/index.js | 3 +- src/styles.less | 12 +- 10 files changed, 957 insertions(+), 10 deletions(-) create mode 100644 files/.gitignore create mode 100644 server.js create mode 100644 src/nodes/modifiers/customWorklet.js diff --git a/dist/app.js b/dist/app.js index 5ea54a5..88e2d34 100644 --- a/dist/app.js +++ b/dist/app.js @@ -246,7 +246,7 @@ module.exports = { output:require('./outputs/index'), modifier:require('./modifiers/index') } -},{"./inputs/index":5,"./modifiers/index":12,"./outputs/index":13}],4:[function(require,module,exports){ +},{"./inputs/index":5,"./modifiers/index":13,"./outputs/index":14}],4:[function(require,module,exports){ module.exports = { default () { return { @@ -687,6 +687,172 @@ ${nodeName}.type.value = "${node.options.type}"; } } },{}],10:[function(require,module,exports){ +function intArrayFromBase64(s) { + var decoded = atob(s); + var bytes = new Uint8Array(decoded.length); + for (var i = 0; i < decoded.length; ++i) { + bytes[i] = decoded.charCodeAt(i); + } + return bytes; +} + +//export function processor(input_ptr: i32, output_ptr: i32, length: i32): void { +const gainTsSource = ` +let gain = 0.5; +for (let byte = 0; byte < length; ++byte) { + let inp = load(input_ptr + 4 * byte); + store(output_ptr + 4 * byte, inp * gain); +} +`; +//} + + + +//Yuk Yuk Yuk +function makeProcessorsFile(id, parsedBinary) { + return ` +const wasmBinary = new Uint8Array([${parsedBinary}]); + +const memory = new WebAssembly.Memory({ + initial: 10 +}); +const module = new WebAssembly.Module(wasmBinary); +const instance = new WebAssembly.Instance(module, { + env: { + abort(msg, file, line, column) { + console.error("abort called at main.ts:" + line + ":" + column); + }, + memory + } +}); + +const result = { + instance +}; + +const exp = result.instance.exports; + +class MyWorkletProcessor extends AudioWorkletProcessor { + constructor() { + super(); + } + + process(inputs, outputs, parameters) { + let input = inputs[0]; + let output = outputs[0]; + let channelCount = input.length; + for (let channel = 0; channel < channelCount; ++channel) { + let arr = new Float32Array(memory.buffer); + for (let i = 0; i < input[channel].length; i++) { + arr[i] = input[channel][i]; + } + // console.log(arr.slice(0,3)); + exp.processor(0, input[channel].length * Float32Array.BYTES_PER_ELEMENT, input[channel].length); + + for (let i = 0; i < input[channel].length; i++) { + output[channel][i] = arr[128 + i]; + } + } + + return true; + + } +} + +registerProcessor('${id}', MyWorkletProcessor); +`; +} + + +let latestParsedBin = null; + + +function makeWorkletPromise(audioCtx, ts) { + const id = '' + Date.now(); //TODO: better ID + return $.ajax(`/makeWorkletBinary/${id}`, { + data: JSON.stringify({ + ts + }), + contentType: 'application/JSON', + type: 'POST' + }).then(binary => { + const parsedBin = intArrayFromBase64(binary); + const fileContent = makeProcessorsFile(id, parsedBin); + latestParsedBin = parsedBin; //More bad things + return audioCtx.audioWorklet.addModule(`data:application/javascript;base64,${btoa(fileContent)}`).then(() => { + return new AudioWorkletNode(audioCtx, id); + }); + }); +} + +let prevTsSrc = null; + +module.exports = { + default () { + return { + kind: 'modifier', + type: 'customWorklet', + options: { + tsSource: gainTsSource + } + } + }, + initWANode(audioCtx, node) { + prevTsSrc = node.options.tsSource; + return makeWorkletPromise(audioCtx, node.options.tsSource); + }, + updateWANode(customNode, node, nodeIndex, graph) { + if (prevTsSrc !== node.options.tsSource) { + return makeWorkletPromise(customNode.context, node.options.tsSource) + .then(workletNode => { + const prevNode = graph.nodes[nodeIndex - 1]; + const thisNode = graph.nodes[nodeIndex]; + const nextNode = graph.nodes[nodeIndex + 1]; + + prevNode.waNode.connect(workletNode); + workletNode.connect(nextNode.waNode); + + try { + prevNode.waNode.disconnect(thisNode.waNode); + thisNode.waNode.disconnect(nexNode.waNode); + } catch (err) {} + + node.waNode = workletNode; + }); + } else { + return Promise.resolve(); + } + }, + renderView(state, affect, node, nodeIndex) { + return [ + h('h3', `Custom`), + ] + }, + renderDetail(state, affect, node, nodeIndex) { + return [ + h('textarea.ts-editor', { + value: node.options.tsSource, + onblur(ev) { + affect.set(`graph.nodes.${nodeIndex}.options.tsSource`, ev.target.value); + } + }), + h('button', 'Save') + ]; + }, + generateCode(nodeName, node) { + if (latestParsedBin) { + const fileContent = makeProcessorsFile(nodeName, latestParsedBin); + return ` +const ${nodeName} = await audioCtx.audioWorklet.addModule(\`data:application/javascript;base64,${btoa(fileContent)}\`).then(() => { + return new AudioWorkletNode(audioCtx, '${nodeName}'); +}); + `; + } else { + return ''; + } + } +} +},{}],11:[function(require,module,exports){ module.exports = { default () { return { @@ -738,7 +904,7 @@ ${nodeName}.delayTime.value = ${node.options.value}; `; } } -},{}],11:[function(require,module,exports){ +},{}],12:[function(require,module,exports){ module.exports = { default () { return { @@ -790,18 +956,19 @@ ${nodeName}.gain.value = ${node.options.value}; `; } } -},{}],12:[function(require,module,exports){ +},{}],13:[function(require,module,exports){ module.exports = { delay: require('./delay'), gain: require('./gain'), biquadFilter: require('./biquadFilter'), - analyser: require('./analyser') + analyser: require('./analyser'), + customWorklet: require('./customWorklet'), }; -},{"./analyser":8,"./biquadFilter":9,"./delay":10,"./gain":11}],13:[function(require,module,exports){ +},{"./analyser":8,"./biquadFilter":9,"./customWorklet":10,"./delay":11,"./gain":12}],14:[function(require,module,exports){ module.exports = { speaker: require('./speaker') }; -},{"./speaker":14}],14:[function(require,module,exports){ +},{"./speaker":15}],15:[function(require,module,exports){ module.exports = { initWANode(audioCtx, node) { return Promise.resolve(audioCtx.destination); diff --git a/dist/style.css b/dist/style.css index e792b99..4fce25c 100644 --- a/dist/style.css +++ b/dist/style.css @@ -99,3 +99,12 @@ input { background: #EB5757; font-size: 13; } +.ts-editor { + padding: 10px; + color: white; + background: #333; + width: 100%; + min-height: 200px; + font-family: monospace; + border: none; +} diff --git a/files/.gitignore b/files/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/files/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/index.html b/index.html index cc7e918..5e0f93b 100644 --- a/index.html +++ b/index.html @@ -4,6 +4,7 @@ + diff --git a/package-lock.json b/package-lock.json index 48e341a..b18bb7d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,3 +1,537 @@ { - "lockfileVersion": 1 + "name": "webaudio-talk-demo", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=" + }, + "accepts": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz", + "integrity": "sha1-63d99gEXI6OxTopywIBcjoZ0a9I=", + "requires": { + "mime-types": "~2.1.18", + "negotiator": "0.6.1" + } + }, + "array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + }, + "assemblyscript": { + "version": "github:AssemblyScript/assemblyscript#c769f65bacd5392e5c9efab112b33244fc0a0c8f", + "from": "github:AssemblyScript/assemblyscript", + "requires": { + "@protobufjs/utf8": "^1.1.0", + "binaryen": "49.0.0-nightly.20180731", + "glob": "^7.1.2", + "long": "^4.0.0" + } + }, + "balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + }, + "binaryen": { + "version": "49.0.0-nightly.20180731", + "resolved": "https://registry.npmjs.org/binaryen/-/binaryen-49.0.0-nightly.20180731.tgz", + "integrity": "sha512-uZ7bizGTMbEOzIwZmGbXIcGFy8IkZjDoNOy+nPnIv7Dy1MiURItE0PRMnpXO2GPOOq6ZALW8pb5xb9MShD0zQQ==" + }, + "body-parser": { + "version": "1.18.3", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz", + "integrity": "sha1-WykhmP/dVTs6DyDe0FkrlWlVyLQ=", + "requires": { + "bytes": "3.0.0", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "~1.6.3", + "iconv-lite": "0.4.23", + "on-finished": "~2.3.0", + "qs": "6.5.2", + "raw-body": "2.3.3", + "type-is": "~1.6.16" + } + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "bytes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", + "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + }, + "content-disposition": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", + "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ=" + }, + "content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + }, + "cookie": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", + "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s=" + }, + "cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + }, + "destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + }, + "encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + }, + "escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + }, + "etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + }, + "express": { + "version": "4.16.3", + "resolved": "https://registry.npmjs.org/express/-/express-4.16.3.tgz", + "integrity": "sha1-avilAjUNsyRuzEvs9rWjTSL37VM=", + "requires": { + "accepts": "~1.3.5", + "array-flatten": "1.1.1", + "body-parser": "1.18.2", + "content-disposition": "0.5.2", + "content-type": "~1.0.4", + "cookie": "0.3.1", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.1.1", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.2", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.3", + "qs": "6.5.1", + "range-parser": "~1.2.0", + "safe-buffer": "5.1.1", + "send": "0.16.2", + "serve-static": "1.13.2", + "setprototypeof": "1.1.0", + "statuses": "~1.4.0", + "type-is": "~1.6.16", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "dependencies": { + "body-parser": { + "version": "1.18.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz", + "integrity": "sha1-h2eKGdhLR9hZuDGZvVm84iKxBFQ=", + "requires": { + "bytes": "3.0.0", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.1", + "http-errors": "~1.6.2", + "iconv-lite": "0.4.19", + "on-finished": "~2.3.0", + "qs": "6.5.1", + "raw-body": "2.3.2", + "type-is": "~1.6.15" + } + }, + "iconv-lite": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz", + "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ==" + }, + "qs": { + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", + "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==" + }, + "raw-body": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.2.tgz", + "integrity": "sha1-vNYMd9Prk83gBQKVw/N5OJvIj4k=", + "requires": { + "bytes": "3.0.0", + "http-errors": "1.6.2", + "iconv-lite": "0.4.19", + "unpipe": "1.0.0" + }, + "dependencies": { + "depd": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz", + "integrity": "sha1-V4O04cRZ8G+lyif5kfPQbnoxA1k=" + }, + "http-errors": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz", + "integrity": "sha1-CgAsyFcHGSp+eUbO7cERVfYOxzY=", + "requires": { + "depd": "1.1.1", + "inherits": "2.0.3", + "setprototypeof": "1.0.3", + "statuses": ">= 1.3.1 < 2" + } + }, + "setprototypeof": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz", + "integrity": "sha1-ZlZ+NwQ+608E2RvWWMDL77VbjgQ=" + } + } + }, + "statuses": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", + "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" + } + } + }, + "finalhandler": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", + "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", + "requires": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.2", + "statuses": "~1.4.0", + "unpipe": "~1.0.0" + }, + "dependencies": { + "statuses": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", + "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" + } + } + }, + "forwarded": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", + "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" + }, + "fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + }, + "glob": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", + "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "http-errors": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.0", + "statuses": ">= 1.4.0 < 2" + } + }, + "iconv-lite": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz", + "integrity": "sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "ipaddr.js": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.8.0.tgz", + "integrity": "sha1-6qM9bd16zo9/b+DJygRA5wZzix4=" + }, + "long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" + }, + "media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + }, + "merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + }, + "methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + }, + "mime": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz", + "integrity": "sha512-KI1+qOZu5DcW6wayYHSzR/tXKCDC5Om4s1z2QJjDULzLcmf3DvzS7oluY4HCTrc+9FiKmWUgeNLg7W3uIQvxtQ==" + }, + "mime-db": { + "version": "1.35.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.35.0.tgz", + "integrity": "sha512-JWT/IcCTsB0Io3AhWUMjRqucrHSPsSf2xKLaRldJVULioggvkJvggZ3VXNNSRkCddE6D+BUI4HEIZIA2OjwIvg==" + }, + "mime-types": { + "version": "2.1.19", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.19.tgz", + "integrity": "sha512-P1tKYHVSZ6uFo26mtnve4HQFE3koh1UWVkp8YUC+ESBHe945xWSoXuHHiGarDqcEZ+whpCDnlNw5LON0kLo+sw==", + "requires": { + "mime-db": "~1.35.0" + } + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "negotiator": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", + "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" + }, + "on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "requires": { + "ee-first": "1.1.1" + } + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "requires": { + "wrappy": "1" + } + }, + "parseurl": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz", + "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M=" + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + }, + "path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, + "proxy-addr": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.4.tgz", + "integrity": "sha512-5erio2h9jp5CHGwcybmxmVqHmnCBZeewlfJ0pex+UW7Qny7OOZXTtH56TGNyBizkgiOwhJtMKrVzDTeKcySZwA==", + "requires": { + "forwarded": "~0.1.2", + "ipaddr.js": "1.8.0" + } + }, + "qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + }, + "range-parser": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", + "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4=" + }, + "raw-body": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.3.tgz", + "integrity": "sha512-9esiElv1BrZoI3rCDuOuKCBRbuApGGaDPQfjSflGxdy4oyzqghxu6klEkkVIvBje+FF0BX9coEv8KqW6X/7njw==", + "requires": { + "bytes": "3.0.0", + "http-errors": "1.6.3", + "iconv-lite": "0.4.23", + "unpipe": "1.0.0" + } + }, + "safe-buffer": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", + "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "send": { + "version": "0.16.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.16.2.tgz", + "integrity": "sha512-E64YFPUssFHEFBvpbbjr44NCLtI1AohxQ8ZSiJjQLskAdKuriYEP6VyGEsRDH8ScozGpkaX1BGvhanqCwkcEZw==", + "requires": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "~1.6.2", + "mime": "1.4.1", + "ms": "2.0.0", + "on-finished": "~2.3.0", + "range-parser": "~1.2.0", + "statuses": "~1.4.0" + }, + "dependencies": { + "statuses": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", + "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" + } + } + }, + "serve-static": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.2.tgz", + "integrity": "sha512-p/tdJrO4U387R9oMjb1oj7qSMaMfmOyd4j9hOFoxZe2baQszgHcSWjuya/CiT5kgZZKRudHNOA0pYXOl8rQ5nw==", + "requires": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.2", + "send": "0.16.2" + } + }, + "setprototypeof": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", + "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==" + }, + "statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" + }, + "type-is": { + "version": "1.6.16", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz", + "integrity": "sha512-HRkVv/5qY2G6I8iab9cI7v1bOIdhm94dVjQCPFElW9W+3GeDOSHmy2EBYe4VTApuzolPcmgFTN3ftVJRKR2J9Q==", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.18" + } + }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + }, + "utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + }, + "vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + } + } } diff --git a/package.json b/package.json index fff0a53..00aac84 100644 --- a/package.json +++ b/package.json @@ -7,5 +7,10 @@ "test": "echo \"Error: no test specified\" && exit 1" }, "author": "", - "license": "ISC" + "license": "ISC", + "dependencies": { + "assemblyscript": "github:AssemblyScript/assemblyscript", + "body-parser": "1.18.3", + "express": "4.16.3" + } } diff --git a/server.js b/server.js new file mode 100644 index 0000000..b737169 --- /dev/null +++ b/server.js @@ -0,0 +1,53 @@ +const asc = require("assemblyscript/cli/asc"); +const fs = require('fs'); + +const express = require('express'); +const bodyParser = require('body-parser'); + +const app = express(); +app.use(bodyParser.json()); // to support JSON-encoded bodies +app.use(bodyParser.urlencoded({ // to support URL-encoded bodies + extended: true +})); + +express.static.mime.define({ + 'application/wasm': ['wasm'] +}); + +app.use(express.static('./')); + +app.post(`/makeWorkletBinary/:id`, function (req, res) { + const id = req.params.id; + const { + ts + } = req.body; + + const tsSource = `export function processor(input_ptr: i32, output_ptr: i32, length: i32): void { + ${ts} + }`; + + fs.writeFileSync(`./files/${id}.ts`, tsSource, 'utf8'); + + asc.main([ + `./files/${id}.ts`, + "--binaryFile", `./files/${id}.wasm`, + "--importMemory" + ], function (err) { + if (err) { + res.status(500).send({ + error: err + }); + } else { + const b64 = fs.readFileSync(`./files/${id}.wasm`).toString('base64'); + res.send(b64); + fs.unlinkSync(`./files/${id}.wasm`); + fs.unlinkSync(`./files/${id}.ts`); + } + }); + +}); + +app.listen(8090, (err) => { + if (err) console.error(err); + else console.info(`Started server on port [8090]`); +}); \ No newline at end of file diff --git a/src/nodes/modifiers/customWorklet.js b/src/nodes/modifiers/customWorklet.js new file mode 100644 index 0000000..a4efa55 --- /dev/null +++ b/src/nodes/modifiers/customWorklet.js @@ -0,0 +1,165 @@ +function intArrayFromBase64(s) { + var decoded = atob(s); + var bytes = new Uint8Array(decoded.length); + for (var i = 0; i < decoded.length; ++i) { + bytes[i] = decoded.charCodeAt(i); + } + return bytes; +} + +//export function processor(input_ptr: i32, output_ptr: i32, length: i32): void { +const gainTsSource = ` +let gain = 0.5; +for (let byte = 0; byte < length; ++byte) { + let inp = load(input_ptr + 4 * byte); + store(output_ptr + 4 * byte, inp * gain); +} +`; +//} + + + +//Yuk Yuk Yuk +function makeProcessorsFile(id, parsedBinary) { + return ` +const wasmBinary = new Uint8Array([${parsedBinary}]); + +const memory = new WebAssembly.Memory({ + initial: 10 +}); +const module = new WebAssembly.Module(wasmBinary); +const instance = new WebAssembly.Instance(module, { + env: { + abort(msg, file, line, column) { + console.error("abort called at main.ts:" + line + ":" + column); + }, + memory + } +}); + +const result = { + instance +}; + +const exp = result.instance.exports; + +class MyWorkletProcessor extends AudioWorkletProcessor { + constructor() { + super(); + } + + process(inputs, outputs, parameters) { + let input = inputs[0]; + let output = outputs[0]; + let channelCount = input.length; + for (let channel = 0; channel < channelCount; ++channel) { + let arr = new Float32Array(memory.buffer); + for (let i = 0; i < input[channel].length; i++) { + arr[i] = input[channel][i]; + } + // console.log(arr.slice(0,3)); + exp.processor(0, input[channel].length * Float32Array.BYTES_PER_ELEMENT, input[channel].length); + + for (let i = 0; i < input[channel].length; i++) { + output[channel][i] = arr[128 + i]; + } + } + + return true; + + } +} + +registerProcessor('${id}', MyWorkletProcessor); +`; +} + + +let latestParsedBin = null; + + +function makeWorkletPromise(audioCtx, ts) { + const id = '' + Date.now(); //TODO: better ID + return $.ajax(`/makeWorkletBinary/${id}`, { + data: JSON.stringify({ + ts + }), + contentType: 'application/JSON', + type: 'POST' + }).then(binary => { + const parsedBin = intArrayFromBase64(binary); + const fileContent = makeProcessorsFile(id, parsedBin); + latestParsedBin = parsedBin; //More bad things + return audioCtx.audioWorklet.addModule(`data:application/javascript;base64,${btoa(fileContent)}`).then(() => { + return new AudioWorkletNode(audioCtx, id); + }); + }); +} + +let prevTsSrc = null; + +module.exports = { + default () { + return { + kind: 'modifier', + type: 'customWorklet', + options: { + tsSource: gainTsSource + } + } + }, + initWANode(audioCtx, node) { + prevTsSrc = node.options.tsSource; + return makeWorkletPromise(audioCtx, node.options.tsSource); + }, + updateWANode(customNode, node, nodeIndex, graph) { + if (prevTsSrc !== node.options.tsSource) { + return makeWorkletPromise(customNode.context, node.options.tsSource) + .then(workletNode => { + const prevNode = graph.nodes[nodeIndex - 1]; + const thisNode = graph.nodes[nodeIndex]; + const nextNode = graph.nodes[nodeIndex + 1]; + + prevNode.waNode.connect(workletNode); + workletNode.connect(nextNode.waNode); + + try { + prevNode.waNode.disconnect(thisNode.waNode); + thisNode.waNode.disconnect(nexNode.waNode); + } catch (err) {} + + node.waNode = workletNode; + }); + } else { + return Promise.resolve(); + } + }, + renderView(state, affect, node, nodeIndex) { + return [ + h('h3', `Custom`), + ] + }, + renderDetail(state, affect, node, nodeIndex) { + return [ + h('textarea.ts-editor', { + value: node.options.tsSource, + onblur(ev) { + affect.set(`graph.nodes.${nodeIndex}.options.tsSource`, ev.target.value); + } + }), + h('button', 'Save') + ]; + }, + generateCode(nodeName, node) { + if (latestParsedBin) { + const fileContent = makeProcessorsFile(nodeName, latestParsedBin); + return ` +const ${nodeName} = await audioCtx.audioWorklet.addModule(\`data:application/javascript;base64,${btoa(fileContent)}\`).then(() => { + return new AudioWorkletNode(audioCtx, '${nodeName}'); +}); + `; + } else { + return ''; + } + } +} \ No newline at end of file diff --git a/src/nodes/modifiers/index.js b/src/nodes/modifiers/index.js index 44b88d1..6425e07 100644 --- a/src/nodes/modifiers/index.js +++ b/src/nodes/modifiers/index.js @@ -2,5 +2,6 @@ module.exports = { delay: require('./delay'), gain: require('./gain'), biquadFilter: require('./biquadFilter'), - analyser: require('./analyser') + analyser: require('./analyser'), + customWorklet: require('./customWorklet'), }; \ No newline at end of file diff --git a/src/styles.less b/src/styles.less index 168958b..5bcdc93 100644 --- a/src/styles.less +++ b/src/styles.less @@ -13,7 +13,7 @@ body { } input { - margin:0; + margin: 0; } * { @@ -101,4 +101,14 @@ input { } } } +} + +.ts-editor { + padding: 10px; + color: white; + background: #333; + width: 100%; + min-height: 200px; + font-family: monospace; + border: none; } \ No newline at end of file