Support p5.js script as a Clip

gulpfile.babel.js

@@ -5,6 +5,7 @@ const webpack = require("webpack");
 const CleanWebpackPlugin = require('clean-webpack-plugin');
 const UglifyJSPlugin = require('uglifyjs-webpack-plugin');
 const ForkTsCheckerWebpackPlugin = require('fork-ts-checker-webpack-plugin')
+const MonacoEditorWebpackPlugin = require('monaco-editor-webpack-plugin')
 const builder = require('electron-builder');
 const nib = require('nib');
 const notifier = require('node-notifier');
@@ -85,14 +86,14 @@ export function compileRendererJs(done) {
         mode: __DEV__ ? 'development' : 'production',
         target: "electron-renderer",
         watch: __DEV__,
-        context: paths.src.root,
+        context: paths.src.frontend,
         entry: {
-            'delir/main': ['./delir/main'],
+            'main': ['./main'],
         },
         output: {
             filename: "[name].js",
             sourceMapFilename: "map/[file].map",
-            path: paths.compiled.root,
+            path: paths.compiled.frontend,
         },
         devtool: __DEV__ ? "#source-map" : 'none',
         resolve: {
@@ -142,6 +143,14 @@ export function compileRendererJs(done) {
                         },
                     ],
                 },
+                {
+                    test: /\.css$/,
+                    include: /node_modules\/monaco-editor/,
+                    use: [
+                        'style-loader',
+                        'css-loader',
+                    ],
+                }
             ]
         },
         plugins: [
@@ -157,6 +166,7 @@ export function compileRendererJs(done) {
             }),
             // preserve require() for native modules
             new webpack.ExternalsPlugin('commonjs', NATIVE_MODULES),
+            new MonacoEditorWebpackPlugin(),
             new ForkTsCheckerWebpackPlugin({
                 tsconfig: join(paths.src.frontend, 'tsconfig.json'),
                 workers: 3,

package.json

@@ -8,7 +8,7 @@
   "license": "MIT",
   "main": "browser/main.js",
   "scripts": {
-    "dev": "cross-env DELIR_ENV=dev gulp",
+    "dev": "cross-env electron-rebuild && cross-env DELIR_ENV=dev gulp",
     "rebuild": "cross-env electron-rebuild",
     "build": "cross-env DELIR_ENV=production gulp publish",
     "declare": "cross-env tsc -p src/delir-core -d --declarationDir src/delir-core/types",
@@ -53,6 +53,7 @@
     "jsdom-global": "3.0.2",
     "lint-staged": "7.2.0",
     "mocha": "5.2.0",
+    "monaco-editor-webpack-plugin": "1.5.1",
     "nib": "1.1.2",
     "node-notifier": "5.2.1",
     "raw-loader": "0.5.1",

packages/delir-core/package.json

@@ -26,6 +26,7 @@
     "fs-promise": "2.0.3",
     "lodash": "4.17.4",
     "mp3": "0.1.0",
+    "p5": "0.7.1",
     "semver": "5.4.1",
     "uuid": "3.1.0"
   }

packages/delir-core/src/declarations.d.ts

@@ -60,6 +60,11 @@ declare module 'font-manager' {
     export {getAvailableFontsSync}
 }
 
+declare module 'p5' {
+    const _: any
+    export = _
+}
+
 // declare module 'fs-extra' {
 //     const _: any
 //     export default _

packages/delir-core/src/engine/pipeline/Task/ClipRenderTask.ts

@@ -3,11 +3,9 @@ import * as _ from 'lodash'
 import * as KeyframeHelper from '../../../helper/keyframe-helper'
 import { TypeDescriptor } from '../../../plugin-support/type-descriptor'
 import { Clip } from '../../../project'
-import { KeyframeValueTypes } from '../../../project/keyframe'
 import { Expression } from '../../../values'
 import * as RendererFactory from '../../renderer'
 import { IRenderer } from '../../renderer/renderer-base'
-import DependencyResolver from '../DependencyResolver'
 import { compileTypeScript } from '../ExpressionCompiler'
 import { ContextSource } from '../ExpressionContext'
 import * as ExpressionContext from '../ExpressionContext'
@@ -23,7 +21,7 @@ export default class ClipRenderTask {
         req: RenderRequest,
     }): ClipRenderTask {
         const rendererParams = RendererFactory.getInfo(clip.renderer).parameter
-        const rendererAssetParamNames = rendererParams.properties.filter(prop => prop.type === 'ASSET').map(prop => prop.propName)
+        const rendererAssetParamNames = rendererParams.properties.filter(prop => prop.type === 'ASSET').map(prop => prop.paramName)
 
         const rawRendererInitParam = KeyframeHelper.calcKeyframeValuesAt(0, clip.placedFrame, rendererParams, clip.keyframes)
         const rendererInitParam: RealParameterValues = { ...(rawRendererInitParam as any) }
@@ -41,10 +39,10 @@ export default class ClipRenderTask {
         }
 
         const rawRendererKeyframeLUT = KeyframeHelper.calcKeyFrames(rendererParams, clip.keyframes, clip.placedFrame, 0, req.durationFrames)
-        const rendererKeyframeLUT: { [paramName: string]: { [frame: number]: RealParameterValueTypes } } = { ...(rawRendererInitParam as any)　}
-        rendererAssetParamNames.forEach(propName => {
+        const rendererKeyframeLUT: { [paramName: string]: { [frame: number]: RealParameterValueTypes } } = { ...(rawRendererKeyframeLUT as any)　}
+        rendererAssetParamNames.forEach(paramName => {
             // resolve asset
-            rendererKeyframeLUT[propName] = _.map(rawRendererKeyframeLUT[propName], value => {
+            rendererKeyframeLUT[paramName] = _.map(rawRendererKeyframeLUT[paramName], value => {
                 return value ? req.resolver.resolveAsset((value as AssetPointerScheme).assetId) : null
             })
         })

packages/delir-core/src/engine/pipeline/Task/EffectRenderTask.ts

@@ -25,7 +25,7 @@ export default class EffectRenderTask {
         const EffectPluginClass = resolver.resolveEffectPlugin(effect.processor)!
 
         const effectParams = EffectPluginClass.provideParameters()
-        const effectAssetParamNames = effectParams.properties.filter(prop => prop.type === 'ASSET').map(prop => prop.propName)
+        const effectAssetParamNames = effectParams.properties.filter(prop => prop.type === 'ASSET').map(prop => prop.paramName)
 
         let effectRenderer = effectCache.get(effect)
 

packages/delir-core/src/engine/pipeline/pipeline.ts

@@ -28,7 +28,7 @@ import EffectRenderTask from './Task/EffectRenderTask'
 import WebGLContextPool from './WebGLContextPool'
 
 export interface ExpressionExecuters {
-    [propName: string]: (exposes: ExpressionContext.ContextSource) => ParameterValueTypes
+    [paramName: string]: (exposes: ExpressionContext.ContextSource) => ParameterValueTypes
 }
 
 interface LayerRenderTask {
@@ -62,12 +62,12 @@ export interface RealParameterValues {
 export const applyExpression = (
     req: RenderingRequest,
     beforeExpressionParams: RealParameterValues,
-    expressions: { [prop: string]: (exposes: ExpressionContext.ContextSource) => RealParameterValueTypes },
-): { [prop: string]: ParameterValueTypes } => {
-    return _.mapValues(beforeExpressionParams, (value, propName) => {
-        if (expressions[propName!]) {
+    expressions: { [param: string]: (exposes: ExpressionContext.ContextSource) => RealParameterValueTypes },
+): { [param: string]: ParameterValueTypes } => {
+    return _.mapValues(beforeExpressionParams, (value, paramName) => {
+        if (expressions[paramName!]) {
             // TODO: Value type Validation
-            const result = expressions[propName!]({
+            const result = expressions[paramName!]({
                 req,
                 clipProperties: beforeExpressionParams,
                 currentValue: value
@@ -402,7 +402,7 @@ export default class Pipeline
 
                 // Lookup current frame prop value from pre-calculated lookup-table
                 const beforeExpressionParams = _.fromPairs(clipTask.rendererParams.properties.map(desc => {
-                    return [desc.propName, clipTask.keyframeLUT[desc.propName][req.frame]]
+                    return [desc.paramName, clipTask.keyframeLUT[desc.paramName][req.frame]]
                 }))
 
                 // Apply expression
@@ -434,8 +434,8 @@ export default class Pipeline
                 // Post process effects
                 for (const effectTask of clipTask.effectRenderTask) {
                     const beforeExpressionEffectorParams = _.fromPairs(effectTask.effectorProps.properties.map(desc => {
-                        return [desc.propName, effectTask.keyframeLUT[desc.propName][req.frame]]
-                    })) as {[propName: string]: ParameterValueTypes}
+                        return [desc.paramName, effectTask.keyframeLUT[desc.paramName][req.frame]]
+                    })) as {[paramName: string]: ParameterValueTypes}
 
                     const afterExpressionEffectorParams = applyExpression(clipScopeReq, beforeExpressionEffectorParams, effectTask.expressions)
 

packages/delir-core/src/engine/renderer/P5js/P5Hooks.ts

@@ -0,0 +1,39 @@
+import DependencyResolver from '@ragg/delir-core/src/engine/pipeline/DependencyResolver'
+import * as P5 from 'p5'
+
+interface Sketch {
+    setup(): void
+    draw(): void
+}
+
+export default class P5Hooks {
+    public p5: any
+    private node: HTMLDivElement
+
+    constructor(private resolver: DependencyResolver) {
+        // nodeを渡さないとdocument.body.appendChildされてしまう！
+        // https://github.com/processing/p5.js/blob/a64959e2722c4cad9327246be494f0b472ccd54c/src/core/rendering.js#L98
+        this.node = document.createElement('div')
+
+        this.p5 = new P5((p5: any) => {
+            p5._loop = false // disable auto rendering
+            p5.setup = () => {} // noop
+            p5.draw = () => {} // noop
+            p5.loadImage = this.loadImage
+        }, this.node)
+    }
+
+    get preloadCount() {
+        return this.p5._preloadCount
+    }
+
+    private loadImage = (path: string, successCallback: (pImg: any) => void, failureCallback: (e: any) => void) => {
+        const match = /^delir:(.+)/.exec(path)
+
+        if (match) {
+            path = 'file://' + this.resolver.resolveAsset(match[1])!.path
+        }
+
+        return P5.prototype.loadImage.call(this.p5, path, successCallback, failureCallback)
+    }
+}

packages/delir-core/src/engine/renderer/P5js/P5js.ts

@@ -0,0 +1,105 @@
+import * as VM from 'vm'
+import P5Hooks from './P5Hooks'
+
+import Type from '../../../plugin-support/type-descriptor'
+import Expression from '../../../values/expression'
+import PreRenderingRequest from '../../pipeline/pre-rendering-request'
+import RenderingRequest from '../../pipeline/render-request'
+import { IRenderer } from '../renderer-base'
+
+interface SketchRendererParams {
+    sketch: Expression
+}
+
+export default class P5jsRenderer implements IRenderer<SketchRendererParams>
+{
+    public static get rendererId(): string { return 'p5js' }
+
+    public static provideAssetAssignMap() {
+        return {}
+    }
+
+    public static provideParameters()
+    {
+        return Type.code('sketch', {
+            label: 'Sketch',
+            langType: 'javascript',
+            defaultValue: new Expression('javascript', 'function setup() {\n    \n}\n\nfunction draw() {\n    \n}\n')
+        })
+    }
+
+    private vmGlobal: any
+    private vmScope: any
+    private p5ex: P5Hooks
+    private canvas: HTMLCanvasElement
+
+    public async beforeRender(req: PreRenderingRequest<SketchRendererParams>)
+    {
+        this.p5ex = new P5Hooks(req.resolver)
+
+        // Make VM environment
+        this.vmScope = this.makeVmScopeVariables(req)
+
+        this.vmGlobal = VM.createContext(new Proxy({
+            console: global.console,
+            p5: this.p5ex.p5
+        }, {
+            get: (target: any, propKey) => {
+                if (target[propKey]) {
+                    return target[propKey]
+                } else if (this.p5ex.p5[propKey]) {
+                    // Expose p5 drawing methods
+                    return typeof this.p5ex.p5[propKey] === 'function' ? this.p5ex.p5[propKey].bind(this.p5ex.p5) : this.p5ex.p5[propKey]
+                } else {
+                    // Dynamic exposing
+                    return this.vmScope[propKey]
+                }
+            }
+        }))
+
+        const vm = new VM.Script(req.parameters.sketch.code, {})
+
+        // Make VM scope binded sketch object
+        vm.runInContext(this.vmGlobal)
+
+        this.canvas = this.p5ex.p5.canvas
+        this.p5ex.p5.createCanvas(req.width, req.height)
+
+        if (typeof this.vmGlobal.setup === 'function') {
+            this.vmGlobal.setup()
+        }
+    }
+
+    public async render(req: RenderingRequest<SketchRendererParams>)
+    {
+        await new Promise(resolve => {
+            const intervalId = setInterval(() => {
+                if (this.p5ex.preloadCount === 0) {
+                    resolve()
+                    clearInterval(intervalId)
+                }
+             })
+        })
+
+        this.vmScope = this.makeVmScopeVariables(req)
+        this.vmGlobal.draw && this.vmGlobal.draw()
+        req.destCanvas.getContext('2d')!.drawImage(this.canvas, 0, 0)
+    }
+
+    private makeVmScopeVariables(req: PreRenderingRequest<any> | RenderingRequest<any>)
+    {
+        return {
+            delir: {
+                ctx: {
+                    width: req.width,
+                    height: req.height,
+                    framerate: req.framerate,
+                    time: (req as RenderingRequest).time,
+                    frame: (req as RenderingRequest).frame,
+                    timeOnClip: (req as RenderingRequest).timeOnClip,
+                    frameOnClip: (req as RenderingRequest).frameOnClip,
+                }
+            }
+        }
+    }
+}

packages/delir-core/src/engine/renderer/index.ts

@@ -7,17 +7,19 @@ import UnknownPluginReferenceException from '../../exceptions/unknown-plugin-ref
 import AdjustmentRenderer from './Adjustment/Adjustment'
 import AudioRenderer from './Audio/Audio'
 import ImageRenderer from './Image/Image'
+import P5jsRenderer from './P5js/P5js'
 import TextRenderer from './Text/Text'
 import VideoRenderer from './Video/Video'
 
-export type AvailableRenderer = 'audio' | 'image' | 'video' | 'text' | 'adjustment'
+export type AvailableRenderer = 'audio' | 'image' | 'video' | 'text' | 'adjustment' | 'p5js'
 
 export const RENDERERS: {[name: string]: IRendererStatic} = {
     audio: AudioRenderer,
     video: VideoRenderer,
     image: ImageRenderer,
     text: TextRenderer,
     adjustment: AdjustmentRenderer,
+    p5js: P5jsRenderer,
 }
 
 interface PluginInfo {