From 568c4ec051ad62f6c2e1dd1fab0e3ec57546eabf Mon Sep 17 00:00:00 2001 From: wong-justin Date: Thu, 3 Aug 2023 15:35:00 -0400 Subject: [PATCH 01/54] mark structure for loading new shader files --- src/webgl/p5.RendererGL.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index 35a809c33f..fc22765f43 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -79,6 +79,11 @@ const defaultShaders = { pointFrag: readFileSync(join(__dirname, '/shaders/point.frag'), 'utf-8') }; +// const filterShaders = { +// [constants.GRAY]: +// readFileSync(join(__dirname, '/shaders/filters/gray.frag'), 'utf-7'), +// }; + /** * @module Rendering * @submodule Rendering From 9f15a50575ad017690e048bc572168322cec9098 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Thu, 3 Aug 2023 15:36:27 -0400 Subject: [PATCH 02/54] destructure args passed to p5.RendererGL.filter --- src/webgl/p5.RendererGL.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index fc22765f43..fafbd7983d 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -956,7 +956,7 @@ p5.RendererGL = class RendererGL extends p5.Renderer { this.curStrokeJoin = join; } - filter(args) { + filter(...args) { // Couldn't create graphics in RendererGL constructor // (led to infinite loop) // so it's just created here once on the initial filter call. @@ -978,7 +978,7 @@ p5.RendererGL = class RendererGL extends p5.Renderer { if (typeof args[0] === 'string') { // TODO, handle filter constants: // this.filterShader = map(args[0], {GRAYSCALE: grayscaleShader, ...}) - // filterOperationParameter = undefined or args[1] + // this.filterShader.setUniform(args[1] if it exists) p5._friendlyError('webgl filter implementation in progress'); return; } From e2f123d2a3042b937d75b41943e46f070f3a0626 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Thu, 3 Aug 2023 15:38:50 -0400 Subject: [PATCH 03/54] change public filter() signature adding an opt-out parameter made things a little more complicated (handling two optional parameters now) using shaders behind a P2D renderer also adds complexity ^ still need to handle that case --- src/image/pixels.js | 69 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 9 deletions(-) diff --git a/src/image/pixels.js b/src/image/pixels.js index 858796f492..dea2119c06 100644 --- a/src/image/pixels.js +++ b/src/image/pixels.js @@ -355,8 +355,11 @@ p5.prototype._copyHelper = ( * POSTERIZE, BLUR, ERODE, DILATE or BLUR. * See Filters.js for docs on * each available filter - * @param {Number} [filterParam] an optional parameter unique + * @param {Number} filterParam an optional parameter unique * to each filter, see above + * @param {Boolean} [useWebGL] a flag to control whether to use fast + * WebGL filters (GPU) or original image + * filters (CPU); defaults to true * * @example *
@@ -514,27 +517,75 @@ p5.prototype._copyHelper = ( * gray square */ +/** + * @method filter + * @param {Constant} filterType + * @param {Boolean} [useWebGL] + */ /** * @method filter * @param {p5.Shader} shaderFilter A shader that's been loaded, with the * frag shader using a `tex0` uniform */ -p5.prototype.filter = function(operation, value) { +p5.prototype.filter = function(...args) { p5._validateParameters('filter', arguments); - // TODO: use shader filters always, and provide an opt out - if (this._renderer.isP3D) { - p5.RendererGL.prototype.filter.call(this._renderer, arguments); + let { shader, operation, value, useWebGL } = parseFilterArgs(...args); + + // when passed a shader, use it directly + if (shader) { + p5.RendererGL.prototype.filter.call(this._renderer, shader); return; } - if (this.canvas !== undefined) { - Filters.apply(this.canvas, Filters[operation], value); - } else { - Filters.apply(this.elt, Filters[operation], value); + // when opting out of webgl, use old pixels method + if (!useWebGL) { + if (this.canvas !== undefined) { + Filters.apply(this.canvas, Filters[operation], value); + } else { + Filters.apply(this.elt, Filters[operation], value); + } + return; + } + + // when this is a webgl renderer, apply constant shader filter + if (this._renderer.isP3D) { + p5.RendererGL.prototype.filter.call(this._renderer, operation, value); + } + + // when this is P2D renderer, create/use hidden webgl renderer + else { + // TODO: create/use hidden webgl renderer and transfer contents to this p2d + p5._friendlyError('webgl filter implementation in progress'); } }; +function parseFilterArgs(...args) { + let result = { + shader: undefined, + operation: undefined, + value: undefined, + useWebGL: true + }; + + if (args[0] instanceof p5.Shader) { + result.shader = args[0]; + return result; + } + else { + result.operation = args[0]; + } + + if (args.length > 1 && typeof args[1] === 'number') { + result.value = args[1]; + } + + if (args[args.length-1] === false) { + result.useWebGL = false; + } + return result; +} + /** * Get a region of pixels, or a single pixel, from the canvas. * From bd8dab45e39b1372d1798fad71d756aab55753d2 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Thu, 3 Aug 2023 16:06:15 -0400 Subject: [PATCH 04/54] comment overloaded function parameters for clarity --- src/image/pixels.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/image/pixels.js b/src/image/pixels.js index dea2119c06..aca47588c7 100644 --- a/src/image/pixels.js +++ b/src/image/pixels.js @@ -561,6 +561,10 @@ p5.prototype.filter = function(...args) { }; function parseFilterArgs(...args) { + // possible parameters: + // - operation, value, [useWebGL] + // - operation, [useWebGL] + // - shader let result = { shader: undefined, operation: undefined, From 9a026453317306ed332d5e2acf8736edc6136076 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Fri, 4 Aug 2023 16:00:00 -0400 Subject: [PATCH 05/54] whitespace --- src/image/pixels.js | 9 +++++---- src/webgl/material.js | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/image/pixels.js b/src/image/pixels.js index aca47588c7..50dbabdacd 100644 --- a/src/image/pixels.js +++ b/src/image/pixels.js @@ -561,10 +561,11 @@ p5.prototype.filter = function(...args) { }; function parseFilterArgs(...args) { - // possible parameters: - // - operation, value, [useWebGL] - // - operation, [useWebGL] - // - shader + // args could be: + // - operation, value, [useWebGL] + // - operation, [useWebGL] + // - shader + let result = { shader: undefined, operation: undefined, diff --git a/src/webgl/material.js b/src/webgl/material.js index feef4ee912..32336c757a 100644 --- a/src/webgl/material.js +++ b/src/webgl/material.js @@ -268,16 +268,16 @@ p5.prototype.createFilterShader = function(fragSrc) { // so pass texcoords on to the fragment shader in a varying variable attribute vec2 aTexCoord; varying vec2 vTexCoord; - + void main() { // transferring texcoords for the frag shader vTexCoord = aTexCoord; - + // copy position with a fourth coordinate for projection (1.0 is normal) vec4 positionVec4 = vec4(aPosition, 1.0); // scale by two and center to achieve correct positioning positionVec4.xy = positionVec4.xy * 2.0 - 1.0; - + gl_Position = positionVec4; } `; From 5bbb9800a9cf880aa8877b21e06a943e4aad837a Mon Sep 17 00:00:00 2001 From: wong-justin Date: Fri, 4 Aug 2023 16:04:56 -0400 Subject: [PATCH 06/54] start adding shaders for filter constants (GRAY) doesn't work yet though --- src/webgl/p5.RendererGL.js | 54 +++++++++++++++----------- src/webgl/shaders/filters/default.vert | 17 ++++++++ src/webgl/shaders/filters/gray.frag | 11 ++++++ 3 files changed, 60 insertions(+), 22 deletions(-) create mode 100644 src/webgl/shaders/filters/default.vert create mode 100644 src/webgl/shaders/filters/gray.frag diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index fafbd7983d..f3c0824139 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -79,10 +79,12 @@ const defaultShaders = { pointFrag: readFileSync(join(__dirname, '/shaders/point.frag'), 'utf-8') }; -// const filterShaders = { -// [constants.GRAY]: -// readFileSync(join(__dirname, '/shaders/filters/gray.frag'), 'utf-7'), -// }; +// TODO: add remaining filter shaders +const filterShaderFrags = { + [constants.GRAY]: + readFileSync(join(__dirname, '/shaders/filters/gray.frag'), 'utf-8') +}; +const filterShaderVert = readFileSync(join(__dirname, '/shaders/filters/default.vert'), 'utf-8'); /** * @module Rendering @@ -975,26 +977,34 @@ p5.RendererGL = class RendererGL extends p5.Renderer { } let pg = this.filterGraphicsLayer; + // use internal shader for filter constants BLUR, INVERT, etc if (typeof args[0] === 'string') { - // TODO, handle filter constants: - // this.filterShader = map(args[0], {GRAYSCALE: grayscaleShader, ...}) - // this.filterShader.setUniform(args[1] if it exists) - p5._friendlyError('webgl filter implementation in progress'); - return; + let operation = args[0]; + let value = args[1]; + this.filterShader = new p5.Shader( + this, + filterShaderVert, + filterShaderFrags[operation] + ); + this.filterShader.setUniform('filterParameter', value); + // TODO: fix error 'INVALID_OPERATION: no valid shader program in use' } - let userShader = args[0]; - - // Copy the user shader once on the initial filter call, - // since it has to be bound to pg and not main - let isSameUserShader = ( - this.filterShader !== undefined && - userShader._vertSrc === this.filterShader._vertSrc && - userShader._fragSrc === this.filterShader._fragSrc - ); - if (!isSameUserShader) { - this.filterShader = - new p5.Shader(pg._renderer, userShader._vertSrc, userShader._fragSrc); - this.filterShader.parentShader = userShader; + // use custom user-supplied shader + else { + let userShader = args[0]; + + // Copy the user shader once on the initial filter call, + // since it has to be bound to pg and not main + let isSameUserShader = ( + this.filterShader !== undefined && + userShader._vertSrc === this.filterShader._vertSrc && + userShader._fragSrc === this.filterShader._fragSrc + ); + if (!isSameUserShader) { + this.filterShader = + new p5.Shader(pg._renderer, userShader._vertSrc, userShader._fragSrc); + this.filterShader.parentShader = userShader; + } } // apply shader to pg diff --git a/src/webgl/shaders/filters/default.vert b/src/webgl/shaders/filters/default.vert new file mode 100644 index 0000000000..c5c74680a9 --- /dev/null +++ b/src/webgl/shaders/filters/default.vert @@ -0,0 +1,17 @@ +attribute vec3 aPosition; +// texcoords only come from p5 to vertex shader +// so pass texcoords on to the fragment shader in a varying variable +attribute vec2 aTexCoord; +varying vec2 vTexCoord; + +void main() { + // transferring texcoords for the frag shader + vTexCoord = aTexCoord; + + // copy position with a fourth coordinate for projection (1.0 is normal) + vec4 positionVec4 = vec4(aPosition, 1.0); + // scale by two and center to achieve correct positioning + positionVec4.xy = positionVec4.xy * 2.0 - 1.0; + + gl_Position = positionVec4; +} diff --git a/src/webgl/shaders/filters/gray.frag b/src/webgl/shaders/filters/gray.frag new file mode 100644 index 0000000000..1b5e53e055 --- /dev/null +++ b/src/webgl/shaders/filters/gray.frag @@ -0,0 +1,11 @@ +precision highp float; + +varying vec2 vTexCoord; + +uniform sampler2D tex0; + +void main() { + vec4 tex = texture2D(tex0, vTexCoord); + float gray = (tex.r + tex.g + tex.b) / 3.0; + gl_FragColor = vec4(gray, gray, gray, 1.0); +} From c1bfee4497c5f70588d9202e4a5d5d4b7680dbd9 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Thu, 10 Aug 2023 08:46:53 -0400 Subject: [PATCH 07/54] fix by creating shaders on secondary graphics layer --- src/webgl/p5.RendererGL.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index f3c0824139..504005326e 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -982,12 +982,11 @@ p5.RendererGL = class RendererGL extends p5.Renderer { let operation = args[0]; let value = args[1]; this.filterShader = new p5.Shader( - this, + pg._renderer, filterShaderVert, filterShaderFrags[operation] ); this.filterShader.setUniform('filterParameter', value); - // TODO: fix error 'INVALID_OPERATION: no valid shader program in use' } // use custom user-supplied shader else { From ef397a1d8b96a8eec3d9818292628b3d055e6440 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Thu, 10 Aug 2023 12:16:30 -0400 Subject: [PATCH 08/54] use old luminance constants for GRAY --- src/webgl/shaders/filters/gray.frag | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/webgl/shaders/filters/gray.frag b/src/webgl/shaders/filters/gray.frag index 1b5e53e055..815388c746 100644 --- a/src/webgl/shaders/filters/gray.frag +++ b/src/webgl/shaders/filters/gray.frag @@ -4,8 +4,13 @@ varying vec2 vTexCoord; uniform sampler2D tex0; +float luma(vec3 color) { + // weighted grayscale with luminance values + return dot(color, vec3(0.2126, 0.7152, 0.0722)); +} + void main() { vec4 tex = texture2D(tex0, vTexCoord); - float gray = (tex.r + tex.g + tex.b) / 3.0; - gl_FragColor = vec4(gray, gray, gray, 1.0); + float gray = luma(tex.rgb); + gl_FragColor = vec4(gray, gray, gray, tex.a); } From 197c9056eae0fa95a0292ec2e6c05c2035c5add5 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Thu, 10 Aug 2023 12:46:47 -0400 Subject: [PATCH 09/54] add texelSize as a default uniform needed for filters that sample neighbors, such as ERODE, DILATE, BLUR --- src/webgl/p5.RendererGL.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index 504005326e..6d9fac41b6 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -1009,6 +1009,7 @@ p5.RendererGL = class RendererGL extends p5.Renderer { // apply shader to pg pg.shader(this.filterShader); this.filterShader.setUniform('tex0', this); + this.filterShader.setUniform('texelSize', [1.0/this.width, 1.0/this.height]); pg.rect(0,0,this.width,this.height); // draw pg contents onto main renderer From 1e34031e47e38d8da3e911eeee674d52d739b957 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Thu, 10 Aug 2023 12:48:24 -0400 Subject: [PATCH 10/54] add ERODE --- src/webgl/p5.RendererGL.js | 4 ++- src/webgl/shaders/filters/erode.frag | 38 ++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 src/webgl/shaders/filters/erode.frag diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index 6d9fac41b6..4994d37003 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -82,7 +82,9 @@ const defaultShaders = { // TODO: add remaining filter shaders const filterShaderFrags = { [constants.GRAY]: - readFileSync(join(__dirname, '/shaders/filters/gray.frag'), 'utf-8') + readFileSync(join(__dirname, '/shaders/filters/gray.frag'), 'utf-8'), + [constants.ERODE]: + readFileSync(join(__dirname, '/shaders/filters/erode.frag'), 'utf-8') }; const filterShaderVert = readFileSync(join(__dirname, '/shaders/filters/default.vert'), 'utf-8'); diff --git a/src/webgl/shaders/filters/erode.frag b/src/webgl/shaders/filters/erode.frag new file mode 100644 index 0000000000..055f4167e6 --- /dev/null +++ b/src/webgl/shaders/filters/erode.frag @@ -0,0 +1,38 @@ +// Increase the bright areas in an image + +precision highp float; + +varying vec2 vTexCoord; + +uniform sampler2D tex0; +uniform vec2 texelSize; + +float luma(vec3 color) { + // based on constants 77, 151, 28 from ERODE in filters.js, + // even though that's different than the luminance constants used in GRAY + return dot(color, vec3(0.3008, 0.5898, 0.1094)); +} + +void main() { + vec4 curColor = texture2D(tex0, vTexCoord); + float curLuminance = luma(curColor.rgb); + + // set current color as the neighbor color with lowest luminance + + vec4 neighbors[4]; + neighbors[0] = texture2D(tex0, vTexCoord + vec2( texelSize.x, 0.0)); + neighbors[1] = texture2D(tex0, vTexCoord + vec2(-texelSize.x, 0.0)); + neighbors[2] = texture2D(tex0, vTexCoord + vec2(0.0, texelSize.y)); + neighbors[3] = texture2D(tex0, vTexCoord + vec2(0.0, -texelSize.y)); + + for (int i = 0; i < 4; i++) { + vec4 color = neighbors[i]; + float lum = luma(color.rgb); + if (lum < curLuminance) { + curColor = color; + curLuminance = lum; + } + } + + gl_FragColor = curColor; +} From 4bf256586efaa7d003b86b3ed58ad025e2fc87b7 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Thu, 10 Aug 2023 15:26:14 -0400 Subject: [PATCH 11/54] add DILATE --- src/webgl/p5.RendererGL.js | 4 ++- src/webgl/shaders/filters/dilate.frag | 38 +++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 src/webgl/shaders/filters/dilate.frag diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index 4994d37003..f8d76975cd 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -84,7 +84,9 @@ const filterShaderFrags = { [constants.GRAY]: readFileSync(join(__dirname, '/shaders/filters/gray.frag'), 'utf-8'), [constants.ERODE]: - readFileSync(join(__dirname, '/shaders/filters/erode.frag'), 'utf-8') + readFileSync(join(__dirname, '/shaders/filters/erode.frag'), 'utf-8'), + [constants.DILATE]: + readFileSync(join(__dirname, '/shaders/filters/dilate.frag'), 'utf-8') }; const filterShaderVert = readFileSync(join(__dirname, '/shaders/filters/default.vert'), 'utf-8'); diff --git a/src/webgl/shaders/filters/dilate.frag b/src/webgl/shaders/filters/dilate.frag new file mode 100644 index 0000000000..58a6c34f88 --- /dev/null +++ b/src/webgl/shaders/filters/dilate.frag @@ -0,0 +1,38 @@ +// Increase the bright areas in an image + +precision highp float; + +varying vec2 vTexCoord; + +uniform sampler2D tex0; +uniform vec2 texelSize; + +float luma(vec3 color) { + // based on constants 77, 151, 28 from DILATE in filters.js, + // even though that's different than the luminance constants used in GRAY + return dot(color, vec3(0.3008, 0.5898, 0.1094)); +} + +void main() { + vec4 curColor = texture2D(tex0, vTexCoord); + float curLuminance = luma(curColor.rgb); + + // set current color as the neighbor color with highest luminance + + vec4 neighbors[4]; + neighbors[0] = texture2D(tex0, vTexCoord + vec2( texelSize.x, 0.0)); + neighbors[1] = texture2D(tex0, vTexCoord + vec2(-texelSize.x, 0.0)); + neighbors[2] = texture2D(tex0, vTexCoord + vec2(0.0, texelSize.y)); + neighbors[3] = texture2D(tex0, vTexCoord + vec2(0.0, -texelSize.y)); + + for (int i = 0; i < 4; i++) { + vec4 color = neighbors[i]; + float lum = luma(color.rgb); + if (lum > curLuminance) { + curColor = color; + curLuminance = lum; + } + } + + gl_FragColor = curColor; +} From ed40a78652f4594eafbdcbe8df7950ba8f909296 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Thu, 10 Aug 2023 15:26:54 -0400 Subject: [PATCH 12/54] fix comments explaining what DILATE/ERODE do the descriptions were swapped so now they match the docs --- src/image/filters.js | 4 ++-- src/webgl/shaders/filters/erode.frag | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/image/filters.js b/src/image/filters.js index d4707438e3..83c47a661a 100644 --- a/src/image/filters.js +++ b/src/image/filters.js @@ -308,7 +308,7 @@ const Filters = { }, /** - * reduces the bright areas in an image + * increases the bright areas in an image * @private * @param {Canvas} canvas */ @@ -395,7 +395,7 @@ const Filters = { }, /** - * increases the bright areas in an image + * reduces the bright areas in an image * @private * @param {Canvas} canvas */ diff --git a/src/webgl/shaders/filters/erode.frag b/src/webgl/shaders/filters/erode.frag index 055f4167e6..adde70ea4c 100644 --- a/src/webgl/shaders/filters/erode.frag +++ b/src/webgl/shaders/filters/erode.frag @@ -1,4 +1,4 @@ -// Increase the bright areas in an image +// Reduces the bright areas in an image precision highp float; From fc4181e2537b94b3fbb65c6d568afb2760df8ce1 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Thu, 10 Aug 2023 17:30:18 -0400 Subject: [PATCH 13/54] add BLUR Low-quality single pass filter, hopefully to be replaced soon. Also uniform/parameter doesn't work --- src/webgl/p5.RendererGL.js | 4 ++- src/webgl/shaders/filters/blur.frag | 50 +++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 src/webgl/shaders/filters/blur.frag diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index f8d76975cd..6974bfeb09 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -86,7 +86,9 @@ const filterShaderFrags = { [constants.ERODE]: readFileSync(join(__dirname, '/shaders/filters/erode.frag'), 'utf-8'), [constants.DILATE]: - readFileSync(join(__dirname, '/shaders/filters/dilate.frag'), 'utf-8') + readFileSync(join(__dirname, '/shaders/filters/dilate.frag'), 'utf-8'), + [constants.BLUR]: + readFileSync(join(__dirname, '/shaders/filters/blur.frag'), 'utf-8'), }; const filterShaderVert = readFileSync(join(__dirname, '/shaders/filters/default.vert'), 'utf-8'); diff --git a/src/webgl/shaders/filters/blur.frag b/src/webgl/shaders/filters/blur.frag new file mode 100644 index 0000000000..1c21a1f91b --- /dev/null +++ b/src/webgl/shaders/filters/blur.frag @@ -0,0 +1,50 @@ +// Single-pass blur filter, taken from Adam Ferriss' repo of shader examples: +// https://github.com/aferriss/p5jsShaderExamples/blob/gh-pages/4_image-effects/4-9_single-pass-blur/effect.frag + +precision highp float; + +// lets grab texcoords just for fun +varying vec2 vTexCoord; + +// our texture coming from p5 +uniform sampler2D tex0; +uniform vec2 texelSize; +uniform float filterParameter; + +void main() { + + vec2 uv = vTexCoord; + + // a single pass blur works by sampling all the neighbor pixels and averaging them up + // this is somewhat inefficient because we have to sample the texture 9 times -- texture2D calls are slow :( + // check out the two-pass-blur example for a better blur approach + // get the webcam as a vec4 using texture2D + + // spread controls how far away from the center we should pull a sample from + // you will start to see artifacts if you crank this up too high + float spread = 4.0; + if (filterParameter > 0.0) { + spread = filterParameter; + } + + // create our offset variable by multiplying the size of a texel with spread + vec2 offset = texelSize * spread; + + // get all the neighbor pixels! + vec4 tex = texture2D(tex0, uv); // middle middle -- the actual texel / pixel + tex += texture2D(tex0, uv + vec2(-offset.x, -offset.y)); // top left + tex += texture2D(tex0, uv + vec2(0.0, -offset.y)); // top middle + tex += texture2D(tex0, uv + vec2(offset.x, -offset.y)); // top right + + tex += texture2D(tex0, uv + vec2(-offset.x, 0.0)); //middle left + tex += texture2D(tex0, uv + vec2(offset.x, 0.0)); //middle right + + tex += texture2D(tex0, uv + vec2(-offset.x, offset.y)); // bottom left + tex += texture2D(tex0, uv + vec2(0.0, offset.y)); // bottom middle + tex += texture2D(tex0, uv + vec2(offset.x, offset.y)); // bottom right + + // we added 9 textures together, so we will divide by 9 to average them out and move the values back into a 0 - 1 range + tex /= 9.0; + + gl_FragColor = tex; +} From a94936e62bfd84df58cd77cbea0b89e99052b54a Mon Sep 17 00:00:00 2001 From: wong-justin Date: Thu, 10 Aug 2023 17:47:40 -0400 Subject: [PATCH 14/54] add partial POSTERIZE need to fix quantize function --- src/webgl/p5.RendererGL.js | 2 ++ src/webgl/shaders/filters/posterize.frag | 34 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 src/webgl/shaders/filters/posterize.frag diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index 6974bfeb09..1c5e9cc2fb 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -89,6 +89,8 @@ const filterShaderFrags = { readFileSync(join(__dirname, '/shaders/filters/dilate.frag'), 'utf-8'), [constants.BLUR]: readFileSync(join(__dirname, '/shaders/filters/blur.frag'), 'utf-8'), + [constants.POSTERIZE]: + readFileSync(join(__dirname, '/shaders/filters/posterize.frag'), 'utf-8') }; const filterShaderVert = readFileSync(join(__dirname, '/shaders/filters/default.vert'), 'utf-8'); diff --git a/src/webgl/shaders/filters/posterize.frag b/src/webgl/shaders/filters/posterize.frag new file mode 100644 index 0000000000..6ed702fec9 --- /dev/null +++ b/src/webgl/shaders/filters/posterize.frag @@ -0,0 +1,34 @@ +// Limit color space for a stylized cartoon / poster effect + +precision highp float; + +varying vec2 vTexCoord; + +uniform sampler2D tex0; +uniform float filterParameter; + +vec3 round(vec3 vals) { + // eg. round(5.5) -> 6.0 + return floor(vals + 0.5); +} + +vec3 quantize(vec3 color, float n) { + // restrict values to N options/bins + // and round each channel to nearest option + // + // eg. when N = 5, options = 0.0, 0.25, 0.50, 0.75, 1.0 + // then posterize (0.1, 0.7, 0.9) -> (0.0, 0.75, 1.0) + + color = color * n; + color = round(color); + color = color / (n - 1.0); + return color; +} + +void main() { + vec4 color = texture2D(tex0, vTexCoord); + + vec3 restrictedColor = quantize(color.rgb, filterParameter); + + gl_FragColor = vec4(restrictedColor.rgb, color.a); +} From 586e933c038b0014a34e13d71c118357adeac00a Mon Sep 17 00:00:00 2001 From: wong-justin Date: Thu, 10 Aug 2023 23:27:15 -0400 Subject: [PATCH 15/54] fix filterParameter by moving setUniform after pg.shader() call --- src/webgl/p5.RendererGL.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index 1c5e9cc2fb..cee002436c 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -986,15 +986,15 @@ p5.RendererGL = class RendererGL extends p5.Renderer { let pg = this.filterGraphicsLayer; // use internal shader for filter constants BLUR, INVERT, etc + let filterParameter = undefined; if (typeof args[0] === 'string') { let operation = args[0]; - let value = args[1]; + filterParameter = args[1]; this.filterShader = new p5.Shader( pg._renderer, filterShaderVert, filterShaderFrags[operation] ); - this.filterShader.setUniform('filterParameter', value); } // use custom user-supplied shader else { @@ -1018,6 +1018,9 @@ p5.RendererGL = class RendererGL extends p5.Renderer { pg.shader(this.filterShader); this.filterShader.setUniform('tex0', this); this.filterShader.setUniform('texelSize', [1.0/this.width, 1.0/this.height]); + // filterParameter only used for POSTERIZE, BLUR, and THRESHOLD + // but shouldn't hurt to always set + this.filterShader.setUniform('filterParameter', filterParameter); pg.rect(0,0,this.width,this.height); // draw pg contents onto main renderer From 1e02d7fd408763b62bb1387c4e9c9f042f7b0f5a Mon Sep 17 00:00:00 2001 From: wong-justin Date: Thu, 10 Aug 2023 23:39:08 -0400 Subject: [PATCH 16/54] adjust some examples for filter() docs - a new one showing useWebGL parameter - simplify shader example, using createFilterShader and bricks.jpg --- src/image/pixels.js | 67 +++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/src/image/pixels.js b/src/image/pixels.js index 50dbabdacd..ba4a2856a8 100644 --- a/src/image/pixels.js +++ b/src/image/pixels.js @@ -468,39 +468,52 @@ p5.prototype._copyHelper = ( * *
* - * createCanvas(100, 100, WEBGL); - * let myShader = createShader( - * `attribute vec3 aPosition; - * attribute vec2 aTexCoord; + * let img; + * function preload() { + * img = loadImage('assets/bricks.jpg'); + * } + * function setup() { + * image(img, 0, 0); + * filter(BLUR, 3, useWebGL=false); + * } + * + *
* - * varying vec2 vTexCoord; + *
+ * + * let img, s; + * function preload() { + * img = loadImage('assets/bricks.jpg'); + * } + * function setup() { + * let fragSrc = `precision highp float; * - * void main() { - * vTexCoord = aTexCoord; - * vec4 positionVec4 = vec4(aPosition, 1.0); - * positionVec4.xy = positionVec4.xy * 2.0 - 1.0; - * gl_Position = positionVec4; - * }`, - * `precision mediump float; - * varying mediump vec2 vTexCoord; + * // x,y coordinates, given from the vertex shader + * varying vec2 vTexCoord; * + * // the canvas contents, given from filter() * uniform sampler2D tex0; - * - * float luma(vec3 color) { - * return dot(color, vec3(0.299, 0.587, 0.114)); - * } + * // a custom variable from the sketch + * uniform float darkness; * * void main() { - * vec2 uv = vTexCoord; - * uv.y = 1.0 - uv.y; - * vec4 sampledColor = texture2D(tex0, uv); - * float gray = luma(sampledColor.rgb); - * gl_FragColor = vec4(gray, gray, gray, 1); - * }` - * ); - * background('RED'); - * filter(myShader); - * describe('a canvas becomes gray after being filtered by shader'); + * // get the color at current pixel + * vec4 color = texture2D(tex0, vTexCoord); + * // set the output color + * color.b = 1.0; + * color *= darkness; + * gl_FragColor = vec4(color.rgb, 1.0); + * }`; + * + * createCanvas(100, 100, WEBGL); + * s = createFilterShader(fragSrc); + * } + * function draw() { + * image(img, -50, -50); + * s.setUniform('darkness', 0.5); + * filter(s); + * describe('a image of bricks tinted dark blue'); + * } * *
* From e0deeebcce0363b1044392df89b617332140d6fe Mon Sep 17 00:00:00 2001 From: wong-justin Date: Fri, 11 Aug 2023 00:04:38 -0400 Subject: [PATCH 17/54] adjust wording for filter() docs - explained why webgl is involved - explained the new parameter - balanced / removed portions considering overlap with createFilterShader() docs --- src/image/pixels.js | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/image/pixels.js b/src/image/pixels.js index ba4a2856a8..f8672cd4e6 100644 --- a/src/image/pixels.js +++ b/src/image/pixels.js @@ -340,15 +340,14 @@ p5.prototype._copyHelper = ( * * --- * - * In WEBGL mode, `filter()` can also accept a shader. The fragment shader - * is given a `uniform sampler2D` named `tex0` that contains the current - * state of the canvas. For more information on using shaders, check - * - * the introduction to shaders tutorial. - * - * See also a selection of shader examples by Adam Ferriss - * that contains many similar filter effects. + * These filter options use WebGL in the background by default (they're faster that way). + * To opt out of this in P2D mode, add a `false` parameter when calling `filter()`. + * This may be useful to keep computation off the GPU or to work around a lack of WebGL support. + * + * On a renderer in WEBGL mode, `filter()` can also accept a user-provided shader. + * The shader will be applied to the canvas and not to any geometries. + * For more information, see createFilterShader(). + * * * @method filter * @param {Constant} filterType either THRESHOLD, GRAY, OPAQUE, INVERT, From 5dc1056514d64c74258020e01a392f53665b94f4 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Fri, 11 Aug 2023 12:42:14 -0400 Subject: [PATCH 18/54] fix POSTERIZE to match old output change round() to floor() --- src/webgl/shaders/filters/posterize.frag | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/webgl/shaders/filters/posterize.frag b/src/webgl/shaders/filters/posterize.frag index 6ed702fec9..2375b8a700 100644 --- a/src/webgl/shaders/filters/posterize.frag +++ b/src/webgl/shaders/filters/posterize.frag @@ -7,20 +7,15 @@ varying vec2 vTexCoord; uniform sampler2D tex0; uniform float filterParameter; -vec3 round(vec3 vals) { - // eg. round(5.5) -> 6.0 - return floor(vals + 0.5); -} - vec3 quantize(vec3 color, float n) { // restrict values to N options/bins - // and round each channel to nearest option + // and floor each channel to nearest value // - // eg. when N = 5, options = 0.0, 0.25, 0.50, 0.75, 1.0 - // then posterize (0.1, 0.7, 0.9) -> (0.0, 0.75, 1.0) + // eg. when N = 5, values = 0.0, 0.25, 0.50, 0.75, 1.0 + // then quantize (0.1, 0.7, 0.9) -> (0.0, 0.5, 1.0) color = color * n; - color = round(color); + color = floor(color); color = color / (n - 1.0); return color; } From 72f789531db6d8118820a4592c2a38a5ca2f715b Mon Sep 17 00:00:00 2001 From: wong-justin Date: Fri, 11 Aug 2023 12:47:50 -0400 Subject: [PATCH 19/54] simpler sketch for manual testing --- .../webgl/filter/sketch.js | 49 ++++--------------- 1 file changed, 10 insertions(+), 39 deletions(-) diff --git a/test/manual-test-examples/webgl/filter/sketch.js b/test/manual-test-examples/webgl/filter/sketch.js index a0d4d54a83..41105e8572 100644 --- a/test/manual-test-examples/webgl/filter/sketch.js +++ b/test/manual-test-examples/webgl/filter/sketch.js @@ -1,44 +1,15 @@ -function setup() { - createCanvas(100, 100, WEBGL); - - let s = createShader(vert, frag); - - // check to see if frag shader changes color as intended - // and that vertex shader preserves position, orientation, scale - background('RED'); - circle(10,25,30); +let img; - filter(s); - - // and that there's no side effects after filter() - circle(-35,-35,30); +function preload() { + img = loadImage('../../../../docs/reference/assets/bricks.jpg'); } -vert = `attribute vec3 aPosition; -attribute vec2 aTexCoord; - -varying vec2 vTexCoord; - -void main() { - vTexCoord = aTexCoord; - vec4 positionVec4 = vec4(aPosition, 1.0); - positionVec4.xy = positionVec4.xy * 2.0 - 1.0; - gl_Position = positionVec4; -}`; - -frag = `precision mediump float; -varying mediump vec2 vTexCoord; - -uniform sampler2D tex0; - -float luma(vec3 color) { - return dot(color, vec3(0.299, 0.587, 0.114)); +function setup() { + createCanvas(200, 200, WEBGL); + img.resize(200, 200); } -void main() { - vec2 uv = vTexCoord; - uv.y = 1.0 - uv.y; - vec4 sampledColor = texture2D(tex0, uv); - float gray = luma(sampledColor.rgb); - gl_FragColor = vec4(gray, gray, gray, 1); -}`; +function draw() { + image(img, -width/2, -height/2); + filter(POSTERIZE, 4); +} From 10e0cac3bdad261eea746210ea5f107fe7626d4a Mon Sep 17 00:00:00 2001 From: wong-justin Date: Fri, 11 Aug 2023 14:09:41 -0400 Subject: [PATCH 20/54] wip, start using shader filters in background of P2D --- src/image/pixels.js | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/src/image/pixels.js b/src/image/pixels.js index f8672cd4e6..7f26580a3f 100644 --- a/src/image/pixels.js +++ b/src/image/pixels.js @@ -8,6 +8,7 @@ import p5 from '../core/main'; import Filters from './filters'; import '../color/p5.Color'; +import * as constants from '../core/constants'; /** * createFilterShader(). * * @@ -488,21 +487,15 @@ p5.prototype._copyHelper = ( * function setup() { * let fragSrc = `precision highp float; * - * // x,y coordinates, given from the vertex shader - * varying vec2 vTexCoord; - * - * // the canvas contents, given from filter() - * uniform sampler2D tex0; - * // a custom variable from the sketch - * uniform float darkness; + * varying vec2 vTexCoord; // x,y coordinates + * uniform sampler2D tex0; // the canvas contents * * void main() { * // get the color at current pixel * vec4 color = texture2D(tex0, vTexCoord); * // set the output color * color.b = 1.0; - * color *= darkness; - * gl_FragColor = vec4(color.rgb, 1.0); + * gl_FragColor = vec4(color); * }`; * * createCanvas(100, 100, WEBGL); @@ -510,9 +503,8 @@ p5.prototype._copyHelper = ( * } * function draw() { * image(img, -50, -50); - * s.setUniform('darkness', 0.5); * filter(s); - * describe('a image of bricks tinted dark blue'); + * describe('a image of bricks tinted blue'); * } * *
From 41cf4ba6297dcf6d1e751bf56f65bce62826411f Mon Sep 17 00:00:00 2001 From: wong-justin Date: Fri, 18 Aug 2023 11:38:46 -0400 Subject: [PATCH 31/54] document new uniforms, and slight rephrasing in createFilterShader --- src/webgl/material.js | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/webgl/material.js b/src/webgl/material.js index 32336c757a..208fa3811c 100644 --- a/src/webgl/material.js +++ b/src/webgl/material.js @@ -187,15 +187,13 @@ p5.prototype.createShader = function(vertSrc, fragSrc) { * Creates a new p5.Shader using only a fragment shader, as a convenience method for creating image effects. * It's like createShader() but with a default vertex shader included. * - * createFilterShader() is intended to be used along with filter() for filtering the entire contents of a canvas in WebGL mode. + * createFilterShader() is intended to be used along with filter() for filtering the contents of a canvas in WebGL mode. + * A filter shader will not be applied to any geometries. * - * Note: - * - The fragment shader is provided with a single texture input uniform called `tex0`. - * This is created specificially for filter shaders to access the canvas contents. - * - * - A filter shader will not apply to a 3D geometry. - * - * - Shaders can only be used in `WEBGL` mode. + * The fragment shader receives some uniforms: + * - `sampler2D tex0`, which contains the canvas contents as a texture + * - `vec2 canvasSize`, which is the width and height of the canvas + * - `vec2 texelSize`, which is the size of a pixel (`1.0/width`, `1.0/height`) * * For more info about filters and shaders, see Adam Ferriss' repo of shader examples * or the introduction to shaders page. @@ -235,7 +233,10 @@ p5.prototype.createShader = function(vertSrc, fragSrc) { * * // the canvas contents, given from filter() * uniform sampler2D tex0; - * // a custom variable from the sketch + * // other useful information from the canvas + * uniform vec2 texelSize; + * uniform vec2 canvasSize; + * // a custom variable from this sketch * uniform float darkness; * * void main() { From 6c2e9cedc698c7ec87eec11d299ebf199cb8d32f Mon Sep 17 00:00:00 2001 From: wong-justin Date: Fri, 18 Aug 2023 16:20:33 -0400 Subject: [PATCH 32/54] add some tests --- test/unit/webgl/p5.RendererGL.js | 65 ++++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 7 deletions(-) diff --git a/test/unit/webgl/p5.RendererGL.js b/test/unit/webgl/p5.RendererGL.js index 4ff0648141..f59a0b8b8c 100644 --- a/test/unit/webgl/p5.RendererGL.js +++ b/test/unit/webgl/p5.RendererGL.js @@ -122,9 +122,9 @@ suite('p5.RendererGL', function() { setup(function() { vert = `attribute vec3 aPosition; attribute vec2 aTexCoord; - + varying vec2 vTexCoord; - + void main() { vTexCoord = aTexCoord; vec4 positionVec4 = vec4(aPosition, 1.0); @@ -134,13 +134,13 @@ suite('p5.RendererGL', function() { frag = `precision highp float; varying vec2 vTexCoord; - + uniform sampler2D tex0; - + float luma(vec3 color) { return dot(color, vec3(0.299, 0.587, 0.114)); } - + void main() { vec2 uv = vTexCoord; uv.y = 1.0 - uv.y; @@ -148,9 +148,18 @@ suite('p5.RendererGL', function() { float gray = luma(sampledColor.rgb); gl_FragColor = vec4(gray, gray, gray, 1); }`; - }); - teardown(function() { + notAllBlack = pixels => { + // black canvas could be an indicator of failed shader logic + for (let i = 0; i < pixels.length; i++) { + if (pixels[i] !== 255 || + pixels[i+1] !== 255 || + pixels[i+2] !== 255) { + return true; + } + } + return false; + }; }); test('filter accepts correct params', function() { @@ -269,6 +278,48 @@ suite('p5.RendererGL', function() { let p2 = pg.pixels; assert.notDeepEqual(p1, p2); }); + + test('POSTERIZE, BLUR, THRESHOLD work without supplied param', function() { + let testDefaultParams = () => { + myp5.createCanvas(3,3, myp5.WEBGL); + myp5.filter(myp5.POSTERIZE); + myp5.filter(myp5.BLUR); + myp5.filter(myp5.THRESHOLD); + }; + assert.doesNotThrow(testDefaultParams, 'this should not throw'); + }); + + test('filter() uses WEBGL implementation behind main P2D canvas', function() { + let renderer = myp5.createCanvas(3,3); + myp5.filter(myp5.BLUR); + assert.isDefined(renderer._pInst.filterGraphicsLayer); + }); + + test('filter() can opt out of WEBGL implementation', function() { + let renderer = myp5.createCanvas(3,3); + myp5.filter(myp5.BLUR, useWebGL=false); + assert.isUndefined(renderer._pInst.filterGraphicsLayer); + }); + + test('filters make changes to canvas', function() { + myp5.createCanvas(20,20); + myp5.circle(10,10,12); + let operations = [ + myp5.BLUR, + myp5.THRESHOLD, + myp5.POSTERIZE, + myp5.INVERT, + myp5.DILATE, + myp5.ERODE, + myp5.GRAY, + myp5.OPAQUE + ]; + for (let operation of operations) { + myp5.filter(operation); + myp5.loadPixels(); + assert(notAllBlack(myp5.pixels)); + } + }); }); test('contours match 2D', function() { From 9dc8887d78325de24755562facf40c3fbf2674d6 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Fri, 18 Aug 2023 16:54:18 -0400 Subject: [PATCH 33/54] store shaders for filters BLUR, INVERT, etc instead of creating on each frame --- src/webgl/p5.RendererGL.js | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index 8deff57886..9f19f3aea6 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -610,6 +610,7 @@ p5.RendererGL = class RendererGL extends p5.Renderer { // for post processing step this.filterShader = undefined; this.filterGraphicsLayer = undefined; + this.defaultFilterShaders = {}; this.textureMode = constants.IMAGE; // default wrap settings @@ -1002,11 +1003,17 @@ p5.RendererGL = class RendererGL extends p5.Renderer { let useDefaultParam = operation in defaults && args[1] === undefined; filterParameter = useDefaultParam ? defaults[operation] : args[1]; - this.filterShader = new p5.Shader( - pg._renderer, - filterShaderVert, - filterShaderFrags[operation] - ); + // Create and store shader for constants once on initial filter call. + // Need to store multiple in case user calls different filters, + // eg. filter(BLUR) then filter(GRAY) + if ( !(operation in this.defaultFilterShaders) ) { + this.defaultFilterShaders[operation] = new p5.Shader( + pg._renderer, + filterShaderVert, + filterShaderFrags[operation] + ); + } + this.filterShader = this.defaultFilterShaders[operation]; } // use custom user-supplied shader else { From b1a76ec3e98f3a1a8ba5325031e861127bdb8fa7 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Sun, 20 Aug 2023 11:19:50 -0400 Subject: [PATCH 34/54] more tests --- test/unit/webgl/p5.RendererGL.js | 35 +++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/test/unit/webgl/p5.RendererGL.js b/test/unit/webgl/p5.RendererGL.js index f59a0b8b8c..f68e9806d4 100644 --- a/test/unit/webgl/p5.RendererGL.js +++ b/test/unit/webgl/p5.RendererGL.js @@ -149,12 +149,13 @@ suite('p5.RendererGL', function() { gl_FragColor = vec4(gray, gray, gray, 1); }`; - notAllBlack = pixels => { - // black canvas could be an indicator of failed shader logic + notAllBlack = (pixels, invert) => { + // black/white canvas could be an indicator of failed shader logic + let val = invert ? 255 : 0; for (let i = 0; i < pixels.length; i++) { - if (pixels[i] !== 255 || - pixels[i+1] !== 255 || - pixels[i+2] !== 255) { + if (pixels[i] !== val || + pixels[i+1] !== val || + pixels[i+2] !== val) { return true; } } @@ -318,8 +319,32 @@ suite('p5.RendererGL', function() { myp5.filter(operation); myp5.loadPixels(); assert(notAllBlack(myp5.pixels)); + assert(notAllBlack(myp5.pixels, invert=true)); } }); + + test('feedback effects can be prevented (ie. clear() works)', function() { + myp5.createCanvas(20,20); + let drawAndFilter = () => { + myp5.circle(5,5,8); + myp5.filter(myp5.BLUR); + }; + let getPixels = () => { + myp5.loadPixels(); + return myp5.pixels.slice(); + }; + + drawAndFilter(); + let p1 = getPixels(); + + for (let i = 0; i < 5; i++) { + myp5.clear(); + drawAndFilter(); + } + let p2 = getPixels(); + + assert.deepEqual(p1, p2); + }); }); test('contours match 2D', function() { From bafee004813b2aa2a44fbe805d3b4b1fc9ca4157 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Sun, 20 Aug 2023 11:20:58 -0400 Subject: [PATCH 35/54] fix feedback effect clear hidden renderers so effects don't accumulate user can still allow feedback effects by not clear()ing main --- src/image/pixels.js | 1 + src/webgl/p5.RendererGL.js | 1 + 2 files changed, 2 insertions(+) diff --git a/src/image/pixels.js b/src/image/pixels.js index 4501a73912..9d13c4aecd 100644 --- a/src/image/pixels.js +++ b/src/image/pixels.js @@ -594,6 +594,7 @@ p5.prototype.filter = function(...args) { // copy secondary webgl renderer back to original p2d canvas this._renderer._pInst.image(this.filterGraphicsLayer, 0, 0); + this.filterGraphicsLayer.clear(); } }; diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index 9f19f3aea6..56b071ee75 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -1034,6 +1034,7 @@ p5.RendererGL = class RendererGL extends p5.Renderer { } // apply shader to pg + pg.clear(); // prevent undesirable feedback effects accumulating secretly pg.shader(this.filterShader); this.filterShader.setUniform('tex0', this); this.filterShader.setUniform('texelSize', [1.0/this.width, 1.0/this.height]); From 3c4afc5fbc6cb50de26d77ffc00d8aaba7380e70 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Sun, 20 Aug 2023 21:38:09 -0400 Subject: [PATCH 36/54] clarify comments --- src/image/pixels.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/image/pixels.js b/src/image/pixels.js index 9d13c4aecd..101fe373f0 100644 --- a/src/image/pixels.js +++ b/src/image/pixels.js @@ -560,7 +560,7 @@ p5.prototype.filter = function(...args) { // when this is P2D renderer, create/use hidden webgl renderer else { - // create/use hidden webgl renderer + // create hidden webgl renderer if it doesn't exist if (!this.filterGraphicsLayer) { // the real _pInst is buried when this is a secondary p5.Graphics const pInst = @@ -594,7 +594,7 @@ p5.prototype.filter = function(...args) { // copy secondary webgl renderer back to original p2d canvas this._renderer._pInst.image(this.filterGraphicsLayer, 0, 0); - this.filterGraphicsLayer.clear(); + this.filterGraphicsLayer.clear(); // prevent feedback effects on p2d canvas } }; From 02d187bad54c174c9ba1e7b7602e2cd8595378d2 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Mon, 21 Aug 2023 10:11:00 -0400 Subject: [PATCH 37/54] support webgl2 fragment shader in createFilterShader() --- src/webgl/material.js | 22 ++++++++++++++++++++-- test/unit/webgl/p5.RendererGL.js | 18 ++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/webgl/material.js b/src/webgl/material.js index 208fa3811c..2c2bd9b2c2 100644 --- a/src/webgl/material.js +++ b/src/webgl/material.js @@ -263,7 +263,7 @@ p5.prototype.createShader = function(vertSrc, fragSrc) { p5.prototype.createFilterShader = function(fragSrc) { this._assert3d('createFilterShader'); p5._validateParameters('createFilterShader', arguments); - let defaultVertSrc = ` + let defaultVertV1 = ` attribute vec3 aPosition; // texcoords only come from p5 to vertex shader // so pass texcoords on to the fragment shader in a varying variable @@ -282,7 +282,25 @@ p5.prototype.createFilterShader = function(fragSrc) { gl_Position = positionVec4; } `; - return new p5.Shader(this._renderer, defaultVertSrc, fragSrc); + let defaultVertV2 = `#version 300 es + in vec3 aPosition; + in vec2 aTexCoord; + out vec2 vTexCoord; + + void main() { + // transferring texcoords for the frag shader + vTexCoord = aTexCoord; + + // copy position with a fourth coordinate for projection (1.0 is normal) + vec4 positionVec4 = vec4(aPosition, 1.0); + // scale by two and center to achieve correct positioning + positionVec4.xy = positionVec4.xy * 2.0 - 1.0; + + gl_Position = positionVec4; + } + `; + let vertSrc = fragSrc.includes('#version 300 es') ? defaultVertV2 : defaultVertV1; + return new p5.Shader(this._renderer, vertSrc, fragSrc); }; /** diff --git a/test/unit/webgl/p5.RendererGL.js b/test/unit/webgl/p5.RendererGL.js index f68e9806d4..058695121a 100644 --- a/test/unit/webgl/p5.RendererGL.js +++ b/test/unit/webgl/p5.RendererGL.js @@ -345,6 +345,24 @@ suite('p5.RendererGL', function() { assert.deepEqual(p1, p2); }); + + test('createFilterShader() accepts shader fragments in webgl version 2', function() { + myp5.createCanvas(5, 5, myp5.WEBGL); + let s = myp5.createFilterShader(`#version 300 es + precision highp float; + in vec2 vTexCoord; + out vec4 outColor; + + uniform sampler2D tex0; + + void main() { + vec4 sampledColor = texture(tex0, vTexCoord); + sampledColor.b = 1.0; + outColor = sampledColor; + } + `); + myp5.filter(s); + }); }); test('contours match 2D', function() { From b5d533cdebb0f5fb7867ed46dd3268c6c551f43c Mon Sep 17 00:00:00 2001 From: wong-justin Date: Mon, 21 Aug 2023 17:02:55 -0400 Subject: [PATCH 38/54] fix whitespace --- test/unit/webgl/p5.RendererGL.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/webgl/p5.RendererGL.js b/test/unit/webgl/p5.RendererGL.js index 058695121a..435027ea2f 100644 --- a/test/unit/webgl/p5.RendererGL.js +++ b/test/unit/webgl/p5.RendererGL.js @@ -347,7 +347,7 @@ suite('p5.RendererGL', function() { }); test('createFilterShader() accepts shader fragments in webgl version 2', function() { - myp5.createCanvas(5, 5, myp5.WEBGL); + myp5.createCanvas(5, 5, myp5.WEBGL); let s = myp5.createFilterShader(`#version 300 es precision highp float; in vec2 vTexCoord; From ec41c1f0ee9c1add50284f4e76baf83140b41fa6 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Tue, 22 Aug 2023 12:36:19 -0400 Subject: [PATCH 39/54] replace single-pass blur with two-pass --- src/webgl/p5.RendererGL.js | 53 +++++++++++++++++++++++------ src/webgl/shaders/filters/blur.frag | 48 ++++++-------------------- 2 files changed, 53 insertions(+), 48 deletions(-) diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index 56b071ee75..48ec993b0a 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -993,10 +993,11 @@ p5.RendererGL = class RendererGL extends p5.Renderer { // use internal shader for filter constants BLUR, INVERT, etc let filterParameter = undefined; + let operation = undefined; if (typeof args[0] === 'string') { - let operation = args[0]; + operation = args[0]; let defaults = { - [constants.BLUR]: 4, + [constants.BLUR]: 3, [constants.POSTERIZE]: 4, [constants.THRESHOLD]: 0.5 }; @@ -1033,16 +1034,46 @@ p5.RendererGL = class RendererGL extends p5.Renderer { } } - // apply shader to pg pg.clear(); // prevent undesirable feedback effects accumulating secretly - pg.shader(this.filterShader); - this.filterShader.setUniform('tex0', this); - this.filterShader.setUniform('texelSize', [1.0/this.width, 1.0/this.height]); - this.filterShader.setUniform('canvasSize', [this.width, this.height]); - // filterParameter only used for POSTERIZE, BLUR, and THRESHOLD - // but shouldn't hurt to always set - this.filterShader.setUniform('filterParameter', filterParameter); - pg.rect(0,0,this.width,this.height); + + // apply blur shader with multiple passes + if (operation === constants.BLUR) { + + // pg is the accumulator. initialize with contents of main renderer (this) + pg.copy( + this, + 0, 0, this.width, this.height, + -this.width/2, -this.height/2, this.width, this.height + ); + // how much to blur, given by user + let steps = filterParameter; + + for (let i = 0; i < steps; i++) { + // first pass averaging horizontal neighbors + pg.shader(this.filterShader); + this.filterShader.setUniform('texelSize', [1/this.width, 1/this.height]); + this.filterShader.setUniform('tex0', pg); + this.filterShader.setUniform('direction', [2, 0]); // 2 is a decent + pg.rect(0,0,this.width,this.height); // default spread + // another pass, this time vertically + pg.shader(this.filterShader); + // this.filterShader.setUniform('texelSize', [1/this.width, 1/this.height]); + this.filterShader.setUniform('tex0', pg); + this.filterShader.setUniform('direction', [0, 2]); + pg.rect(0,0,this.width,this.height); + } + } + // every other shader gets single pass onto pg + else { + pg.shader(this.filterShader); + this.filterShader.setUniform('tex0', this); + this.filterShader.setUniform('texelSize', [1/this.width, 1/this.height]); + this.filterShader.setUniform('canvasSize', [this.width, this.height]); + // filterParameter uniform only used for POSTERIZE, and THRESHOLD + // but shouldn't hurt to always set + this.filterShader.setUniform('filterParameter', filterParameter); + pg.rect(0,0,this.width,this.height); + } // draw pg contents onto main renderer this._pInst.push(); diff --git a/src/webgl/shaders/filters/blur.frag b/src/webgl/shaders/filters/blur.frag index ca2e13a910..c2e13aa87f 100644 --- a/src/webgl/shaders/filters/blur.frag +++ b/src/webgl/shaders/filters/blur.frag @@ -1,47 +1,21 @@ -// Single-pass blur filter, taken from Adam Ferriss' repo of shader examples: +// Two-pass blur filter, unweighted kernel. +// See also a similar blur at Adam Ferriss' repo of shader examples: // https://github.com/aferriss/p5jsShaderExamples/blob/gh-pages/4_image-effects/4-9_single-pass-blur/effect.frag precision highp float; -// lets grab texcoords just for fun -varying vec2 vTexCoord; - -// our texture coming from p5 uniform sampler2D tex0; +varying vec2 vTexCoord; +uniform vec2 direction; uniform vec2 texelSize; -uniform float filterParameter; - -void main() { - - vec2 uv = vTexCoord; - - // a single pass blur works by sampling all the neighbor pixels and averaging them up - // this is somewhat inefficient because we have to sample the texture 9 times -- texture2D calls are slow :( - // check out the two-pass-blur example for a better blur approach - // get the webcam as a vec4 using texture2D - - // spread controls how far away from the center we should pull a sample from - // you will start to see artifacts if you crank this up too high - float spread = max(0.0, filterParameter); - - // create our offset variable by multiplying the size of a texel with spread - vec2 offset = texelSize * spread; - - // get all the neighbor pixels! - vec4 tex = texture2D(tex0, uv); // middle middle -- the actual texel / pixel - tex += texture2D(tex0, uv + vec2(-offset.x, -offset.y)); // top left - tex += texture2D(tex0, uv + vec2(0.0, -offset.y)); // top middle - tex += texture2D(tex0, uv + vec2(offset.x, -offset.y)); // top right - - tex += texture2D(tex0, uv + vec2(-offset.x, 0.0)); //middle left - tex += texture2D(tex0, uv + vec2(offset.x, 0.0)); //middle right - - tex += texture2D(tex0, uv + vec2(-offset.x, offset.y)); // bottom left - tex += texture2D(tex0, uv + vec2(0.0, offset.y)); // bottom middle - tex += texture2D(tex0, uv + vec2(offset.x, offset.y)); // bottom right - // we added 9 textures together, so we will divide by 9 to average them out and move the values back into a 0 - 1 range - tex /= 9.0; +void main(){ + + vec4 tex = texture2D(tex0, vTexCoord); + tex += texture2D(tex0, vTexCoord - texelSize * direction); + tex += texture2D(tex0, vTexCoord + texelSize * direction); + + tex /= 3.0; gl_FragColor = tex; } From 8ccfe59c2cf5d5392f0cb3f9fe2c2ff541836bdb Mon Sep 17 00:00:00 2001 From: wong-justin Date: Wed, 23 Aug 2023 13:39:27 -0400 Subject: [PATCH 40/54] adjust initial blur two-pass by using uniform(tex0, this) --- src/webgl/p5.RendererGL.js | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index 48ec993b0a..f0c877d304 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -1039,12 +1039,21 @@ p5.RendererGL = class RendererGL extends p5.Renderer { // apply blur shader with multiple passes if (operation === constants.BLUR) { - // pg is the accumulator. initialize with contents of main renderer (this) - pg.copy( - this, - 0, 0, this.width, this.height, - -this.width/2, -this.height/2, this.width, this.height - ); + // initial binding and setup + pg.shader(this.filterShader); + this.filterShader.setUniform('texelSize', [1/this.width, 1/this.height]); + + // do initial horizontal and vertical pass, + // starting with parent renderer as tex0 uniform + this.filterShader.setUniform('tex0', this); // vertically flips first + this.filterShader.setUniform('flipped', false); // so undo it + this.filterShader.setUniform('direction', [2, 0]); + pg.rect(0,0,this.width,this.height); + this.filterShader.setUniform('tex0', pg); // all other passes are unflipped + this.filterShader.setUniform('flipped', true); + this.filterShader.setUniform('direction', [0, 2]); // 2 is a decent + pg.rect(0,0,this.width,this.height); // default spread + // how much to blur, given by user let steps = filterParameter; From 347401b9aa201273af9567a38b1fa83be3cd0343 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Wed, 23 Aug 2023 13:40:47 -0400 Subject: [PATCH 41/54] add variable to reverse flipping effect of previous commit --- src/webgl/shaders/filters/blur.frag | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/webgl/shaders/filters/blur.frag b/src/webgl/shaders/filters/blur.frag index c2e13aa87f..2706331cbc 100644 --- a/src/webgl/shaders/filters/blur.frag +++ b/src/webgl/shaders/filters/blur.frag @@ -8,12 +8,18 @@ uniform sampler2D tex0; varying vec2 vTexCoord; uniform vec2 direction; uniform vec2 texelSize; +uniform float flipped; void main(){ + + vec2 uv = vTexCoord; + if (flipped == 1.0) { + uv.y = 1.0 - uv.y; + } - vec4 tex = texture2D(tex0, vTexCoord); - tex += texture2D(tex0, vTexCoord - texelSize * direction); - tex += texture2D(tex0, vTexCoord + texelSize * direction); + vec4 tex = texture2D(tex0, uv); + tex += texture2D(tex0, uv - texelSize * direction); + tex += texture2D(tex0, uv + texelSize * direction); tex /= 3.0; From 39def400a12ffc79450fd656b0cd58e355e667d8 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Wed, 23 Aug 2023 13:41:13 -0400 Subject: [PATCH 42/54] remaining steps for blur, including removing extra shader calls --- src/webgl/p5.RendererGL.js | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index f0c877d304..7dfcd43727 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -1054,25 +1054,19 @@ p5.RendererGL = class RendererGL extends p5.Renderer { this.filterShader.setUniform('direction', [0, 2]); // 2 is a decent pg.rect(0,0,this.width,this.height); // default spread - // how much to blur, given by user - let steps = filterParameter; - - for (let i = 0; i < steps; i++) { - // first pass averaging horizontal neighbors - pg.shader(this.filterShader); - this.filterShader.setUniform('texelSize', [1/this.width, 1/this.height]); + // perform remaining steps, accumulating on pg + let steps = filterParameter; // how much to blur, given by user + for (let i = 1; i < steps; i++) { this.filterShader.setUniform('tex0', pg); - this.filterShader.setUniform('direction', [2, 0]); // 2 is a decent - pg.rect(0,0,this.width,this.height); // default spread - // another pass, this time vertically - pg.shader(this.filterShader); - // this.filterShader.setUniform('texelSize', [1/this.width, 1/this.height]); + this.filterShader.setUniform('direction', [2, 0]); + pg.rect(0,0,this.width,this.height); + this.filterShader.setUniform('tex0', pg); this.filterShader.setUniform('direction', [0, 2]); pg.rect(0,0,this.width,this.height); } } - // every other shader gets single pass onto pg + // every other non-blur shader uses single pass else { pg.shader(this.filterShader); this.filterShader.setUniform('tex0', this); From 0e6a95b6dfb8df92626166b007e00b5c61812efd Mon Sep 17 00:00:00 2001 From: wong-justin Date: Mon, 28 Aug 2023 08:47:00 -0400 Subject: [PATCH 43/54] revise default vertex shader to deal with depth issue #6367 --- src/webgl/material.js | 16 ++++++++++------ src/webgl/p5.RendererGL.js | 1 - 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/webgl/material.js b/src/webgl/material.js index 2c2bd9b2c2..857519d71e 100644 --- a/src/webgl/material.js +++ b/src/webgl/material.js @@ -264,6 +264,9 @@ p5.prototype.createFilterShader = function(fragSrc) { this._assert3d('createFilterShader'); p5._validateParameters('createFilterShader', arguments); let defaultVertV1 = ` + uniform mat4 uModelViewMatrix; + uniform mat4 uProjectionMatrix; + attribute vec3 aPosition; // texcoords only come from p5 to vertex shader // so pass texcoords on to the fragment shader in a varying variable @@ -276,13 +279,15 @@ p5.prototype.createFilterShader = function(fragSrc) { // copy position with a fourth coordinate for projection (1.0 is normal) vec4 positionVec4 = vec4(aPosition, 1.0); - // scale by two and center to achieve correct positioning - positionVec4.xy = positionVec4.xy * 2.0 - 1.0; - gl_Position = positionVec4; + // project to 3D space + gl_Position = uProjectionMatrix * uModelViewMatrix * positionVec4; } `; let defaultVertV2 = `#version 300 es + uniform mat4 uModelViewMatrix; + uniform mat4 uProjectionMatrix; + in vec3 aPosition; in vec2 aTexCoord; out vec2 vTexCoord; @@ -293,10 +298,9 @@ p5.prototype.createFilterShader = function(fragSrc) { // copy position with a fourth coordinate for projection (1.0 is normal) vec4 positionVec4 = vec4(aPosition, 1.0); - // scale by two and center to achieve correct positioning - positionVec4.xy = positionVec4.xy * 2.0 - 1.0; - gl_Position = positionVec4; + // project to 3D space + gl_Position = uProjectionMatrix * uModelViewMatrix * positionVec4; } `; let vertSrc = fragSrc.includes('#version 300 es') ? defaultVertV2 : defaultVertV1; diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index 7dfcd43727..da81dddb1d 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -1081,7 +1081,6 @@ p5.RendererGL = class RendererGL extends p5.Renderer { // draw pg contents onto main renderer this._pInst.push(); this._pInst.noStroke(); // don't draw triangles for plane() geometry - this._pInst.scale(1, -1); // vertically flip output this._pInst.texture(pg); this._pInst.plane(this.width, this.height); this._pInst.pop(); From be091427684ed699d1fdad537d0bb82f55385ec9 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Mon, 28 Aug 2023 10:43:32 -0400 Subject: [PATCH 44/54] fix rect() calls --- src/webgl/p5.RendererGL.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index da81dddb1d..aab44adc5d 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -988,6 +988,8 @@ p5.RendererGL = class RendererGL extends p5.Renderer { constants.WEBGL, pInst ); + // geometries/borders on this layer should always be invisible + this.filterGraphicsLayer.noStroke(); } let pg = this.filterGraphicsLayer; @@ -1075,7 +1077,7 @@ p5.RendererGL = class RendererGL extends p5.Renderer { // filterParameter uniform only used for POSTERIZE, and THRESHOLD // but shouldn't hurt to always set this.filterShader.setUniform('filterParameter', filterParameter); - pg.rect(0,0,this.width,this.height); + pg.rect(-this.width/2, -this.height/2, this.width, this.height); } // draw pg contents onto main renderer From 3e2294582d28c5c96989f57c2591ec3bb0d0c3bd Mon Sep 17 00:00:00 2001 From: wong-justin Date: Mon, 28 Aug 2023 17:28:24 -0400 Subject: [PATCH 45/54] update other default vertex shader as well --- src/webgl/shaders/filters/default.vert | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/webgl/shaders/filters/default.vert b/src/webgl/shaders/filters/default.vert index c5c74680a9..ebe0b2a864 100644 --- a/src/webgl/shaders/filters/default.vert +++ b/src/webgl/shaders/filters/default.vert @@ -1,3 +1,6 @@ +uniform mat4 uModelViewMatrix; +uniform mat4 uProjectionMatrix; + attribute vec3 aPosition; // texcoords only come from p5 to vertex shader // so pass texcoords on to the fragment shader in a varying variable @@ -10,8 +13,7 @@ void main() { // copy position with a fourth coordinate for projection (1.0 is normal) vec4 positionVec4 = vec4(aPosition, 1.0); - // scale by two and center to achieve correct positioning - positionVec4.xy = positionVec4.xy * 2.0 - 1.0; + // positionVec4.y = 1.0 - positionVec4.y; - gl_Position = positionVec4; + gl_Position = uProjectionMatrix * uModelViewMatrix * positionVec4; } From eca2c7698a7052965ce0e1a4b61aea553ab5072e Mon Sep 17 00:00:00 2001 From: wong-justin Date: Mon, 28 Aug 2023 17:34:55 -0400 Subject: [PATCH 46/54] switch to using main and secondary renderers for blur passes instead of only secondary --- src/webgl/p5.RendererGL.js | 68 +++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index aab44adc5d..39a9dc16bc 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -1015,8 +1015,18 @@ p5.RendererGL = class RendererGL extends p5.Renderer { filterShaderVert, filterShaderFrags[operation] ); + + // two-pass blur filter needs another shader attached to main + if (operation === constants.BLUR) { + this.otherBlurShader = new p5.Shader( + this, + filterShaderVert, + filterShaderFrags[constants.BLUR] + ); + } } this.filterShader = this.defaultFilterShaders[operation]; + } // use custom user-supplied shader else { @@ -1041,32 +1051,30 @@ p5.RendererGL = class RendererGL extends p5.Renderer { // apply blur shader with multiple passes if (operation === constants.BLUR) { - // initial binding and setup + // setup + this._pInst.push(); + this._pInst.noStroke(); pg.shader(this.filterShader); + this._pInst.shader(this.otherBlurShader); this.filterShader.setUniform('texelSize', [1/this.width, 1/this.height]); - - // do initial horizontal and vertical pass, - // starting with parent renderer as tex0 uniform - this.filterShader.setUniform('tex0', this); // vertically flips first - this.filterShader.setUniform('flipped', false); // so undo it - this.filterShader.setUniform('direction', [2, 0]); - pg.rect(0,0,this.width,this.height); - this.filterShader.setUniform('tex0', pg); // all other passes are unflipped - this.filterShader.setUniform('flipped', true); - this.filterShader.setUniform('direction', [0, 2]); // 2 is a decent - pg.rect(0,0,this.width,this.height); // default spread - - // perform remaining steps, accumulating on pg - let steps = filterParameter; // how much to blur, given by user - for (let i = 1; i < steps; i++) { - this.filterShader.setUniform('tex0', pg); - this.filterShader.setUniform('direction', [2, 0]); - pg.rect(0,0,this.width,this.height); - - this.filterShader.setUniform('tex0', pg); - this.filterShader.setUniform('direction', [0, 2]); - pg.rect(0,0,this.width,this.height); + this.otherBlurShader.setUniform('texelSize', [1/this.width, 1/this.height]); + + // two-pass blur, repeated more with higher parameter + let steps = filterParameter; + for (let i = 0; i < steps; i++) { + // main contents onto pg + this.filterShader.setUniform('tex0', this); + this.filterShader.setUniform('direction', [1, 0]); // horiz pass + pg.rect(-this.width/2, -this.height/2, this.width, this.height); + + // pg contents onto main + this.otherBlurShader.setUniform('tex0', pg); + this.otherBlurShader.setUniform('direction', [0, 1]); // vert pass + this._pInst.rect( + -this.width/2, -this.height/2, this.width, this.height + ); } + this._pInst.pop(); } // every other non-blur shader uses single pass else { @@ -1078,14 +1086,14 @@ p5.RendererGL = class RendererGL extends p5.Renderer { // but shouldn't hurt to always set this.filterShader.setUniform('filterParameter', filterParameter); pg.rect(-this.width/2, -this.height/2, this.width, this.height); - } - // draw pg contents onto main renderer - this._pInst.push(); - this._pInst.noStroke(); // don't draw triangles for plane() geometry - this._pInst.texture(pg); - this._pInst.plane(this.width, this.height); - this._pInst.pop(); + // draw pg contents onto main renderer + this._pInst.push(); + this._pInst.noStroke(); // don't draw triangles for plane() geometry + this._pInst.texture(pg); + this._pInst.plane(this.width, this.height); + this._pInst.pop(); + } } blendMode(mode) { From b0ef9280af1b4c38296dd7d5299ef45903003e13 Mon Sep 17 00:00:00 2001 From: Adam Ferriss Date: Mon, 28 Aug 2023 23:56:52 -0700 Subject: [PATCH 47/54] prevent error when in webgl mode and attempting to use cpu filters --- src/image/pixels.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/image/pixels.js b/src/image/pixels.js index 101fe373f0..45eeb9f8ea 100644 --- a/src/image/pixels.js +++ b/src/image/pixels.js @@ -544,7 +544,7 @@ p5.prototype.filter = function(...args) { } // when opting out of webgl, use old pixels method - if (!useWebGL) { + if (!useWebGL && !this._renderer.isP3D) { if (this.canvas !== undefined) { Filters.apply(this.canvas, Filters[operation], value); } else { @@ -553,6 +553,10 @@ p5.prototype.filter = function(...args) { return; } + if(!useWebGL && this._renderer.isP3D) { + console.warn('filter() with useWebGL=false is not supported in WEBGL'); + } + // when this is a webgl renderer, apply constant shader filter if (this._renderer.isP3D) { p5.RendererGL.prototype.filter.call(this._renderer, operation, value); From 547578b5c2dc58488af080876abaef80ea81f50f Mon Sep 17 00:00:00 2001 From: Adam Ferriss Date: Tue, 29 Aug 2023 00:02:24 -0700 Subject: [PATCH 48/54] testing different manual examples --- .../webgl/filter/index.html | 2 +- .../webgl/filter/sketch.js | 24 +++++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/test/manual-test-examples/webgl/filter/index.html b/test/manual-test-examples/webgl/filter/index.html index 656e6c469d..f91fd65214 100644 --- a/test/manual-test-examples/webgl/filter/index.html +++ b/test/manual-test-examples/webgl/filter/index.html @@ -7,11 +7,11 @@ - + diff --git a/test/manual-test-examples/webgl/filter/sketch.js b/test/manual-test-examples/webgl/filter/sketch.js index 41105e8572..7e1d6e6065 100644 --- a/test/manual-test-examples/webgl/filter/sketch.js +++ b/test/manual-test-examples/webgl/filter/sketch.js @@ -1,15 +1,29 @@ + let img; function preload() { - img = loadImage('../../../../docs/reference/assets/bricks.jpg'); + img = loadImage('../../../../docs/reference/assets/moonwalk.jpg'); } +let pg; + function setup() { - createCanvas(200, 200, WEBGL); - img.resize(200, 200); + // img.resize(600, 600); + createCanvas(img.width, img.height); + pg = createGraphics(img.width, img.height, WEBGL); } + function draw() { - image(img, -width/2, -height/2); - filter(POSTERIZE, 4); + if(pg.webglVersion === P2D){ + pg.image(img, 0, 0, width, height); + } else { + pg.image(img, -width / 2, -height / 2, width, height); + } + + if(mouseIsPressed){ + pg.filter(BLUR, 10); + } + + image(pg, 0, 0, width, height); } From 44d8f2cc6be05d0ac045f24186f119ced9360992 Mon Sep 17 00:00:00 2001 From: Adam Ferriss Date: Tue, 29 Aug 2023 00:04:20 -0700 Subject: [PATCH 49/54] Move blur loop into the shader and use a second temp buffer for rendering the blur filter. --- src/webgl/p5.RendererGL.js | 75 +++++++++++++++++------------ src/webgl/shaders/filters/blur.frag | 22 ++++++--- 2 files changed, 59 insertions(+), 38 deletions(-) diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index 39a9dc16bc..c714e57b8d 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -610,6 +610,7 @@ p5.RendererGL = class RendererGL extends p5.Renderer { // for post processing step this.filterShader = undefined; this.filterGraphicsLayer = undefined; + this.filterGraphicsLayerTemp = undefined; this.defaultFilterShaders = {}; this.textureMode = constants.IMAGE; @@ -978,8 +979,10 @@ p5.RendererGL = class RendererGL extends p5.Renderer { // so it's just created here once on the initial filter call. if (!this.filterGraphicsLayer) { // the real _pInst is buried when this is a secondary p5.Graphics + const pInst = this._pInst instanceof p5.Graphics ? this._pInst._pInst : this._pInst; + // create secondary layer this.filterGraphicsLayer = new p5.Graphics( @@ -991,6 +994,8 @@ p5.RendererGL = class RendererGL extends p5.Renderer { // geometries/borders on this layer should always be invisible this.filterGraphicsLayer.noStroke(); } + + let pg = this.filterGraphicsLayer; // use internal shader for filter constants BLUR, INVERT, etc @@ -1016,13 +1021,19 @@ p5.RendererGL = class RendererGL extends p5.Renderer { filterShaderFrags[operation] ); - // two-pass blur filter needs another shader attached to main - if (operation === constants.BLUR) { - this.otherBlurShader = new p5.Shader( - this, - filterShaderVert, - filterShaderFrags[constants.BLUR] - ); + // two-pass blur filter needs another graphics layer + if(!this.filterGraphicsLayerTemp) { + const pInst = this._pInst instanceof p5.Graphics ? + this._pInst._pInst : this._pInst; + // create secondary layer + this.filterGraphicsLayerTemp = + new p5.Graphics( + this.width, + this.height, + constants.WEBGL, + pInst + ); + this.filterGraphicsLayerTemp.noStroke(); } } this.filterShader = this.defaultFilterShaders[operation]; @@ -1047,6 +1058,7 @@ p5.RendererGL = class RendererGL extends p5.Renderer { } pg.clear(); // prevent undesirable feedback effects accumulating secretly + this.filterGraphicsLayerTemp.clear(); // apply blur shader with multiple passes if (operation === constants.BLUR) { @@ -1054,26 +1066,27 @@ p5.RendererGL = class RendererGL extends p5.Renderer { // setup this._pInst.push(); this._pInst.noStroke(); + + // draw main to temp buffer + this.filterGraphicsLayerTemp.image(this, -this.width/2, -this.height/2); + pg.shader(this.filterShader); - this._pInst.shader(this.otherBlurShader); this.filterShader.setUniform('texelSize', [1/this.width, 1/this.height]); - this.otherBlurShader.setUniform('texelSize', [1/this.width, 1/this.height]); - - // two-pass blur, repeated more with higher parameter - let steps = filterParameter; - for (let i = 0; i < steps; i++) { - // main contents onto pg - this.filterShader.setUniform('tex0', this); - this.filterShader.setUniform('direction', [1, 0]); // horiz pass - pg.rect(-this.width/2, -this.height/2, this.width, this.height); - - // pg contents onto main - this.otherBlurShader.setUniform('tex0', pg); - this.otherBlurShader.setUniform('direction', [0, 1]); // vert pass - this._pInst.rect( - -this.width/2, -this.height/2, this.width, this.height - ); - } + this.filterShader.setUniform('steps', Math.max(1, filterParameter)); + + // horiz pass + this.filterShader.setUniform('direction', [1, 0]); + this.filterShader.setUniform('tex0', this.filterGraphicsLayerTemp); + pg.rect(-this.width/2, -this.height/2, this.width, this.height); + + // read back to temp buffer + this.filterGraphicsLayerTemp.image(pg, -this.width/2, -this.height/2); + + // vert pass + this.filterShader.setUniform('direction', [0, 1]); + this.filterShader.setUniform('tex0', this.filterGraphicsLayerTemp); + pg.rect(-this.width/2, -this.height/2, this.width, this.height); + this._pInst.pop(); } // every other non-blur shader uses single pass @@ -1087,13 +1100,13 @@ p5.RendererGL = class RendererGL extends p5.Renderer { this.filterShader.setUniform('filterParameter', filterParameter); pg.rect(-this.width/2, -this.height/2, this.width, this.height); - // draw pg contents onto main renderer - this._pInst.push(); - this._pInst.noStroke(); // don't draw triangles for plane() geometry - this._pInst.texture(pg); - this._pInst.plane(this.width, this.height); - this._pInst.pop(); } + // draw pg contents onto main renderer + this._pInst.push(); + this._pInst.noStroke(); + this._pInst.image(pg, -this.width/2, -this.height/2, + this.width, this.height); + this._pInst.pop(); } blendMode(mode) { diff --git a/src/webgl/shaders/filters/blur.frag b/src/webgl/shaders/filters/blur.frag index 2706331cbc..f57bdb6dd9 100644 --- a/src/webgl/shaders/filters/blur.frag +++ b/src/webgl/shaders/filters/blur.frag @@ -1,27 +1,35 @@ +precision highp float; + // Two-pass blur filter, unweighted kernel. // See also a similar blur at Adam Ferriss' repo of shader examples: // https://github.com/aferriss/p5jsShaderExamples/blob/gh-pages/4_image-effects/4-9_single-pass-blur/effect.frag -precision highp float; uniform sampler2D tex0; varying vec2 vTexCoord; uniform vec2 direction; uniform vec2 texelSize; uniform float flipped; +uniform float steps; void main(){ + const float maxIterations = 100.0; vec2 uv = vTexCoord; if (flipped == 1.0) { uv.y = 1.0 - uv.y; } - + vec4 tex = texture2D(tex0, uv); - tex += texture2D(tex0, uv - texelSize * direction); - tex += texture2D(tex0, uv + texelSize * direction); - - tex /= 3.0; + float sum = 1.0; + + vec2 offset = direction * texelSize; + for(float i = 1.0; i <= maxIterations; i++) { + if( i > steps) break; + tex += texture2D(tex0, uv + i * offset); + tex += texture2D(tex0, uv - i * offset); + sum += 2.0; + } - gl_FragColor = tex; + gl_FragColor = tex / sum; } From b7454db462e71ae111c305162fc64f97f96f4ca7 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Tue, 29 Aug 2023 14:23:29 -0400 Subject: [PATCH 50/54] remove unused code --- src/webgl/shaders/filters/default.vert | 1 - 1 file changed, 1 deletion(-) diff --git a/src/webgl/shaders/filters/default.vert b/src/webgl/shaders/filters/default.vert index ebe0b2a864..ee73804cec 100644 --- a/src/webgl/shaders/filters/default.vert +++ b/src/webgl/shaders/filters/default.vert @@ -13,7 +13,6 @@ void main() { // copy position with a fourth coordinate for projection (1.0 is normal) vec4 positionVec4 = vec4(aPosition, 1.0); - // positionVec4.y = 1.0 - positionVec4.y; gl_Position = uProjectionMatrix * uModelViewMatrix * positionVec4; } From d0a2fe994812dafe1c440c44730a2ca169d3b7ba Mon Sep 17 00:00:00 2001 From: wong-justin Date: Wed, 30 Aug 2023 08:49:22 -0400 Subject: [PATCH 51/54] fix clear()ing at the right time --- src/webgl/p5.RendererGL.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/webgl/p5.RendererGL.js b/src/webgl/p5.RendererGL.js index c714e57b8d..0142172791 100644 --- a/src/webgl/p5.RendererGL.js +++ b/src/webgl/p5.RendererGL.js @@ -1058,11 +1058,12 @@ p5.RendererGL = class RendererGL extends p5.Renderer { } pg.clear(); // prevent undesirable feedback effects accumulating secretly - this.filterGraphicsLayerTemp.clear(); // apply blur shader with multiple passes if (operation === constants.BLUR) { + this.filterGraphicsLayerTemp.clear(); // prevent feedback effects here too + // setup this._pInst.push(); this._pInst.noStroke(); From b3583f702ad0d34d348412c56e3472553b02f390 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Wed, 30 Aug 2023 08:49:51 -0400 Subject: [PATCH 52/54] remove unused uniform preflipping is not necessary now with new vertex shader --- src/webgl/shaders/filters/blur.frag | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/webgl/shaders/filters/blur.frag b/src/webgl/shaders/filters/blur.frag index f57bdb6dd9..f7aca92ed4 100644 --- a/src/webgl/shaders/filters/blur.frag +++ b/src/webgl/shaders/filters/blur.frag @@ -9,16 +9,12 @@ uniform sampler2D tex0; varying vec2 vTexCoord; uniform vec2 direction; uniform vec2 texelSize; -uniform float flipped; uniform float steps; void main(){ const float maxIterations = 100.0; vec2 uv = vTexCoord; - if (flipped == 1.0) { - uv.y = 1.0 - uv.y; - } vec4 tex = texture2D(tex0, uv); float sum = 1.0; From 1bfe9c7defd82d4808d1a717adaa19078ba9fb50 Mon Sep 17 00:00:00 2001 From: wong-justin Date: Wed, 30 Aug 2023 09:09:49 -0400 Subject: [PATCH 53/54] add tests for filter parameters --- test/unit/webgl/p5.RendererGL.js | 66 ++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/test/unit/webgl/p5.RendererGL.js b/test/unit/webgl/p5.RendererGL.js index 435027ea2f..6ce2d33324 100644 --- a/test/unit/webgl/p5.RendererGL.js +++ b/test/unit/webgl/p5.RendererGL.js @@ -363,6 +363,72 @@ suite('p5.RendererGL', function() { `); myp5.filter(s); }); + + test('BLUR parameters make different output', function() { + myp5.createCanvas(10, 10, myp5.WEBGL); + let startDraw = () => { + myp5.clear(); + myp5.fill('RED'); + myp5.circle(0,0,8); + }; + let getPixels = () => { + myp5.loadPixels(); + return myp5.pixels.slice(); + }; + startDraw(); + myp5.filter(myp5.BLUR, 3); + let p1 = getPixels(); + startDraw(); + myp5.filter(myp5.BLUR, 10); + let p2 = getPixels(); + startDraw(); + myp5.filter(myp5.BLUR, 50); + let p3 = getPixels(); + assert.notDeepEqual(p1,p2); + assert.notDeepEqual(p2,p3); + }); + + test('POSTERIZE parameters make different output', function() { + myp5.createCanvas(10, 10, myp5.WEBGL); + let startDraw = () => { + myp5.clear(); + myp5.fill('CORAL'); + myp5.circle(0,0,8); + myp5.fill('CORNFLOWERBLUE'); + myp5.circle(2,2,8); + }; + let getPixels = () => { + myp5.loadPixels(); + return myp5.pixels.slice(); + }; + startDraw(); + myp5.filter(myp5.POSTERIZE, 2); + let p1 = getPixels(); + startDraw(); + myp5.filter(myp5.POSTERIZE, 4); + let p2 = getPixels(); + assert.notDeepEqual(p1,p2); + }); + + test('THRESHOLD parameters make different output', function() { + myp5.createCanvas(10, 10, myp5.WEBGL); + let startDraw = () => { + myp5.clear(); + myp5.fill('RED'); + myp5.circle(0,0,8); + }; + let getPixels = () => { + myp5.loadPixels(); + return myp5.pixels.slice(); + }; + startDraw(); + myp5.filter(myp5.THRESHOLD, 0.1); + let p1 = getPixels(); + startDraw(); + myp5.filter(myp5.THRESHOLD, 0.9); + let p2 = getPixels(); + assert.notDeepEqual(p1,p2); + }); }); test('contours match 2D', function() { From 121137f5f11408affcfb9c179fcab3c52e324928 Mon Sep 17 00:00:00 2001 From: Adam Ferriss Date: Thu, 31 Aug 2023 13:54:40 -0700 Subject: [PATCH 54/54] Update pixels.js Change language around gaussian blur --- src/image/pixels.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/image/pixels.js b/src/image/pixels.js index 45eeb9f8ea..a857bbc8c7 100644 --- a/src/image/pixels.js +++ b/src/image/pixels.js @@ -329,9 +329,11 @@ p5.prototype._copyHelper = ( * results are most noticeable in the lower ranges. The default parameter is 4. * * `BLUR` - * Executes a Gaussian blur with the level parameter specifying the extent + * Executes a blur with the level parameter specifying the extent * of the blurring. If no parameter is used, the blur is equivalent to - * Gaussian blur of radius 4. Larger values increase the blur. + * a blur of radius 4. Larger values increase the blur. In P2D mode a + * gaussian blur is performed on the CPU. When in webGL mode, a box blur is + * used instead. * * `ERODE` * Reduces the light areas. No parameter is used.