Order independent transparency (#1418)

* skip second framebuffer & dynamic attachment ids

* draft OIT implementation

* combine shaders

* rename and simply colorbuffer attachment

* reduce SSAO buffer size and reuse HDR_color

* add some convenience

* fix typos

* fix error for nonstandard prerenderers

* cleanup

* fix transparency with transparency = false

i.e. scatter, text, etc

* remove alpha from weight sum

* switch back to Float32 for position buffer

* reuse HDR color buffer for FXAA

* add note on buffer reuse

* cleanup some comments

* fix discoloration

* remove warning

* use that opaque alpha = 0

* add OIT test

* add transparency

* avoid reading and writing to the same colorbuffer

* reduce position buffer size

* fix 3d contour

* Update runtests.jl

* Update runtests.jl

* add docs for transparency and OIT

Co-authored-by: Simon <sdanisch@protonmail.com>

CairoMakie/test/runtests.jl

@@ -62,7 +62,8 @@ excludes = Set([
     "Connected Sphere",
     # markers too big, close otherwise, needs to be assimilated with glmakie
     "Unicode Marker",
-    "Depth Shift"
+    "Depth Shift",
+    "Order Independent Transparency"
 ])
 excludes2 = Set(["short_tests_90", "short_tests_111", "short_tests_35", "short_tests_13", "short_tests_3"])
 

GLMakie/assets/shader/fragment_output.frag

@@ -3,17 +3,38 @@
 layout(location=0) out vec4 fragment_color;
 layout(location=1) out uvec2 fragment_groupid;
 {{buffers}}
+// resolves to:
+// // if transparency == true
+// layout(location=2) out float coverage; 
 
+// // if transparency == false && enable_SSAO[] = true
+// layout(location=2) out vec3 fragment_position;
+// layout(location=3) out vec3 fragment_normal_occlusion;
 
-in vec4 o_view_pos;
+
+in vec3 o_view_pos;
 in vec3 o_normal;
 
 void write2framebuffer(vec4 color, uvec2 id){
     if(color.a <= 0.0)
         discard;
-    // For FXAA & SSAO
-    fragment_color = color;
+
     // For plot/sprite picking
     fragment_groupid = id;
+
     {{buffer_writes}}
+    // resolves to:
+
+    // // if transparency == true
+    // float weight = color.a * max(0.01, 3000 * pow((1 - gl_FragCoord.z), 3));
+    // fragment_color = weight * color;
+    // coverage = color.a;
+
+    // // if transparency == false && enable_SSAO[] = true
+    // fragment_color = color;
+    // fragment_position = o_view_pos;
+    // fragment_normal_occlusion.xyz = o_normal;
+
+    // // else
+    // fragment_color = color;
 }

GLMakie/assets/shader/heatmap.vert

@@ -12,7 +12,7 @@ uniform float depth_shift;
 out vec2 o_uv;
 flat out uvec2 o_objectid;
 
-out vec4 o_view_pos;
+out vec3 o_view_pos;
 out vec3 o_normal;
 
 ivec2 ind2sub(ivec2 dim, int linearindex){
@@ -21,7 +21,7 @@ ivec2 ind2sub(ivec2 dim, int linearindex){
 
 void main(){
     //Outputs for ssao, which we don't use for 2d shaders like heatmap/image
-    o_view_pos = vec4(0);
+    o_view_pos = vec3(0);
     o_normal = vec3(0);
 
     int index = gl_InstanceID;

GLMakie/assets/shader/image.vert

@@ -11,15 +11,15 @@ uniform float depth_shift;
 out vec2 o_uv;
 flat out uvec2 o_objectid;
 
-out vec4 o_view_pos;
+out vec3 o_view_pos;
 out vec3 o_normal;
 
 vec4 _position(vec3 p){return vec4(p,1);}
 vec4 _position(vec2 p){return vec4(p,0,1);}
 
 void main(){
     //Outputs for ssao, which we don't use for 2d shaders like heatmap/image
-    o_view_pos = vec4(0);
+    o_view_pos = vec3(0);
     o_normal = vec3(0);
     o_uv = texturecoordinates;
     o_objectid = uvec2(objectid, gl_VertexID+1);

GLMakie/assets/shader/line_segment.geom

@@ -38,12 +38,12 @@ void emit_vertex(vec2 position, vec2 uv, int index)
 
 uniform int max_primtives;
 
-out vec4 o_view_pos;
+out vec3 o_view_pos;
 out vec3 o_normal;
 
 void main(void)
 {
-    o_view_pos = vec4(0);
+    o_view_pos = vec3(0);
     o_normal = vec3(0);
     // get the four vertices passed to the shader:
     vec2 p0 = screen_space(gl_in[0].gl_Position); // start of previous segment

GLMakie/assets/shader/lines.geom

@@ -44,12 +44,12 @@ void emit_vertex(vec2 position, vec2 uv, int index, float ratio)
 uniform int max_primtives;
 const float infinity = 1.0 / 0.0;
 
-out vec4 o_view_pos;
+out vec3 o_view_pos;
 out vec3 o_normal;
 
 void main(void)
 {
-    o_view_pos = vec4(0);
+    o_view_pos = vec3(0);
     o_normal = vec3(0);
     // We mark each of the four vertices as valid or not. Vertices can be
     // marked invalid on input (eg, if they contain NaN). We also mark them

GLMakie/assets/shader/postprocessing/OIT_blend.frag

@@ -0,0 +1,25 @@
+{{GLSL_VERSION}}
+
+// Based on https://jcgt.org/published/0002/02/09/
+// See https://github.com/JuliaPlots/Makie.jl/issues/1390
+
+in vec2 frag_uv;
+
+// contains sum_i C_i * weight(depth_i, alpha_i)
+uniform sampler2D sum_color;
+// contains pod_i (1 - alpha_i)
+uniform sampler2D prod_alpha;
+
+out vec4 fragment_color;
+
+void main(void)
+{
+    vec4 summed_color_weight = texture(sum_color, frag_uv);
+    float transmittance = texture(prod_alpha, frag_uv).r;
+
+    vec3 weighted_transparent = summed_color_weight.rgb / max(summed_color_weight.a, 0.00001);
+    vec3 full_weighted_transparent = weighted_transparent * (1 - transmittance);
+
+    fragment_color.rgb = full_weighted_transparent;
+    fragment_color.a = transmittance;
+}

GLMakie/assets/shader/sprites.geom

@@ -97,12 +97,12 @@ mat2 diagm(vec2 v){
     return mat2(v.x, 0.0, 0.0, v.y);
 }
 
-out vec4 o_view_pos;
+out vec3 o_view_pos;
 out vec3 o_normal;
 
 void main(void)
 {
-    o_view_pos = vec4(0);
+    o_view_pos = vec3(0);
     o_normal = vec3(0);
 
     // emit quad as triangle strip

GLMakie/assets/shader/util.vert

@@ -225,7 +225,7 @@ vec4 _color(Nothing color, float intensity, sampler1D color_map, vec2 color_norm
     return color_lookup(intensity, color_map, color_norm);
 }
 
-out vec4 o_view_pos;
+out vec3 o_view_pos;
 out vec3 o_normal;
 out vec3 o_lightdir;
 out vec3 o_camdir;
@@ -242,17 +242,18 @@ void render(vec4 position_world, vec3 normal, mat4 view, mat4 projection, vec3 l
     // normal in world space
     o_normal = normalmatrix * normal;
     // position in view space (as seen from camera)
-    o_view_pos = view * position_world;
+    vec4 view_pos = view * position_world;
     // position in clip space (w/ depth)
-    gl_Position = projection * o_view_pos;
+    gl_Position = projection * view_pos;
     gl_Position.z += gl_Position.w * depth_shift;
     // direction to light
-    o_lightdir = normalize(view*vec4(lightposition, 1.0) - o_view_pos).xyz;
+    o_lightdir = normalize(view*vec4(lightposition, 1.0) - view_pos).xyz;
     // direction to camera
     // This is equivalent to
-    // normalize(view*vec4(eyeposition, 1.0) - o_view_pos).xyz
+    // normalize(view*vec4(eyeposition, 1.0) - view_pos).xyz
     // (by definition `view * eyeposition = 0`)
-    o_camdir = normalize(-o_view_pos).xyz;
+    o_camdir = normalize(-view_pos).xyz;
+    o_view_pos = view_pos.xyz / view_pos.w;
 }
 
 //

GLMakie/assets/shader/volume.vert

@@ -10,13 +10,13 @@ uniform vec3 lightposition;
 uniform mat4 modelinv;
 uniform float depth_shift;
 
-out vec4 o_view_pos;
+out vec3 o_view_pos;
 out vec3 o_normal;
 
 void main()
 {
     // TODO set these in volume.frag
-    o_view_pos = vec4(0);
+    o_view_pos = vec3(0);
     o_normal = vec3(0);
     vec4 world_vert = model * vec4(vertices, 1);
     frag_vert = world_vert.xyz;

GLMakie/src/GLAbstraction/GLRender.jl

@@ -156,6 +156,10 @@ end
 function enabletransparency()
     glEnablei(GL_BLEND, 0)
     glDisablei(GL_BLEND, 1)
-    glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
+    # This does:
+    # target.rgb = source.a * source.rgb + (1 - source.a) * target.rgb
+    # target.a = 0 * source.a + 1 * target.a
+    # the latter is required to keep target.a = 1 for the OIT pass
+    glBlendFuncSeparate(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA, GL_ZERO, GL_ONE)
     return
 end

GLMakie/src/GLAbstraction/GLRenderObject.jl

@@ -42,12 +42,34 @@ function (sp::StandardPrerender)()
         glEnable(GL_DEPTH_TEST)
         glDepthFunc(GL_LEQUAL)
     end
-    # Disable depth write for transparent objects
-    glDepthMask(sp.transparency[] ? GL_FALSE : GL_TRUE)
+
     # Disable cullface for now, untill all rendering code is corrected!
     glDisable(GL_CULL_FACE)
     # glCullFace(GL_BACK)
-    enabletransparency()
+
+    if sp.transparency[]
+        # disable depth buffer writing
+        glDepthMask(GL_FALSE)
+
+        # Blending
+        glEnable(GL_BLEND)
+        glBlendEquation(GL_FUNC_ADD)
+
+        # buffer 0 contains weight * color.rgba, should do sum
+        # destination <- 1 * source + 1 * destination
+        glBlendFunci(0, GL_ONE, GL_ONE)
+
+        # buffer 1 is objectid, do nothing
+        glDisablei(1, GL_BLEND)
+
+        # buffer 2 is color.a, should do product
+        # destination <- 0 * source + (1 - source) * destination
+        glBlendFunci(2, GL_ZERO, GL_ONE_MINUS_SRC_COLOR)
+
+    else
+        glDepthMask(GL_TRUE)
+        enabletransparency()
+    end
 end
 
 struct StandardPostrender

GLMakie/src/GLVisualize/visualize/image_like.jl

@@ -28,8 +28,15 @@ function _default(main::MatTypes{T}, ::Style, data::Dict) where T <: Colorant
         end => to_uvmesh
         preferred_camera = :orthographic_pixel
         fxaa = false
-        shader = GLVisualizeShader("fragment_output.frag", "image.vert", "texture.frag",
-            view = Dict("uv_swizzle" => "o_uv.$(spatialorder)"))
+        transparency = false
+        shader = GLVisualizeShader(
+            "fragment_output.frag", "image.vert", "texture.frag",
+            view = Dict(
+                "uv_swizzle" => "o_uv.$(spatialorder)",
+                "buffers" => output_buffers(to_value(transparency)),
+                "buffer_writes" => output_buffer_writes(to_value(transparency))
+            )
+        )
     end
 end
 
@@ -56,7 +63,14 @@ function gl_heatmap(main::MatTypes{T}, data::Dict) where T <: AbstractFloat
         stroke_width::Float32 = 0.0f0
         levels::Float32 = 0f0
         stroke_color = RGBA{Float32}(0,0,0,0)
-        shader = GLVisualizeShader("fragment_output.frag", "heatmap.vert", "intensity.frag")
+        transparency = false
+        shader = GLVisualizeShader(
+            "fragment_output.frag", "heatmap.vert", "intensity.frag",
+            view = Dict(
+                "buffers" => output_buffers(to_value(transparency)),
+                "buffer_writes" => output_buffer_writes(to_value(transparency))
+            )
+        )
         fxaa = false
     end
     return data
@@ -87,20 +101,23 @@ function _default(main::VolumeTypes{T}, s::Style, data::Dict) where T <: VolumeE
         model = Mat4f(I)
         modelinv = const_lift(inv, model)
         color_map = default(Vector{RGBA}, s) => Texture
-        color_norm = color_map == nothing ? nothing : const_lift(extrema2f0, main)
-        color = color_map == nothing ? default(RGBA, s) : nothing
+        color_norm = color_map === nothing ? nothing : const_lift(extrema2f0, main)
+        color = color_map === nothing ? default(RGBA, s) : nothing
 
         algorithm = MaximumIntensityProjection
         absorption = 1f0
         isovalue = 0.5f0
         isorange = 0.01f0
         enable_depth = true
+        transparency = false
         shader = GLVisualizeShader(
             "fragment_output.frag", "util.vert", "volume.vert", "volume.frag",
             view = Dict(
                 "depth_init"  => vol_depth_init(to_value(enable_depth)),
                 "depth_main"  => vol_depth_main(to_value(enable_depth)),
-                "depth_write" => vol_depth_write(to_value(enable_depth))
+                "depth_write" => vol_depth_write(to_value(enable_depth)),
+                "buffers" => output_buffers(to_value(transparency)),
+                "buffer_writes" => output_buffer_writes(to_value(transparency))
             )
         )
         prerender = VolumePrerender(data[:transparency], data[:overdraw])
@@ -136,7 +153,14 @@ function _default(main::VolumeTypes{T}, s::Style, data::Dict) where T <: RGBA
         color = color_map === nothing ? default(RGBA, s) : nothing
 
         algorithm = AbsorptionRGBA
-        shader = GLVisualizeShader("fragment_output.frag", "util.vert", "volume.vert", "volume.frag")
+        transparency = false
+        shader = GLVisualizeShader(
+            "fragment_output.frag", "util.vert", "volume.vert", "volume.frag",
+            view = Dict(
+                "buffers" => output_buffers(to_value(transparency)),
+                "buffer_writes" => output_buffer_writes(to_value(transparency))
+            )
+        )
         prerender = VolumePrerender(data[:transparency], data[:overdraw])
         postrender = () -> glDisable(GL_CULL_FACE)
     end

GLMakie/src/GLVisualize/visualize/lines.jl

@@ -79,7 +79,14 @@ function line_visualization(position::Union{VectorTypes{T}, MatTypes{T}}, data::
             len0 = length(p) - 1
             return isempty(p) ? Cuint[] : Cuint[0; 0:len0; len0]
         end => to_index_buffer
-        shader              = GLVisualizeShader("fragment_output.frag", "util.vert", "lines.vert", "lines.geom", "lines.frag")
+        transparency = false
+        shader              = GLVisualizeShader(
+            "fragment_output.frag", "util.vert", "lines.vert", "lines.geom", "lines.frag",
+            view = Dict(
+                "buffers" => output_buffers(to_value(transparency)),
+                "buffer_writes" => output_buffer_writes(to_value(transparency))
+            )
+        )
         gl_primitive        = GL_LINE_STRIP_ADJACENCY
         valid_vertex        = const_lift(p_vec) do points
             map(p-> Float32(all(isfinite, p)), points)
@@ -115,7 +122,14 @@ function _default(positions::VectorTypes{T}, s::style"linesegment", data::Dict)
         fxaa                = false
         indices             = const_lift(length, positions) => to_index_buffer
         # TODO update boundingbox
-        shader              = GLVisualizeShader("fragment_output.frag", "util.vert", "line_segment.vert", "line_segment.geom", "lines.frag")
+        transparency = false
+        shader              = GLVisualizeShader(
+            "fragment_output.frag", "util.vert", "line_segment.vert", "line_segment.geom", "lines.frag",
+            view = Dict(
+                "buffers" => output_buffers(to_value(transparency)),
+                "buffer_writes" => output_buffer_writes(to_value(transparency))
+            )
+        )
         gl_primitive        = GL_LINES
     end
     if !isa(pattern, Texture) && pattern != nothing

GLMakie/src/GLVisualize/visualize/mesh.jl

@@ -12,9 +12,14 @@ function _default(mesh::TOrSignal{M}, s::Style, data::Dict) where M <: GeometryB
         color_norm = nothing
         fetch_pixel = false
         uv_scale = Vec2f(1)
+        transparency = false
         shader = GLVisualizeShader(
-            "fragment_output.frag", "util.vert", "standard.vert", "standard.frag",
-            view = Dict("light_calc" => light_calc(shading))
+            "util.vert", "standard.vert", "standard.frag", "fragment_output.frag",
+            view = Dict(
+                "light_calc" => light_calc(shading),
+                "buffers" => output_buffers(to_value(transparency)),
+                "buffer_writes" => output_buffer_writes(to_value(transparency))
+            )
         )
     end
 end