Reviving CairoMakie/GLMakie integration

.github/workflows/glmakie.yaml

@@ -44,7 +44,7 @@ jobs:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
       - uses: julia-actions/cache@v1
-      - run: sudo apt-get update && sudo apt-get install -y xorg-dev mesa-utils xvfb libgl1 freeglut3-dev libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev libxext-dev
+      - run: sudo apt-get update && sudo apt-get install -y xorg-dev mesa-utils xvfb libgl1 freeglut3-dev libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev libxext-dev xsettingsd x11-xserver-utils
       - name: Install Julia dependencies
         shell: julia --project=monorepo {0}
         run: |

CairoMakie/src/CairoMakie.jl

@@ -29,6 +29,7 @@ include("infrastructure.jl")
 include("utils.jl")
 include("primitives.jl")
 include("overrides.jl")
+include("glmakie_integration.jl")
 
 function __init__()
     activate!()

CairoMakie/src/glmakie_integration.jl

@@ -0,0 +1,168 @@
+# Upstreamable code
+
+# TODO: make this function more efficient!
+function alpha_colorbuffer(Backend, screen::MakieScreen)
+    img = try
+        scene = screen.root_scene
+        display(screen)
+        bg = scene.backgroundcolor[]
+        scene.backgroundcolor[] = RGBAf(0, 0, 0, 1)
+        b1 = Makie.colorbuffer(screen)
+        scene.backgroundcolor[] = RGBAf(1, 1, 1, 1)
+        b2 = Makie.colorbuffer(screen)
+        scene.backgroundcolor[] = bg
+        map(infer_alphacolor, b1, b2)
+    catch e
+        println("Error: something failed in alpha colorbuffer!")
+        rethrow(e)
+    finally
+    end
+
+    return img
+end
+
+function infer_alphacolor(rgb1, rgb2)
+    rgb1 == rgb2 && return RGBAf(rgb1.r, rgb1.g, rgb1.b, 1)
+    c1 = Float64.((rgb1.r, rgb1.g, rgb1.b))
+    c2 = Float64.((rgb2.r, rgb2.g, rgb2.b))
+    alpha = @. 1 - (c1 - c2) * -1 # ( / (0 - 1))
+    meanalpha = clamp(sum(alpha) / 3, 0, 1)
+    meanalpha == 0 && return RGBAf(0, 0, 0, 0)
+    c = @. clamp((c1 / meanalpha), 0, 1)
+    return RGBAf(c..., meanalpha)
+end
+
+function create_render_scene(plot::Combined; scale::Real = 1)
+    scene = Makie.parent_scene(plot)
+    w, h = Int.(scene.px_area[].widths)
+
+    # We create a dummy scene to render to, which inherits its parent's
+    # transformation and camera.
+    # WARNING: lights, SSAO and axis do not update when the original Scene's
+    # attributes do.  This is because they are stored as fields in the Scene
+    # struct, not as attributes.
+    render_scene = Makie.Scene(
+        camera = Makie.camera(scene),
+        lights = scene.lights,
+        ssao = scene.ssao,
+        show_axis = false,
+        backgroundcolor = :transparent
+    )
+
+    # continually keep the pixel area updated
+    on(pixelarea(scene); update = true) do px_area
+        Makie.resize!(render_scene, (px_area.widths .* scale)...)
+    end
+
+    # link the transofrmation attributes
+    Makie.Observables.connect!(render_scene.transformation.transform_func, scene.transformation.transform_func)
+    Makie.Observables.connect!(render_scene.transformation.rotation      , scene.transformation.rotation)
+    Makie.Observables.connect!(render_scene.transformation.scale         , scene.transformation.scale)
+    Makie.Observables.connect!(render_scene.transformation.translation   , scene.transformation.translation)
+    Makie.Observables.connect!(render_scene.transformation.model         , scene.transformation.model)
+
+    # push only the relevant plot to the scene
+    push!(render_scene, plot)
+
+    return render_scene
+
+end
+
+"""
+    plot2img(backend::Makie.AbstractBackend, plot::Combined; scale::Real = 1, use_backgroundcolor = false)
+"""
+function plot2img(Backend, plot::Combined; scale::Real = 1, use_backgroundcolor = false)
+    parent = Makie.parent_scene(plot)
+    # obtain or create the render scene
+    screen_ind = findfirst(x -> x isa Backend.Screen, parent.current_screens)
+    render_screen = if isnothing(screen_ind)
+        Backend.Screen(parent; visible = false, px_per_unit = scale)
+    else
+        parent.current_screens[screen_ind]
+    end
+
+    img = if use_backgroundcolor
+        Makie.colorbuffer(render_screen)
+    else # render with transparency, using the alpha-colorbuffer hack
+        alpha_colorbuffer(Backend, render_screen)
+    end
+
+    return img
+end
+
+# Utility function to remove rendercaches
+function purge_render_cache!(sc::Scene)
+    haskey(scene.attributes, :_render_scenes) && delete!(scene.attributes, :_render_scenes)
+    purge_render_cache!.(scene.children)
+end
+purge_render_cache!(fig::Figure) = purge_render_cache!(fig.scene)
+
+# Rendering pipeline
+
+# This goes as follows:
+# The first time a plot is encountered which has to be rasterized,
+# we create the rasterization scene, and connect it to the original Scene's
+# attributes.
+# Then, we retrieve this scene from `plot._render_scene`, an attribute of the plot
+# which we set in the previous step to contain the scene.
+# This retrieval
+
+function draw_plot_as_image_with_backend(Backend, scene::Scene, screen::Screen, plot; scale = 1)
+
+        # special case if CairoMakie is the backend, since
+        # we don't want an infinite loop.
+        if Backend == @__MODULE__
+            draw_plot_as_image(scene, screen, plot, scale)
+            return
+        end
+
+        w, h = Int.(scene.px_area[].widths)
+
+        img = plot2img(Backend, plot; scale = scale, use_backgroundcolor = false)
+        Makie.save("hi.png", img) # cache for debugging
+
+        surf = Cairo.CairoARGBSurface(to_uint32_color.(img))
+
+        Cairo.rectangle(screen.context, 0, 0, w, h)
+        Cairo.save(screen.context)
+        Cairo.scale(screen.context, w / surf.width, h / surf.height)
+        Cairo.set_source_surface(screen.context, surf, 0, 0)
+        p = Cairo.get_source(screen.context)
+        Cairo.pattern_set_extend(p, Cairo.EXTEND_PAD) # avoid blurry edges
+        Cairo.pattern_set_filter(p, Cairo.FILTER_NEAREST)
+        Cairo.fill(screen.context)
+        Cairo.restore(screen.context)
+
+        return
+
+end
+
+function draw_scene_as_image(Backend, scene::Scene, screen::Screen; scale = 1)
+    w, h = Int.(scene.px_area[].widths)
+
+    render_scene = create_render_scene(scene.plots[begin]; scale = scale)
+    if length(scene.plots) > 1
+        push!.(Ref(render_scene), scene.plots[(begin+1):end])
+    end
+
+    img = if RGBAf(Makie.to_color(scene.backgroundcolor[])) == RGBAf(0,0,0,0)
+        alpha_colorbuffer(Backend, scene)
+    else
+        Makie.colorbuffer(scene)
+    end
+
+    Makie.save("hi.png", img) # cache for debugging
+
+    surf = Cairo.CairoARGBSurface(to_uint32_color.(img))
+
+    Cairo.rectangle(screen.context, 0, 0, w, h)
+    Cairo.save(screen.context)
+    Cairo.scale(screen.context, w / surf.width, h / surf.height)
+    Cairo.set_source_surface(screen.context, surf, 0, 0)
+    p = Cairo.get_source(screen.context)
+    Cairo.pattern_set_extend(p, Cairo.EXTEND_PAD) # avoid blurry edges
+    Cairo.pattern_set_filter(p, Cairo.FILTER_NEAREST)
+    Cairo.fill(screen.context)
+    Cairo.restore(screen.context)
+
+end

CairoMakie/src/infrastructure.jl

@@ -41,9 +41,20 @@ function cairo_draw(screen::Screen, scene::Scene)
         # rasterize it when plotting to vector backends, by using the `rasterize`
         # keyword argument.  This can be set to a Bool or an Int which describes
         # the density of rasterization (in terms of a direct scaling factor.)
-        if to_value(get(p, :rasterize, false)) != false && should_rasterize
-            draw_plot_as_image(pparent, screen, p, p[:rasterize][])
-        else # draw vector
+        # This can also be set to a `Module` (i.e., )
+        if to_value(get(p, :rasterize, false)) != false
+            if p.rasterize[] isa Union{Bool, Int} && should_rasterize
+                draw_plot_as_image(pparent, screen, p, p[:rasterize][])
+            elseif p.rasterize[] isa Union{<: Module, Tuple{<: Module, Int}}
+                backend = p.rasterize[] isa Module ? p.rasterize[] : p.rasterize[][1]
+                scale   = p.rasterize[] isa Module ? 1 : p.rasterize[][2]
+                draw_plot_as_image_with_backend(backend, pparent, screen, p; scale = scale)
+            else # rasterization option was not recognized, or should_rasterize
+                 # was false and backend was not selected.
+                draw_plot(pparent, screen, p)
+            end
+        else # draw vector, only if a parent plot has not been rasterized
+
             draw_plot(pparent, screen, p)
         end
         Cairo.restore(screen.context)

GLMakie/Project.toml

@@ -27,7 +27,7 @@ Colors = "0.11, 0.12"
 FileIO = "1.6"
 FixedPointNumbers = "0.7, 0.8"
 FreeTypeAbstraction = "0.10"
-GLFW = "3"
+GLFW = "3.3"
 GeometryBasics = "0.4.1"
 Makie = "=0.19.2"
 MeshIO = "0.4"

GLMakie/assets/shader/distance_shape.frag

@@ -25,6 +25,7 @@ uniform float           stroke_width;
 uniform float           glow_width;
 uniform int             shape; // shape is a uniform for now. Making them a in && using them for control flow is expected to kill performance
 uniform vec2            resolution;
+uniform float           px_per_unit;
 uniform bool            transparent_picking;
 
 flat in float           f_viewport_from_u_scale;
@@ -97,7 +98,9 @@ void stroke(vec4 strokecolor, float signed_distance, float width, inout vec4 col
 
 void glow(vec4 glowcolor, float signed_distance, float inside, inout vec4 color){
     if (glow_width > 0.0){
-        float outside = (abs(signed_distance)-stroke_width)/glow_width;
+        float s_stroke_width = px_per_unit * stroke_width;
+        float s_glow_width = px_per_unit * glow_width;
+        float outside = (abs(signed_distance)-s_stroke_width)/s_glow_width;
         float alpha = 1-outside;
         color = mix(vec4(glowcolor.rgb, glowcolor.a*alpha), color, inside);
     }
@@ -145,13 +148,14 @@ void main(){
     // See notes in geometry shader where f_viewport_from_u_scale is computed.
     signed_distance *= f_viewport_from_u_scale;
 
-    float inside_start = max(-stroke_width, 0.0);
+    float s_stroke_width = px_per_unit * stroke_width;
+    float inside_start = max(-s_stroke_width, 0.0);
     float inside = aastep(inside_start, signed_distance);
     vec4 final_color = f_bg_color;
 
     fill(f_color, image, tex_uv, inside, final_color);
-    stroke(f_stroke_color, signed_distance, -stroke_width, final_color);
-    glow(f_glow_color, signed_distance, aastep(-stroke_width, signed_distance), final_color);
+    stroke(f_stroke_color, signed_distance, -s_stroke_width, final_color);
+    glow(f_glow_color, signed_distance, aastep(-s_stroke_width, signed_distance), final_color);
     // TODO: In 3D, we should arguably discard fragments outside the sprite
     //       But note that this may interfere with object picking.
     // if (final_color == f_bg_color)

GLMakie/assets/shader/sprites.geom

@@ -38,6 +38,7 @@ uniform float stroke_width;
 uniform float glow_width;
 uniform int shape; // for RECTANGLE hack below
 uniform vec2 resolution;
+uniform float px_per_unit;
 uniform float depth_shift;
 
 in int g_primitive_index[];
@@ -138,7 +139,7 @@ void main(void)
                              0.0,         1.0/vclip.w, 0.0,         0.0,
                              0.0,         0.0,         1.0/vclip.w, 0.0,
                              -vclip.xyz/(vclip.w*vclip.w),          0.0);
-    mat2 dxyv_dxys = diagm(0.5*resolution) * mat2(d_ndc_d_clip*trans);
+    mat2 dxyv_dxys = diagm(0.5*px_per_unit*resolution) * mat2(d_ndc_d_clip*trans);
     // Now, our buffer size is expressed in viewport pixels but we get back to
     // the sprite coordinate system using the scale factor of the
     // transformation (for isotropic transformations). For anisotropic

GLMakie/src/GLAbstraction/GLUniforms.jl

@@ -200,6 +200,11 @@ gl_convert(s::Vector{Matrix{T}}) where {T<:Colorant} = Texture(s)
 gl_convert(s::Nothing) = s
 
 
+# special overloads for rasterization
+gl_convert(s::Module) = nothing
+gl_convert(s::Tuple{Module, Any}) = nothing
+
+
 isa_gl_struct(x::AbstractArray) = false
 isa_gl_struct(x::NATIVE_TYPES) = false
 isa_gl_struct(x::Colorant) = false

GLMakie/src/drawing_primitives.jl

@@ -107,6 +107,7 @@ function cached_robj!(robj_func, screen, scene, x::AbstractPlot)
             gl_attributes[:ambient] = ambientlight.color
         end
         gl_attributes[:track_updates] = screen.config.render_on_demand
+        gl_attributes[:px_per_unit] = screen.px_per_unit
 
         robj = robj_func(gl_attributes)