[Flutter GPU] Texture binding, index binding, attachments, depth state. #48386

bdero · 2023-11-26T17:34:02Z

Now rendering textured 3D models!

Combined depth+stencil attachment.
Allow for multiple color attachments.
Add blend mode configuration.
Fix uniform ordering for vertex shaders.
Texture binding and sampling options.
Index buffer binding.
Depth configuration.

Screen.Recording.2023-11-26.at.9.17.51.AM.mov

  void paint(Canvas canvas, Size size) {
    /// Allocate a new renderable texture.
    final gpu.Texture? renderTexture = gpu.gpuContext.createTexture(
        gpu.StorageMode.devicePrivate, 300, 300,
        enableRenderTargetUsage: true,
        enableShaderReadUsage: true,
        coordinateSystem: gpu.TextureCoordinateSystem.renderToTexture);
    if (renderTexture == null) {
      return;
    }

    final gpu.Texture? depthTexture = gpu.gpuContext.createTexture(
        gpu.StorageMode.deviceTransient, 300, 300,
        format: gpu.gpuContext.defaultDepthStencilFormat,
        enableRenderTargetUsage: true,
        coordinateSystem: gpu.TextureCoordinateSystem.renderToTexture);
    if (depthTexture == null) {
      return;
    }

    /// Create the command buffer. This will be used to submit all encoded
    /// commands at the end.
    final commandBuffer = gpu.gpuContext.createCommandBuffer();

    /// Define a render target. This is just a collection of attachments that a
    /// RenderPass will write to.
    final renderTarget = gpu.RenderTarget.singleColor(
      colorAttachment: gpu.ColorAttachment(texture: renderTexture),
      depthStencilAttachment: gpu.DepthStencilAttachment(
          texture: depthTexture, depthClearValue: 1.0),
    );

    /// Add a render pass encoder to the command buffer so that we can start
    /// encoding commands.
    final encoder = commandBuffer.createRenderPass(renderTarget);

    /// Load a shader bundle asset.
    final library = gpu.ShaderLibrary.fromAsset('TestLibrary')!;

    /// Create a RenderPipeline using shaders from the asset.
    final vertex = library['TextureVertex']!;
    final fragment = library['TextureFragment']!;
    final pipeline = gpu.gpuContext.createRenderPipeline(vertex, fragment);

    encoder.bindPipeline(pipeline);

    encoder.setDepthWriteEnable(true);
    encoder.setDepthCompareOperation(gpu.CompareFunction.less);

    /// (Optional) Configure blending for the first color attachment.
    encoder.setColorBlendEnable(true);
    encoder.setColorBlendEquation(gpu.ColorBlendEquation(
        colorBlendOperation: gpu.BlendOperation.add,
        sourceColorBlendFactor: gpu.BlendFactor.one,
        destinationColorBlendFactor: gpu.BlendFactor.oneMinusSourceAlpha,
        alphaBlendOperation: gpu.BlendOperation.add,
        sourceAlphaBlendFactor: gpu.BlendFactor.one,
        destinationAlphaBlendFactor: gpu.BlendFactor.oneMinusSourceAlpha));

    /// Append quick geometry and uniforms to a host buffer that will be
    /// automatically uploaded to the GPU later on.
    final transients = gpu.HostBuffer();
    final vertices = transients.emplace(float32(<double>[
      -1, -1, -1, /* */ 0, 0, /* */ 1, 0, 0, 1, //
      1, -1, -1, /*  */ 1, 0, /* */ 0, 1, 0, 1, //
      1, 1, -1, /*   */ 1, 1, /* */ 0, 0, 1, 1, //
      -1, 1, -1, /*  */ 0, 1, /* */ 0, 0, 0, 1, //
      -1, -1, 1, /*  */ 0, 0, /* */ 0, 1, 1, 1, //
      1, -1, 1, /*   */ 1, 0, /* */ 1, 0, 1, 1, //
      1, 1, 1, /*    */ 1, 1, /* */ 1, 1, 0, 1, //
      -1, 1, 1, /*   */ 0, 1, /* */ 1, 1, 1, 1, //
    ]));
    final indices = transients.emplace(uint16(<int>[
      0, 1, 3, 3, 1, 2, //
      1, 5, 2, 2, 5, 6, //
      5, 4, 6, 6, 4, 7, //
      4, 0, 7, 7, 0, 3, //
      3, 2, 7, 7, 2, 6, //
      4, 5, 0, 0, 5, 1, //
    ]));
    final mvp = transients.emplace(float32Mat(Matrix4(
          0.5, 0, 0, 0, //
          0, 0.5, 0, 0, //
          0, 0, 0.2, 0, //
          0, 0, 0.5, 1, //
        ) *
        Matrix4.rotationX(time) *
        Matrix4.rotationY(time * seedX) *
        Matrix4.rotationZ(time * seedY)));

    /// Bind the vertex and index buffer.
    encoder.bindVertexBuffer(vertices, 8);
    encoder.bindIndexBuffer(indices, gpu.IndexType.int16, 36);

    final mvpSlot = pipeline.vertexShader.getUniformSlot('mvp')!;
    encoder.bindUniform(mvpSlot, mvp);

    final sampledTexture = gpu.gpuContext.createTexture(
        gpu.StorageMode.hostVisible, 5, 5,
        enableShaderReadUsage: true);
    sampledTexture!.overwrite(uint32(<int>[
      0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, //
      0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, //
      0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, //
      0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, //
      0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, //
    ]));

    final texSlot = pipeline.fragmentShader.getUniformSlot('tex')!;
    encoder.bindTexture(texSlot, sampledTexture);

    /// And finally, we append a draw call.
    encoder.draw();

    /// Submit all of the previously encoded passes. Passes are encoded in the
    /// same order they were created in.
    commandBuffer.submit();

    /// Wrap the Flutter GPU texture as a ui.Image and draw it like normal!
    final image = renderTexture.asImage();

    canvas.drawImage(image, Offset(-renderTexture.width / 2, 0), Paint());
  }

TextureVertex

uniform mat4 mvp;

in vec3 position;
in vec2 texture_coords;
in vec4 color;
out vec2 v_texture_coords;
out vec4 v_color;

void main() {
  v_texture_coords = texture_coords;
  v_color = color;
  gl_Position = mvp * vec4(position, 1.0);
}

TextureFragment

uniform sampler2D tex;

in vec2 v_texture_coords;
in vec4 v_color;
out vec4 frag_color;

void main() {
  frag_color = v_color * texture(tex, v_texture_coords);
}

jonahwilliams

Some nits on the Dart API

Overall I dislike the pattern of validating in the constructor of an Object via thrown exception. Not sure what the best alternative would be.

jonahwilliams · 2023-11-26T20:37:27Z

lib/gpu/fixtures.h

+extern unsigned char kFlutterGPUTextureVertIPLR[];
+
+constexpr unsigned int kFlutterGPUTextureFragIPLRLength = 800;
+extern unsigned char kFlutterGPUTextureFragIPLR[];


Suggested change

extern unsigned char kFlutterGPUTextureFragIPLR[];

extern unsigned char kFlutterGPUTextureFragIPLR[];

jonahwilliams · 2023-11-26T20:39:48Z