Draw Quad with custom shader DrawCallback

[soufianekhiat]

Version/Branch of Dear ImGui:

master

Back-ends:

imgui_impl_dx11.cpp + imgui_impl_win32.cpp

Compiler, OS:

MSVC 2022

Full config/build information:

No response

Details:

Attempt to render a quad with a custom shader. I wasn't able to see how to hook my code with DrawCall back and the reset of RenderStates.
Here an example I tried with win32/dx11 backend.
This code is not working, and produce a gpu hang.

XY Problem: Various rendering function I had on DearWidgets rely on vertex shader interpolation, so for a given gradient I need lot of vertices. It will be a simpler (no brainer) to have a trivial pixel shader with 4 vertices instead.

Screenshots/Video:

No response

Minimal, Complete and Verifiable Example code:

typedef void* ImShaderID;
struct ImShader
{
    ImShaderID vs;
    ImShaderID ps;
};

bool CreateShader(ImShader& shader)
{
    ImGui_ImplDX10_Data* bd = ImGui_ImplDX10_GetData();
    if (!bd->pd3dDevice)
        return false;

    ID3D10VertexShader* pVertexShader = (ID3D10VertexShader*)shader.vs;
    ID3D10PixelShader* pPixelShader = (ID3D10PixelShader*)shader.ps;
    // Create the vertex shader
    {
        static const char* vertexShader =
            "cbuffer vertexBuffer : register(b0) \
            {\
              float4x4 ProjectionMatrix; \
            };\
            struct VS_INPUT\
            {\
              float2 pos : POSITION;\
              float4 col : COLOR0;\
              float2 uv  : TEXCOORD0;\
            };\
            \
            struct PS_INPUT\
            {\
              float4 pos : SV_POSITION;\
              float4 col : COLOR0;\
              float2 uv  : TEXCOORD0;\
            };\
            \
            PS_INPUT main(VS_INPUT input)\
            {\
              PS_INPUT output;\
              output.pos = mul( ProjectionMatrix, float4(input.pos.xy, 0.f, 1.f));\
              output.col = input.col;\
              output.uv  = input.uv;\
              return output;\
            }";

        ID3DBlob* vertexShaderBlob;
        if (FAILED(D3DCompile(vertexShader, strlen(vertexShader), nullptr, nullptr, nullptr, "main", "vs_4_0", 0, 0, &vertexShaderBlob, nullptr)))
            return false; // NB: Pass ID3DBlob* pErrorBlob to D3DCompile() to get error showing in (const char*)pErrorBlob->GetBufferPointer(). Make sure to Release() the blob!
        if (bd->pd3dDevice->CreateVertexShader(vertexShaderBlob->GetBufferPointer(), vertexShaderBlob->GetBufferSize(), &pVertexShader) != S_OK)
        {
            vertexShaderBlob->Release();
            return false;
        }
    }

    // Create the pixel shader
    {
        static const char* pixelShader =
            "struct PS_INPUT\
            {\
            float4 pos : SV_POSITION;\
            float4 col : COLOR0;\
            float2 uv  : TEXCOORD0;\
            };\
            sampler sampler0;\
            Texture2D texture0;\
            \
            float4 main(PS_INPUT input) : SV_Target\
            {\
            float4 out_col = float4(0.0f, 1.0f, 0.0f, 1.0f); \
            return out_col; \
            }";

        ID3DBlob* pixelShaderBlob;
        if (FAILED(D3DCompile(pixelShader, strlen(pixelShader), nullptr, nullptr, nullptr, "main", "ps_4_0", 0, 0, &pixelShaderBlob, nullptr)))
            return false; // NB: Pass ID3DBlob* pErrorBlob to D3DCompile() to get error showing in (const char*)pErrorBlob->GetBufferPointer(). Make sure to Release() the blob!
        if (bd->pd3dDevice->CreatePixelShader(pixelShaderBlob->GetBufferPointer(), pixelShaderBlob->GetBufferSize(), &pPixelShader) != S_OK)
        {
            pixelShaderBlob->Release();
            return false;
        }
        pixelShaderBlob->Release();
    }

    shader.vs = (ImShaderID)pVertexShader;
    shader.ps = (ImShaderID)pPixelShader;

    return true;
}

struct ImDrawCallData
{
    ImVector<ImDrawVert> vtx;
    ImVector<ImDrawIdx> idx;
    ImShader shader;
    bool is_dirty = true;
};

ImDrawCallData drawCall;

void ImDrawCallbackCustomShader(const ImDrawList* parent_list, const ImDrawCmd* pcmd)
{
    ImGui_ImplDX10_Data* bd = ImGui_ImplDX10_GetData();
    ID3D10Device* ctx = bd->pd3dDevice;

    ImDrawData* draw_data = ImGui::GetDrawData();
    ImVec2 position = draw_data->DisplayPos;

    ImGuiContext* context = ImGui::GetCurrentContext();

    ImGuiPlatformIO& platformIO = context->PlatformIO;
    ImGuiViewport* viewport = platformIO.Viewports.front();

    static ID3D10Buffer* pVB = NULL;
    static ID3D10Buffer* pIB = NULL;

    if ( pVB == NULL )
    {
        D3D10_BUFFER_DESC desc;
        memset(&desc, 0, sizeof(D3D10_BUFFER_DESC));
        desc.Usage = D3D10_USAGE_DYNAMIC;
        desc.ByteWidth = drawCall.vtx.size() * sizeof(ImDrawVert);
        desc.BindFlags = D3D10_BIND_VERTEX_BUFFER;
        desc.CPUAccessFlags = D3D10_CPU_ACCESS_WRITE;
        desc.MiscFlags = 0;
        if (ctx->CreateBuffer(&desc, nullptr, &pVB) < 0)
            return;
    }
    if ( pIB == NULL )
    {
        D3D10_BUFFER_DESC desc;
        memset(&desc, 0, sizeof(D3D10_BUFFER_DESC));
        desc.Usage = D3D10_USAGE_DYNAMIC;
        desc.ByteWidth = drawCall.idx.size() * sizeof(ImDrawIdx);
        desc.BindFlags = D3D10_BIND_INDEX_BUFFER;
        desc.CPUAccessFlags = D3D10_CPU_ACCESS_WRITE;
        if (ctx->CreateBuffer(&desc, nullptr, &pIB) < 0)
            return;
    }

    if (drawCall.is_dirty)
    {
        ImDrawVert* vtx_dst = nullptr;
        ImDrawIdx* idx_dst = nullptr;
        pVB->Map(D3D10_MAP_WRITE_DISCARD, 0, (void**)&vtx_dst);
        pIB->Map(D3D10_MAP_WRITE_DISCARD, 0, (void**)&idx_dst);
        memcpy(vtx_dst, drawCall.vtx.Data, drawCall.vtx.Size * sizeof(ImDrawVert));
        memcpy(idx_dst, drawCall.idx.Data, drawCall.idx.Size * sizeof(ImDrawIdx));
        pVB->Unmap();
        pIB->Unmap();
        drawCall.is_dirty = false;
    }

    unsigned int stride = sizeof(ImDrawVert);
    unsigned int offset = 0;
    ctx->IASetVertexBuffers(0, 1, &pVB, &stride, &offset);
    ctx->IASetIndexBuffer(pIB, sizeof(ImDrawIdx) == 2 ? DXGI_FORMAT_R16_UINT : DXGI_FORMAT_R32_UINT, 0);
    ctx->IASetPrimitiveTopology(D3D10_PRIMITIVE_TOPOLOGY_TRIANGLELIST);
    ctx->VSSetShader((ID3D10VertexShader*)drawCall.shader.vs);
    ctx->PSSetShader((ID3D10PixelShader*)drawCall.shader.ps);

    ctx->DrawIndexed(drawCall.idx.size(), 0, 0);
}

void DrawQuadWithCustomShader( ImDrawList* drawList, ImShader& sr )
{
    ImU32 col = IM_COL32(255, 0, 0, 255);
    ImVec2 a(0.0f, 0.0f), c(255.0f, 255.0f);
    ImVec2  b(c.x, a.y), d(a.x, c.y), uv(ImGui::GetFontTexUvWhitePixel());

    if (drawCall.is_dirty)
    {
        drawCall.idx.resize(6);
        drawCall.idx[0] = (ImDrawIdx)(0);
        drawCall.idx[1] = (ImDrawIdx)(1);
        drawCall.idx[2] = (ImDrawIdx)(2);
        drawCall.idx[3] = (ImDrawIdx)(0);
        drawCall.idx[4] = (ImDrawIdx)(2);
        drawCall.idx[5] = (ImDrawIdx)(3);
        drawCall.vtx.resize(4);
        drawCall.vtx[0].pos = a; drawCall.vtx[0].uv = uv; drawCall.vtx[0].col = col;
        drawCall.vtx[1].pos = b; drawCall.vtx[1].uv = uv; drawCall.vtx[1].col = col;
        drawCall.vtx[2].pos = c; drawCall.vtx[2].uv = uv; drawCall.vtx[2].col = col;
        drawCall.vtx[3].pos = d; drawCall.vtx[3].uv = uv; drawCall.vtx[3].col = col;
    }

    drawList->AddCallback(&ImDrawCallbackCustomShader, &drawCall);
    drawList->AddCallback((ImDrawCallback)ImDrawCallback_ResetRenderState, NULL);
}

// In code
            ImGui::Begin("Custom Quad");
            DrawQuadWithCustomShader(ImGui::GetWindowDrawList(), shaders);
            ImGui::End();

[ocornut]

This code is not working, and produce a gpu hang.

I'm not sure how we can help, it's seems likely a problem with your use of DX11.

It will be a simpler (no brainer) to have a trivial pixel shader with 4 vertices instead.

It doesn't seem simpler to me IMHO because you are going to need lots of shader variants.
But maybe you can find a way to let dear imgui generate the mesh and only act on the shader.

[soufianekhiat]

I'm not sure how we can help, it's seems likely a problem with your use of DX11.

If it's dx11 problem I can work on it. I never saw a proper usage of draw callback I'm not sure of the proper use.

It doesn't seem simpler to me IMHO because you are going to need lots of shader variants.
But maybe you can find a way to let dear imgui generate the mesh and only act on the shader.

True! More shader variant, but traded for more flexibility, for a simple linear gradient I need lot of vertices (based on where the gradient start end).
What do you mean by the last sentence?
Current I had to tesselate the geometry
https://github.com/soufianekhiat/DearWidgets/blob/rework/src/api/dear_widgets.cpp#L2078
From a simple geometry:
https://github.com/soufianekhiat/DearWidgets/blob/rework/src/api/dear_widgets.cpp#L2355
To have a sense of a simple linear gradient:
https://github.com/soufianekhiat/DearWidgets/blob/rework/src/api/dear_widgets.cpp#L2439

[soufianekhiat]

Here an illustration of the problem:

[ocornut]

You should probably use RenderDoc or another gpu debugger to understand why your code is crashing.

What do you mean by the last sentence?

DrawQuadWithCustomShader output a simple quad which can already be output by ImDrawList, so maybe you can simply your callback logic to set shader + some uniforms for your shader, and not attempt to create/bind buffers.

AddCallback(bind your shader and data)
AddRectFilled()
AddCallback(ImDrawCallback_ResetRenderState)

[soufianekhiat]

[Fixed]

ImGui::Begin( "Custom Shader" );
	ImDrawList* draw = ImGui::GetWindowDrawList();
	ImVec2 cur = ImGui::GetCursorScreenPos();
	draw->AddCallback( &DrawCustomShaderQuad, &shader );
	ImRect bb( cur, cur + ImGui::GetContentRegionAvail() );
	draw->AddImageQuad( img, bb.GetBL(), bb.GetBR(), bb.GetTR(), bb.GetTL(), ImVec2( 0, 0 ), ImVec2( 1, 0 ), ImVec2( 1, 1 ), ImVec2( 0, 1 ), IM_COL32_WHITE );
	draw->AddCallback( ImDrawCallback_ResetRenderState, NULL );
ImGui::End();

Your second recommendation is much simple, it simplify the memory management etc. And rely on unique Vertex Buffer.

void DrawCustomShaderQuad( const ImDrawList* parent_list, const ImDrawCmd* cmd )
{
	ImPlatform::ImShader* shaders = ( ImPlatform::ImShader* )cmd->UserCallbackData;
	ImGui_ImplDX11_Data* bd = ImGui_ImplDX11_GetBackendData();
	ID3D11DeviceContext* ctx = bd->pd3dDeviceContext;
	ctx->PSSetShader( ( ID3D11PixelShader* )( shaders->ps ), nullptr, 0 );
}

[ocornut]

I guess you’ll need to pass some extra data to your shader but that’s the general idea yes (as long as the vertex format used by ImDrawList is sufficient for you).

[soufianekhiat]

Yes indeed It need extra for to add more data etc.
https://github.com/soufianekhiat/ImPlatform/blob/main/ImPlatform/ImPlatform.cpp#L1173
If I need another Vertex Format I'll need to create my own data struct for draw call, and by pass the ImDrawList pass.

Interpretation of that available on ImPlatform:
https://github.com/soufianekhiat/ImPlatform/blob/main/ImPlatformDemo/main.cpp#L106
https://github.com/soufianekhiat/ImPlatform/blob/main/ImPlatformDemo/main.cpp#L201
https://github.com/soufianekhiat/ImPlatform/blob/main/ImPlatformDemo/main.cpp#L201
Hidden under 'IM_SUPPORT_CUSTOM_SHADER' while I don't support "enough" backend.

[soufianekhiat]

Sidenote to have it portable for GLSL and GLSL I prefix all shaders with:
https://github.com/soufianekhiat/ImPlatform/blob/main/ImPlatform/ImPlatform.cpp#L1016

[ocornut]

Good to hear. Do you need anything else on my end or can we close this?