Global scan #665

kpentaris · 2024-03-11T18:34:43Z

Description

Initial implementation of the Global Scan in HLSL

Testing

Not yet finished

TODO list:

Still need to test the HLSL code as well as the porting of the example code to the new API of vulkan_1_3 branch

…but not working yet

devshgraphicsprogrammingjenkins · 2024-03-11T18:35:02Z

[CI]: Can one of the admins verify this patch?

…al_scan

devshgraphicsprogramming · 2024-04-02T09:42:06Z

include/nbl/builtin/hlsl/scan/declarations.hlsl

+#ifndef NBL_BUILTIN_MAX_SCAN_LEVELS
+#define NBL_BUILTIN_MAX_SCAN_LEVELS 7
+#endif


do as NBL_CONSTEXPR instead of define

also this better be formulated slightly differently, in terms of reduction passes, not reduction + downsweep

devshgraphicsprogramming · 2024-04-02T09:42:30Z

include/nbl/builtin/hlsl/scan/declarations.hlsl

+		uint32_t topLevel;
+		uint32_t temporaryStorageOffset[NBL_BUILTIN_MAX_SCAN_LEVELS/2];


use uint16_t for this

devshgraphicsprogramming · 2024-04-02T09:42:56Z

include/nbl/builtin/hlsl/scan/declarations.hlsl

+		uint32_t topLevel;
+		uint32_t temporaryStorageOffset[NBL_BUILTIN_MAX_SCAN_LEVELS/2];
+	};
+
 	Parameters_t getParameters();


no need for such a forward decl anymore

devshgraphicsprogramming · 2024-04-02T09:52:12Z

include/nbl/builtin/hlsl/scan/declarations.hlsl

+    struct DefaultSchedulerParameters_t
+    {
+        uint32_t finishedFlagOffset[NBL_BUILTIN_MAX_SCAN_LEVELS-1];
+        uint32_t cumulativeWorkgroupCount[NBL_BUILTIN_MAX_SCAN_LEVELS];
+
+    };
+
+    DefaultSchedulerParameters_t getSchedulerParameters();


split out schedulers into separate headers

devshgraphicsprogramming · 2024-04-02T09:53:37Z

include/nbl/builtin/hlsl/scan/declarations.hlsl

 	template<typename Storage_t>
 	void getData(
-		inout Storage_t data,
-		in uint levelInvocationIndex,
-		in uint localWorkgroupIndex,
-		in uint treeLevel,
-		in uint pseudoLevel
+		NBL_REF_ARG(Storage_t) data,
+		NBL_CONST_REF_ARG(uint32_t) levelInvocationIndex,
+		NBL_CONST_REF_ARG(uint32_t) localWorkgroupIndex,
+		NBL_CONST_REF_ARG(uint32_t) treeLevel,
+		NBL_CONST_REF_ARG(uint32_t) pseudoLevel
 	);
-}
-}
-}
-#define _NBL_HLSL_SCAN_GET_PADDED_DATA_DECLARED_
-#endif

-#ifndef _NBL_HLSL_SCAN_SET_DATA_DECLARED_
-namespace nbl
-{
-namespace hlsl
-{
-namespace scan
-{
 	template<typename Storage_t>
 	void setData(
-		in Storage_t data,
-		in uint levelInvocationIndex,


no more forward declarations, just let it be an accessor.

Also the level const-ref-args can be rolled up into a single struct SDataIndex

devshgraphicsprogramming · 2024-04-02T09:56:28Z

include/nbl/builtin/hlsl/scan/descriptors.hlsl

+// Copyright (C) 2023 - DevSH Graphics Programming Sp. z O.O.
+// This file is part of the "Nabla Engine".
+// For conditions of distribution and use, see copyright notice in nabla.h

+#ifndef _NBL_HLSL_SCAN_DESCRIPTORS_INCLUDED_
+#define _NBL_HLSL_SCAN_DESCRIPTORS_INCLUDED_

-// choerent -> globallycoherent
+#include "nbl/builtin/hlsl/scan/declarations.hlsl"
+#include "nbl/builtin/hlsl/workgroup/basic.hlsl"
+
+// coherent -> globallycoherent
+
+namespace nbl
+{
+namespace hlsl
+{
+namespace scan
+{
+
+template<uint32_t scratchElementCount=scratchSz> // (REVIEW): This should be externally defined. Maybe change the scratch buffer to RWByteAddressBuffer? Annoying to manage though...
+struct Scratch
+{
+    uint32_t workgroupsStarted;
+    uint32_t data[scratchElementCount];
+};
+
+[[vk::binding(0 ,0)]] RWStructuredBuffer<uint32_t /*Storage_t*/> scanBuffer; // (REVIEW): Make the type externalizable. Decide how (#define?)
+[[vk::binding(1 ,0)]] RWStructuredBuffer<Scratch> globallycoherent scanScratchBuf; // (REVIEW): Check if globallycoherent can be used with Vulkan Mem Model
+
+template<typename Storage_t, bool isExclusive=false>
+void getData(
+    NBL_REF_ARG(Storage_t) data,
+    NBL_CONST_REF_ARG(uint32_t) levelInvocationIndex,
+    NBL_CONST_REF_ARG(uint32_t) localWorkgroupIndex,
+    NBL_CONST_REF_ARG(uint32_t) treeLevel,
+    NBL_CONST_REF_ARG(uint32_t) pseudoLevel
+)
+{
+    const Parameters_t params = getParameters(); // defined differently for direct and indirect shaders
+
+    uint32_t offset = levelInvocationIndex;
+    const bool notFirstOrLastLevel = bool(pseudoLevel);
+    if (notFirstOrLastLevel)
+		offset += params.temporaryStorageOffset[pseudoLevel-1u];
+


remove this whole file, it should be userspace

devshgraphicsprogramming · 2024-04-02T09:57:12Z

include/nbl/builtin/hlsl/scan/direct.hlsl

+// TODO: Can we make it a static variable?
+groupshared uint32_t wgScratch[SharedScratchSz];
+
+#include "nbl/builtin/hlsl/workgroup/arithmetic.hlsl"
+
+template<uint16_t offset>
+struct WGScratchProxy
+{
+	uint32_t get(const uint32_t ix)
+	{
+		return wgScratch[ix+offset];
+	}
+	void set(const uint32_t ix, const uint32_t value)
+	{
+		wgScratch[ix+offset] = value;
+	}
+
+    uint32_t atomicAdd(uint32_t ix, uint32_t val)
+    {
+        return glsl::atomicAdd(wgScratch[ix + offset], val);
+    }
+
+	void workgroupExecutionAndMemoryBarrier()
+	{
+		nbl::hlsl::glsl::barrier();
+		//nbl::hlsl::glsl::memoryBarrierShared(); implied by the above
+	}
+};
+static WGScratchProxy<0> accessor;


scratches are userspace

use accessors

devshgraphicsprogramming · 2024-04-02T09:57:35Z

include/nbl/builtin/hlsl/scan/direct.hlsl

+/**
+ * Required since we rely on SubgroupContiguousIndex instead of 
+ * gl_LocalInvocationIndex which means to match the global index 
+ * we can't use the gl_GlobalInvocationID but an index based on 
+ * SubgroupContiguousIndex.
+ */
+uint32_t globalIndex()
+{
+	return nbl::hlsl::glsl::gl_WorkGroupID().x*WORKGROUP_SIZE+nbl::hlsl::workgroup::SubgroupContiguousIndex();
+}


we have this in a header already

devshgraphicsprogramming · 2024-04-02T09:57:53Z

include/nbl/builtin/hlsl/scan/direct.hlsl

+struct ScanPushConstants
+{
+    nbl::hlsl::scan::Parameters_t scanParams;
+    nbl::hlsl::scan::DefaultSchedulerParameters_t schedulerParams;
+};
+
+[[vk::push_constant]]
+ScanPushConstants spc;


everything affecting the pipeline layout should be userspace

devshgraphicsprogramming · 2024-04-02T09:58:52Z

include/nbl/builtin/hlsl/scan/indirect.hlsl

-#ifndef _NBL_HLSL_MAIN_DEFINED_
-[numthreads(_NBL_HLSL_WORKGROUP_SIZE_, 1, 1)]
-void CSMain()
+[numthreads(WORKGROUP_SIZE,1,1)]
+void main()
 {
-	if (bool(nbl::hlsl::scan::getIndirectElementCount()))
-		nbl::hlsl::scan::main();
+    if(bool(nbl::hlsl::scan::getIndirectElementCount())) {
+        // TODO call main from virtual_workgroup.hlsl
+    }


all this should be userspace, also there will be very little difference between direct and indirect

devshgraphicsprogramming · 2024-04-02T10:03:20Z

include/nbl/builtin/hlsl/scan/virtual_workgroup.hlsl

 namespace nbl
 {
 namespace hlsl
 {
 namespace scan
 {
-	template<class Binop, class Storage_t>
-	void virtualWorkgroup(in uint treeLevel, in uint localWorkgroupIndex)
+	template<class Binop, typename Storage_t, bool isExclusive, uint16_t ItemCount, class Accessor, class device_capabilities=void>


why not template on the workgroup scan instead?

Within you can alias and extract:

binop

storage_t

exclusive or not

item count

smem accessor

device traits necessary

devshgraphicsprogramming · 2024-04-02T10:04:02Z

include/nbl/builtin/hlsl/scan/virtual_workgroup.hlsl

+		const uint32_t levelInvocationIndex = localWorkgroupIndex * glsl::gl_WorkGroupSize().x + SubgroupContiguousIndex();
+		const bool lastInvocationInGroup = SubgroupContiguousIndex() == (gl_WorkGroupSize().x - 1);


shouldn't ItemCount be used instead of glsl::gl_WorkGroupSize().x ?

most definitely

devshgraphicsprogramming · 2024-06-16T07:36:49Z

include/nbl/builtin/hlsl/scan/default_scheduler.hlsl

-			default:
-				break;
-#if NBL_BUILTIN_MAX_SCAN_LEVELS>7
+        const uint32_t workgroupSizeLog2 = firstbithigh(glsl::gl_WorkGroupSize().x);


modify it to not use Vulkan/SPIR-V environment builtins, that way we can use it from C++ too!

(So take workgroup size - or more accurately item per workgroup count - from the outside)

devshgraphicsprogramming · 2024-06-16T07:37:55Z

include/nbl/builtin/hlsl/scan/declarations.hlsl

-#endif
+    // REVIEW: Putting topLevel second allows better alignment for packing of constant variables, assuming lastElement has length 4. (https://learn.microsoft.com/en-us/windows/win32/direct3dhlsl/dx-graphics-hlsl-packing-rules)
+    struct Parameters_t {
+        uint32_t lastElement[NBL_BUILTIN_MAX_LEVELS/2+1];


add documentation about what lastElement is used for

devshgraphicsprogramming · 2024-06-16T07:54:30Z

include/nbl/builtin/hlsl/scan/default_scheduler.hlsl

+                _schedulerParams.finishedFlagOffset[1] = 1u;
+
+                _scanParams.temporaryStorageOffset[0] = 2u;
+                break;
+            case 2u:
+                _schedulerParams.cumulativeWorkgroupCount[1] = _schedulerParams.cumulativeWorkgroupCount[0]+WorkgroupCount(1);
+                _schedulerParams.cumulativeWorkgroupCount[2] = _schedulerParams.cumulativeWorkgroupCount[1]+1u;
+                _schedulerParams.cumulativeWorkgroupCount[3] = _schedulerParams.cumulativeWorkgroupCount[2]+WorkgroupCount(1);
+                _schedulerParams.cumulativeWorkgroupCount[4] = _schedulerParams.cumulativeWorkgroupCount[3]+WorkgroupCount(0);
+                // climb up
+                _schedulerParams.finishedFlagOffset[1] = WorkgroupCount(1);
+                _schedulerParams.finishedFlagOffset[2] = _schedulerParams.finishedFlagOffset[1]+1u;
+                // climb down
+                _schedulerParams.finishedFlagOffset[3] = _schedulerParams.finishedFlagOffset[1]+2u;
+
+                _scanParams.temporaryStorageOffset[0] = _schedulerParams.finishedFlagOffset[3]+WorkgroupCount(1);
+                _scanParams.temporaryStorageOffset[1] = _scanParams.temporaryStorageOffset[0]+WorkgroupCount(0);
+                break;
+            case 3u:
+                _schedulerParams.cumulativeWorkgroupCount[1] = _schedulerParams.cumulativeWorkgroupCount[0]+WorkgroupCount(1);
+                _schedulerParams.cumulativeWorkgroupCount[2] = _schedulerParams.cumulativeWorkgroupCount[1]+WorkgroupCount(2);
+                _schedulerParams.cumulativeWorkgroupCount[3] = _schedulerParams.cumulativeWorkgroupCount[2]+1u;


because due to forward progress guarantees you can only do upsweep or downsweep in a single dispatch, you don't need the "climb down"

you probably don't even need the cumulative workgroup counts, as the way you'd find out if your current workgroup truly is the last one, is by checking the return value of the finished flag atomic counter.

e.g. markFinished

uint32_t howManyDone = flagsAccessor.atomicAdd(finishedOffset[level]+virtualWorkgroupIndexInThisLevel); //uint32_t currentLastWorkgroup = currentLastElement>>ItemsPerWorkgroupLog2; uint32_t nextLevelLastWorkgroup = currentLastWorkgroup >>ItemsPerWorkgroupLog2; uint32_t virtualWorkgroupIndexInNextLevel = virtualWorkgroupIndexInThisLevel>>ItemsPerWorkgroupLog2; // NOT: either last workgroup, or in the last bundle of workgroups and last workgroup dynamically there if (howManyDone!=(ItemsPerWorkgroup-1) && (virtualWorkgroupIndexInNextLevel!=nextLevelLastWorkgroup || (virtualWorkgroupIndexInThisLevel&(ItemsPerWorkgroupLog2-1))!=howManyDone) ) return; // last one takes over currentLastWorkgroup = nextLevelLastWorkgroup ;

btw I'd try to rewrite this as a loop instead of a weird switch that was only written like this to avoid recursion in GLSL

devshgraphicsprogramming · 2024-06-16T16:20:06Z

include/nbl/builtin/hlsl/scan/default_scheduler.hlsl

+    /**
+     * treeLevel - the current level in the Blelloch Scan
+     * localWorkgroupIndex - the workgroup index the current invocation is a part of in the specific virtual dispatch. 
+     * For example, if we have dispatched 10 workgroups and we the virtu    al workgroup number is 35, then the localWorkgroupIndex should be 5.
+     */
+    template<class Accessor>
+    bool getWork(NBL_CONST_REF_ARG(DefaultSchedulerParameters_t) params, NBL_CONST_REF_ARG(uint32_t) topLevel, NBL_REF_ARG(uint32_t) treeLevel, NBL_REF_ARG(uint32_t) localWorkgroupIndex, NBL_REF_ARG(Accessor) sharedScratch)
+    {


you can't do any flagging via sharedmemory because you want inter-workgroup communication, must be a BDA address!

even better a bda::__ptr<uint32_t> cause then you'll have atomicAdd methods

devshgraphicsprogramming · 2024-06-16T16:21:03Z

include/nbl/builtin/hlsl/scan/default_scheduler.hlsl

+		sharedScratch.workgroupExecutionAndMemoryBarrier();
+		const uint32_t globalWorkgroupIndex = sharedScratch.get(0u);
+
+        treeLevel = sharedScratch.get(1u);
+        if(treeLevel>lastLevel)
+            return true;
+
+        localWorkgroupIndex = globalWorkgroupIndex;
+        const bool dependentLevel = treeLevel != 0u;
+        if(dependentLevel) 
+        {
+            const uint32_t prevLevel = treeLevel - 1u;
+            localWorkgroupIndex -= params.cumulativeWorkgroupCount[prevLevel];
+            if(workgroup::SubgroupContiguousIndex() == 0u) 
+            {
+                uint32_t dependentsCount = 1u;
+                if(treeLevel <= topLevel) 
+                {
+                    dependentsCount = glsl::gl_WorkGroupSize().x;
+                    const bool lastWorkgroup = (globalWorkgroupIndex+1u)==params.cumulativeWorkgroupCount[treeLevel];
+                    if (lastWorkgroup) 
+                    {
+                        const Parameters_t scanParams = getParameters();
+                        dependentsCount = scanParams.lastElement[treeLevel]+1u;
+                        if (treeLevel<topLevel) 
+                        {
+                            dependentsCount -= scanParams.lastElement[treeLevel+1u] * glsl::gl_WorkGroupSize().x;
+                        }
+                    }
+                }
+                uint32_t dependentsFinishedFlagOffset = localWorkgroupIndex;
+                if (treeLevel>topLevel) // !(prevLevel<topLevel) TODO: merge with `else` above?
+                    dependentsFinishedFlagOffset /= glsl::gl_WorkGroupSize().x;
+                dependentsFinishedFlagOffset += params.finishedFlagOffset[prevLevel];
+                while (scanScratchBuf[0].data[dependentsFinishedFlagOffset]!=dependentsCount)
+                    glsl::memoryBarrierBuffer();


I expect a rewrite of this into "last one out closes the door"

devshgraphicsprogramming · 2024-06-16T16:21:43Z

include/nbl/builtin/hlsl/scan/default_scheduler.hlsl

+            if (treeLevel<topLevel)
+            {
+                finishedFlagOffset += localWorkgroupIndex/glsl::gl_WorkGroupSize().x;
+                glsl::atomicAdd(scanScratchBuf[0].data[finishedFlagOffset], 1u);
+            }
+            else if (treeLevel!=(topLevel<<1u))
+            {
+                finishedFlagOffset += localWorkgroupIndex;
+                scanScratchBuf[0].data[finishedFlagOffset] = 1u;
+            }


comments about what and why is happenning in the if statements

devshgraphicsprogramming · 2024-06-16T16:23:51Z

include/nbl/builtin/hlsl/scan/scheduler.hlsl

+  template<class Accessor>
+  bool getWork(NBL_REF_ARG(Accessor) accessor)


whats the accessor for? needs a better name

Also why don't you store the accessor past the create as a member?

It should now work with all sizes of inputs.

… global_scan

…nges

devshgraphicsprogramming · 2024-07-10T16:04:54Z

include/nbl/builtin/hlsl/scan/scheduler.hlsl

+        // could do scanScratchBuf[0u].workgroupsStarted[SubgroupContiguousIndex()] = 0u but don't know how many invocations are live during this call
+        if(workgroup::SubgroupContiguousIndex() == 0u)
+        {
+            for(uint32_t i = 0; i < params.topLevel; i++)
+            {
+                scanScratchBuf[0u].workgroupsStarted[i] = 0u;
+            }
+        }


don't you know the workgroup size?

My bad, this is actually not used anymore. I think it was being used inside an inRage at some point? Not sure. However we can't reset the workgroupStarted buffer within the shader at the end of the Reduce step (which was the initial goal) because it's possible that there are still workgroups that are "alive" where they won't be doing any work but will still increase the workgroupStarted buffer and some times we end up doing the reset before those WGs exit, ending up with some 1 values instead of all 0s.

kpentaris added 8 commits January 29, 2024 12:51

Merge branch 'atomicAdd-fix' into global_scan

a34827a

First commit for global scan implementation. Ported most of the code …

d61a2aa

…but not working yet

Global scan migration of code to new APIs

0eec0ee

Merge branch 'Devsh-Graphics-Programming:master' into global_scan

2185b4e

First commit for global scan implementation. Ported most of the code …

2df4ad7

…but not working yet

Global scan migration of code to new APIs

20a60f5

Merge remote

960dbf2

Merge branch 'Devsh-Graphics-Programming:master' into global_scan

2be8f13

kpentaris added 4 commits March 15, 2024 11:21

Merge branch 'master' into global_scan

6366697

Merge branch 'master' into global_scan

26cb75d

Update CScanner.h to new NBL API

7117cc3

Merge branch 'master' of https://github.com/kpentaris/Nabla into glob…

fa979c7

…al_scan

devshgraphicsprogramming reviewed Apr 2, 2024

View reviewed changes

kpentaris added 5 commits April 4, 2024 00:40

Merge branch 'master' into global_scan

53e1655

Merge branch 'master' into global_scan

0bcc325

Merge branch 'master' into global_scan

71f4398

Fix CScanner and related shaders to properly compile

51b4f74

Merge branch 'master' into global_scan

4b29049

devshgraphicsprogramming reviewed Jun 16, 2024

View reviewed changes

kpentaris added 9 commits June 17, 2024 11:11

Fix issues with global reduce algorithm

40953e3

It should now work with all sizes of inputs.

Merge branch 'Devsh-Graphics-Programming:master' into global_scan

cc43691

Merge branch 'master' of https://github.com/kpentaris/Nabla

3a5d1ff

Merge branch 'master' into global_scan

658ac5b

Merge branch 'global_scan' of https://github.com/kpentaris/Nabla into…

d4a947d

… global_scan

Merge upstream master

03ae90a

refactor global reduce to properly work with required global scan cha…

3d252d0

…nges

Remove unused pseudoLevel parameter

dd2cc09

Fix workgroupFinishFlagsOffset[0] being twice the needed size

ce78fc0

devshgraphicsprogramming reviewed Jul 10, 2024

View reviewed changes

kpentaris added 13 commits July 21, 2024 14:46

Merge upstream master

f6d7adb

Revert .gitmodules pointing to examples fork

0d71686

Merge master

3ce449d

Merge master

6a0c6c6

merge master

e467ece

Merge remote branch

5dae823

Merge master to global_scan branch

79ec513

merge upstream Nable to local

7f4fcd5

Merge upstream to local

462fec5

merge upstream master to branch

3f9bdd8

update 3rdparty to point to master submodules

201636d

update the hlsl/scan module paths to CMakeLists

21bfa0f

merge submodules changes to branch

1c32528

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Global scan #665

Global scan #665

kpentaris commented Mar 11, 2024

devshgraphicsprogrammingjenkins commented Mar 11, 2024

devshgraphicsprogramming Apr 2, 2024

devshgraphicsprogramming Apr 2, 2024

devshgraphicsprogramming Apr 2, 2024

devshgraphicsprogramming Apr 2, 2024

devshgraphicsprogramming Apr 2, 2024

devshgraphicsprogramming Apr 2, 2024

devshgraphicsprogramming Apr 2, 2024 •

edited

Loading

devshgraphicsprogramming Apr 2, 2024

devshgraphicsprogramming Jun 14, 2024

devshgraphicsprogramming Apr 2, 2024

devshgraphicsprogramming Apr 2, 2024

devshgraphicsprogramming Apr 2, 2024

devshgraphicsprogramming Apr 2, 2024

devshgraphicsprogramming Apr 2, 2024

devshgraphicsprogramming Jun 14, 2024

devshgraphicsprogramming Jun 16, 2024

devshgraphicsprogramming Jun 16, 2024

devshgraphicsprogramming Jun 16, 2024 •

edited

Loading

devshgraphicsprogramming Jun 16, 2024

devshgraphicsprogramming Jun 16, 2024

devshgraphicsprogramming Jun 16, 2024

devshgraphicsprogramming Jun 16, 2024

devshgraphicsprogramming Jun 16, 2024

devshgraphicsprogramming Jun 16, 2024

devshgraphicsprogramming Jul 10, 2024

kpentaris Jul 10, 2024

		uint32_t topLevel;
		uint32_t temporaryStorageOffset[NBL_BUILTIN_MAX_SCAN_LEVELS/2];

		const uint32_t levelInvocationIndex = localWorkgroupIndex * glsl::gl_WorkGroupSize().x + SubgroupContiguousIndex();
		const bool lastInvocationInGroup = SubgroupContiguousIndex() == (gl_WorkGroupSize().x - 1);

		template<class Accessor>
		bool getWork(NBL_REF_ARG(Accessor) accessor)

Global scan #665

Are you sure you want to change the base?

Global scan #665

Conversation

kpentaris commented Mar 11, 2024

Description

Testing

TODO list:

devshgraphicsprogrammingjenkins commented Mar 11, 2024

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

devshgraphicsprogramming Apr 2, 2024 • edited Loading

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

devshgraphicsprogramming Jun 16, 2024 • edited Loading

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

devshgraphicsprogramming Apr 2, 2024 •

edited

Loading

devshgraphicsprogramming Jun 16, 2024 •

edited

Loading