Skip to content

Commit

Permalink
Vulkan: Use timeline semaphores for synchronization if possible (o3de…
Browse files Browse the repository at this point in the history
…#17761)

Signed-off-by: Martin Sattlecker <martin.sattlecker@huawei.com>
Co-authored-by: Martin Sattlecker <martin.sattlecker@huawei.com>
Co-authored-by: Joerg H. Mueller <joerg.mueller@huawei.com>
Co-authored-by: Martin Winter <martin.winter@huawei.com>
  • Loading branch information
4 people committed May 6, 2024
1 parent 15e7640 commit d9b7dbe
Show file tree
Hide file tree
Showing 27 changed files with 994 additions and 409 deletions.
123 changes: 123 additions & 0 deletions Gems/Atom/RHI/Vulkan/Code/Source/RHI/BinaryFence.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include <Atom/RHI.Reflect/VkAllocator.h>
#include <Atom/RHI.Reflect/Vulkan/Conversion.h>
#include <RHI/Device.h>
#include <RHI/BinaryFence.h>

namespace AZ
{
namespace Vulkan
{
RHI::Ptr<FenceBase> BinaryFence::Create(Fence& fence)
{
return aznew BinaryFence(fence);
}

VkFence BinaryFence::GetNativeFence() const
{
return m_nativeFence;
}

void BinaryFence::SetNameInternal(const AZStd::string_view& name)
{
if (IsInitialized() && !name.empty())
{
Debug::SetNameToObject(
reinterpret_cast<uint64_t>(m_nativeFence), name.data(), VK_OBJECT_TYPE_FENCE, static_cast<Device&>(GetDevice()));
}
}

RHI::ResultCode BinaryFence::InitInternal(RHI::Device& baseDevice, RHI::FenceState initialState)
{
RETURN_RESULT_IF_UNSUCCESSFUL(Base::InitInternal(baseDevice, initialState));
auto& device = static_cast<Device&>(baseDevice);

VkFenceCreateInfo createInfo{};
createInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
createInfo.pNext = nullptr;
switch (initialState)
{
case RHI::FenceState::Reset:
createInfo.flags = 0;
break;
case RHI::FenceState::Signaled:
createInfo.flags = VK_FENCE_CREATE_SIGNALED_BIT;
break;
default:
return RHI::ResultCode::InvalidArgument;
}

const VkResult result =
device.GetContext().CreateFence(device.GetNativeDevice(), &createInfo, VkSystemAllocator::Get(), &m_nativeFence);
AssertSuccess(result);

RETURN_RESULT_IF_UNSUCCESSFUL(ConvertResult(result));
m_signalEvent = {};
SetName(GetName());
return RHI::ResultCode::Success;
}

void BinaryFence::ShutdownInternal()
{
if (m_nativeFence != VK_NULL_HANDLE)
{
auto& device = static_cast<Device&>(GetDevice());
device.GetContext().DestroyFence(device.GetNativeDevice(), m_nativeFence, VkSystemAllocator::Get());
m_nativeFence = VK_NULL_HANDLE;
}
Base::ShutdownInternal();
}

void BinaryFence::SignalOnCpuInternal()
{
// Vulkan doesn't have an API to signal fences from CPU.
// Because of this we need to use a queue to signal the fence.
auto& device = static_cast<Device&>(GetDevice());
device.GetCommandQueueContext().GetCommandQueue(RHI::HardwareQueueClass::Graphics).Signal(m_fence);
}

void BinaryFence::WaitOnCpuInternal() const
{
// According to the standard we can't wait until the event (like VkSubmitQueue) happens first.
if (m_signalEvent)
{
m_signalEvent->Wait(m_waitDependencies);
}
auto& device = static_cast<Device&>(GetDevice());
AssertSuccess(device.GetContext().WaitForFences(device.GetNativeDevice(), 1, &m_nativeFence, VK_FALSE, UINT64_MAX));
}

void BinaryFence::ResetInternal()
{
auto& device = static_cast<Device&>(GetDevice());
AssertSuccess(device.GetContext().ResetFences(device.GetNativeDevice(), 1, &m_nativeFence));
m_inSignalledState = false;
}

RHI::FenceState BinaryFence::GetFenceStateInternal() const
{
auto& device = static_cast<Device&>(GetDevice());
VkResult result = device.GetContext().GetFenceStatus(device.GetNativeDevice(), m_nativeFence);
switch (result)
{
case VK_SUCCESS:
return RHI::FenceState::Signaled;
case VK_NOT_READY:
return RHI::FenceState::Reset;
case VK_ERROR_DEVICE_LOST:
AZ_Assert(false, "Device is lost.");
break;
default:
AZ_Assert(false, "Fence state is unknown.");
break;
}
return RHI::FenceState::Reset;
}
} // namespace Vulkan
} // namespace AZ
59 changes: 59 additions & 0 deletions Gems/Atom/RHI/Vulkan/Code/Source/RHI/BinaryFence.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once

#include <RHI/FenceBase.h>

namespace AZ
{
namespace Vulkan
{
class Device;

// BinaryFence is based on VkFence
// Cannot natively be signalled from the CPU
// Signalling from the CPU is emulated by submitting a signal command to the Graphics queue
// The signal command must also be submitted before we can wait for the fence to be signalled
// Used if the device does not support timeline semaphores (Vulkan version < 1.2)
class BinaryFence final : public FenceBase
{
using Base = FenceBase;

public:
AZ_CLASS_ALLOCATOR(BinaryFence, AZ::ThreadPoolAllocator);
AZ_RTTI(BinaryFence, "{FE8974F0-8C64-48A7-8BF2-89E92F911AA4}", Base);

static RHI::Ptr<FenceBase> Create(Fence& fence);
~BinaryFence() = default;

VkFence GetNativeFence() const;

private:
BinaryFence(Fence& fence)
: m_fence{ fence } {};

//////////////////////////////////////////////////////////////////////////
// RHI::Object
void SetNameInternal(const AZStd::string_view& name) override;
//////////////////////////////////////////////////////////////////////////

//////////////////////////////////////////////////////////////////////
// FenceBase
RHI::ResultCode InitInternal(RHI::Device& device, RHI::FenceState initialState) override;
void ShutdownInternal() override;
void SignalOnCpuInternal() override;
void WaitOnCpuInternal() const override;
void ResetInternal() override;
RHI::FenceState GetFenceStateInternal() const override;
//////////////////////////////////////////////////////////////////////

VkFence m_nativeFence = VK_NULL_HANDLE;
Fence& m_fence;
};
} // namespace Vulkan
} // namespace AZ
39 changes: 39 additions & 0 deletions Gems/Atom/RHI/Vulkan/Code/Source/RHI/BinarySemaphore.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include <Atom/RHI.Reflect/VkAllocator.h>
#include <Atom/RHI.Reflect/Vulkan/Conversion.h>
#include <RHI/Device.h>
#include <RHI/Semaphore.h>

namespace AZ
{
namespace Vulkan
{
RHI::Ptr<Semaphore> BinarySemaphore::Create()
{
return aznew BinarySemaphore();
}

RHI::ResultCode BinarySemaphore::InitInternal(Device& device)
{
VkSemaphoreCreateInfo createInfo{};
createInfo.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO;
createInfo.pNext = 0;
createInfo.flags = 0;

const VkResult result =
device.GetContext().CreateSemaphore(device.GetNativeDevice(), &createInfo, VkSystemAllocator::Get(), &m_nativeSemaphore);
AssertSuccess(result);

RETURN_RESULT_IF_UNSUCCESSFUL(ConvertResult(result));

SetName(GetName());
return RHI::ResultCode::Success;
}
} // namespace Vulkan
} // namespace AZ
37 changes: 37 additions & 0 deletions Gems/Atom/RHI/Vulkan/Code/Source/RHI/BinarySemaphore.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once

#include <RHI/Semaphore.h>

namespace AZ
{
namespace Vulkan
{
class Device;

// Semaphore based on a basic VkSemaphore
// Is used if the device does not support timeline semaphores and always for the SwapChain
// due to limitations discussed here: https://www.khronos.org/blog/vulkan-timeline-semaphores
class BinarySemaphore final : public Semaphore
{
using Base = RHI::DeviceObject;

public:
AZ_RTTI(BinarySemaphore, "{CA8937A8-98C8-4A6A-8C82-771145E4175C}", Semaphore);
AZ_CLASS_ALLOCATOR(BinarySemaphore, AZ::ThreadPoolAllocator);

static RHI::Ptr<Semaphore> Create();
~BinarySemaphore() = default;

private:
RHI::ResultCode InitInternal(Device& device) override;
BinarySemaphore() = default;
};
} // namespace Vulkan
} // namespace AZ
20 changes: 15 additions & 5 deletions Gems/Atom/RHI/Vulkan/Code/Source/RHI/Device.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,22 @@
*
*/

#include <Atom_RHI_Vulkan_Platform.h>
#include <Atom/RHI.Reflect/VkAllocator.h>
#include <Atom/RHI.Reflect/Vulkan/Conversion.h>
#include <Atom/RHI.Reflect/Vulkan/PlatformLimitsDescriptor.h>
#include <Atom/RHI.Reflect/Vulkan/VulkanBus.h>
#include <Atom/RHI.Reflect/Vulkan/XRVkDescriptors.h>
#include <Atom/RHI/Factory.h>
#include <Atom/RHI/RHISystemInterface.h>
#include <Atom/RHI/RHIMemoryStatisticsInterface.h>
#include <Atom/RHI/RHISystemInterface.h>
#include <Atom/RHI/TransientAttachmentPool.h>
#include <Atom_RHI_Vulkan_Platform.h>
#include <AzCore/Debug/Trace.h>
#include <AzCore/std/containers/set.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/Debug/Trace.h>
#include <RHI/AsyncUploadQueue.h>
#include <RHI/Buffer.h>
#include <RHI/BufferPool.h>
#include <Atom/RHI.Reflect/Vulkan/Conversion.h>
#include <RHI/CommandList.h>
#include <RHI/CommandQueue.h>
#include <RHI/Device.h>
Expand All @@ -32,7 +33,6 @@
#include <RHI/WSISurface.h>
#include <RHI/WindowSurfaceBus.h>
#include <Vulkan_Traits_Platform.h>
#include <Atom/RHI.Reflect/VkAllocator.h>

namespace AZ
{
Expand Down Expand Up @@ -415,6 +415,11 @@ namespace AZ
semaphoreAllocDescriptor.m_collectLatency = RHI::Limits::Device::FrameCountMax;
m_semaphoreAllocator.Init(semaphoreAllocDescriptor);

SwapChainSemaphoreAllocator::Descriptor swapChainSemaphoreAllocDescriptor;
swapChainSemaphoreAllocDescriptor.m_device = this;
swapChainSemaphoreAllocDescriptor.m_collectLatency = RHI::Limits::Device::FrameCountMax;
m_swapChaiSemaphoreAllocator.Init(swapChainSemaphoreAllocDescriptor);

m_imageMemoryRequirementsCache.SetInitFunction([](auto& cache) { cache.set_capacity(MemoryRequirementsCacheSize); });
m_bufferMemoryRequirementsCache.SetInitFunction([](auto& cache) { cache.set_capacity(MemoryRequirementsCacheSize); });

Expand Down Expand Up @@ -565,6 +570,11 @@ namespace AZ
return m_semaphoreAllocator;
}

SwapChainSemaphoreAllocator& Device::GetSwapChainSemaphoreAllocator()
{
return m_swapChaiSemaphoreAllocator;
}

const AZStd::vector<VkQueueFamilyProperties>& Device::GetQueueFamilyProperties() const
{
return m_queueFamilyProperties;
Expand Down
14 changes: 8 additions & 6 deletions Gems/Atom/RHI/Vulkan/Code/Source/RHI/Device.h
Original file line number Diff line number Diff line change
Expand Up @@ -7,28 +7,29 @@
*/
#pragma once

#include <Atom/RHI.Loader/LoaderContext.h>
#include <Atom/RHI.Reflect/BufferDescriptor.h>
#include <Atom/RHI/Device.h>
#include <Atom/RHI/MemoryStatisticsBuilder.h>
#include <Atom/RHI/ObjectCache.h>
#include <Atom/RHI/RHISystemInterface.h>
#include <Atom/RHI/ThreadLocalContext.h>
#include <Atom/RHI.Loader/LoaderContext.h>
#include <Atom/RHI.Reflect/BufferDescriptor.h>
#include <AtomCore/std/containers/lru_cache.h>
#include <AzCore/Memory/SystemAllocator.h>
#include <AzCore/RTTI/TypeInfo.h>
#include <AzCore/std/containers/array.h>
#include <AzCore/std/containers/list.h>
#include <AzCore/std/containers/unordered_map.h>
#include <AzCore/std/containers/unordered_set.h>
#include <AzCore/std/parallel/mutex.h>
#include <AzCore/std/parallel/lock.h>
#include <AzCore/std/parallel/mutex.h>
#include <AzCore/std/smart_ptr/unique_ptr.h>
#include <AzCore/std/utils.h>
#include <AtomCore/std/containers/lru_cache.h>
#include <RHI/BindlessDescriptorPool.h>
#include <RHI/Buffer.h>
#include <RHI/CommandList.h>
#include <RHI/CommandPool.h>
#include <RHI/CommandListAllocator.h>
#include <RHI/CommandPool.h>
#include <RHI/CommandQueueContext.h>
#include <RHI/DescriptorSetLayout.h>
#include <RHI/Framebuffer.h>
Expand All @@ -39,7 +40,6 @@
#include <RHI/RenderPass.h>
#include <RHI/Sampler.h>
#include <RHI/SemaphoreAllocator.h>
#include <RHI/BindlessDescriptorPool.h>

namespace AZ
{
Expand Down Expand Up @@ -103,6 +103,7 @@ namespace AZ
CommandQueueContext& GetCommandQueueContext();
const CommandQueueContext& GetCommandQueueContext() const;
SemaphoreAllocator& GetSemaphoreAllocator();
SwapChainSemaphoreAllocator& GetSwapChainSemaphoreAllocator();

const AZStd::vector<VkQueueFamilyProperties>& GetQueueFamilyProperties() const;

Expand Down Expand Up @@ -213,6 +214,7 @@ namespace AZ
RHI::Ptr<AsyncUploadQueue> m_asyncUploadQueue;
CommandListAllocator m_commandListAllocator;
SemaphoreAllocator m_semaphoreAllocator;
SwapChainSemaphoreAllocator m_swapChaiSemaphoreAllocator;

// New VkImageUsageFlags are inserted in the map in a lazy way.
// Because of this, the map containing the usages per formar is mutable to keep the
Expand Down
Loading

0 comments on commit d9b7dbe

Please sign in to comment.