From 85e0da62ba6f1dce6dcb061c43c0c8f96f4c700e Mon Sep 17 00:00:00 2001 From: Ben Vanik Date: Wed, 10 Jul 2024 21:29:49 -0700 Subject: [PATCH] Adding some CTS variants for indirect command buffers. (#17846) This is a simple start that tests a few commands to test the infrastructure for record/replay of indirect command buffers and the validation mechanism. We don't have a good way in the CTS yet to conditionally run tests based on whether validation is compiled into the build but I'll be looking into that for testing failure cases in future PRs (for now I've just tested manually to verify errors are propagated). --- runtime/src/iree/hal/command_buffer.h | 5 + .../src/iree/hal/command_buffer_validation.c | 52 +- runtime/src/iree/hal/cts/CMakeLists.txt | 42 ++ runtime/src/iree/hal/cts/allocator_test.h | 18 +- .../src/iree/hal/cts/buffer_mapping_test.h | 42 +- .../hal/cts/command_buffer_copy_buffer_test.h | 275 +++++++++ .../hal/cts/command_buffer_dispatch_test.h | 134 +++-- .../hal/cts/command_buffer_fill_buffer_test.h | 331 +++++++++++ .../cts/command_buffer_push_constants_test.h | 12 +- .../src/iree/hal/cts/command_buffer_test.h | 556 +----------------- .../cts/command_buffer_update_buffer_test.h | 164 ++++++ runtime/src/iree/hal/cts/cts_test_base.h | 196 ++++-- .../iree/hal/cts/descriptor_set_layout_test.h | 18 +- runtime/src/iree/hal/cts/driver_test.h | 68 ++- runtime/src/iree/hal/cts/event_test.h | 16 +- .../src/iree/hal/cts/executable_cache_test.h | 16 +- runtime/src/iree/hal/cts/file_test.h | 12 +- .../src/iree/hal/cts/pipeline_layout_test.h | 18 +- .../iree/hal/cts/semaphore_submission_test.h | 39 +- runtime/src/iree/hal/cts/semaphore_test.h | 41 +- .../command_buffer_dispatch_test.mlir | 20 +- .../hal/drivers/cuda/pending_queue_actions.c | 7 +- .../iree/hal/drivers/hip/cts/CMakeLists.txt | 13 +- runtime/src/iree/hal/drivers/hip/hip_device.c | 20 +- .../hal/drivers/hip/pending_queue_actions.c | 7 +- .../iree/hal/drivers/local_sync/sync_device.c | 34 +- .../iree/hal/drivers/local_task/task_queue.c | 163 +++-- .../iree/hal/drivers/vulkan/vulkan_device.cc | 24 +- 28 files changed, 1385 insertions(+), 958 deletions(-) create mode 100644 runtime/src/iree/hal/cts/command_buffer_copy_buffer_test.h create mode 100644 runtime/src/iree/hal/cts/command_buffer_fill_buffer_test.h create mode 100644 runtime/src/iree/hal/cts/command_buffer_update_buffer_test.h diff --git a/runtime/src/iree/hal/command_buffer.h b/runtime/src/iree/hal/command_buffer.h index 91fb184218e0..38ac03d05f9b 100644 --- a/runtime/src/iree/hal/command_buffer.h +++ b/runtime/src/iree/hal/command_buffer.h @@ -451,6 +451,11 @@ iree_hal_buffer_binding_table_empty(void) { return table; } +static inline bool iree_hal_buffer_binding_table_is_empty( + iree_hal_buffer_binding_table_t binding_table) { + return binding_table.count == 0; +} + // Returns an unretained buffer specified in |buffer_ref| or from // |binding_table| with the slot specified if indirect. If the caller needs to // preserve the buffer for longer than the (known) lifetime of the binding table diff --git a/runtime/src/iree/hal/command_buffer_validation.c b/runtime/src/iree/hal/command_buffer_validation.c index 9086303aeeb6..87f3a4bb9e67 100644 --- a/runtime/src/iree/hal/command_buffer_validation.c +++ b/runtime/src/iree/hal/command_buffer_validation.c @@ -123,15 +123,16 @@ static iree_status_t iree_hal_command_buffer_validate_binding_requirements( // it are in range. if (requirements.max_byte_offset > 0) { iree_device_size_t end = binding.offset + requirements.max_byte_offset; - if (IREE_UNLIKELY(end > binding.length)) { - return iree_make_status(IREE_STATUS_OUT_OF_RANGE, - "at least one command attempted to access an " - "address outside of the valid bound buffer " - "range (length=%" PRIdsz ", end(inc)=%" PRIdsz - ", binding offset=%" PRIdsz - ", binding length=%" PRIdsz ")", - requirements.max_byte_offset, end - 1, - binding.offset, binding.length); + if (IREE_UNLIKELY(end > binding.offset + binding.length)) { + return iree_make_status( + IREE_STATUS_OUT_OF_RANGE, + "at least one command attempted to access an " + "address outside of the valid bound buffer " + "range (length=%" PRIdsz ", end(inc)=%" PRIdsz + ", binding offset=%" PRIdsz ", binding length=%" PRIdsz + ", binding end(inc)=%" PRIdsz ")", + requirements.max_byte_offset, end - 1, binding.offset, binding.length, + binding.offset + binding.length - 1); } } @@ -188,8 +189,11 @@ static iree_status_t iree_hal_command_buffer_validate_buffer_requirements( table_requirements->type |= requirements.type; table_requirements->max_byte_offset = iree_max( table_requirements->max_byte_offset, requirements.max_byte_offset); - table_requirements->min_byte_alignment = iree_device_size_lcm( - table_requirements->min_byte_alignment, requirements.min_byte_alignment); + if (requirements.min_byte_alignment) { + table_requirements->min_byte_alignment = + iree_device_size_lcm(table_requirements->min_byte_alignment, + requirements.min_byte_alignment); + } return iree_ok_status(); } @@ -430,14 +434,19 @@ iree_status_t iree_hal_command_buffer_copy_buffer_validation( // Check for overlap - just like memcpy we don't handle that. // Note that it's only undefined behavior if violated so we are ok if tricky // situations (subspans of subspans of binding table subranges etc) make it - // through. - if (iree_hal_buffer_test_overlap(source_ref.buffer, source_ref.offset, - source_ref.length, target_ref.buffer, - target_ref.offset, target_ref.length) != - IREE_HAL_BUFFER_OVERLAP_DISJOINT) { - return iree_make_status( - IREE_STATUS_INVALID_ARGUMENT, - "source and target ranges overlap within the same buffer"); + // through. This is only possible if both buffers are directly referenced - + // we _could_ try to catch this for indirect references by stashing the + // overlap check metadata for validation when the binding table is available + // but that's too costly to be worth it. + if (source_ref.buffer && target_ref.buffer) { + if (iree_hal_buffer_test_overlap(source_ref.buffer, source_ref.offset, + source_ref.length, target_ref.buffer, + target_ref.offset, target_ref.length) != + IREE_HAL_BUFFER_OVERLAP_DISJOINT) { + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "source and target ranges overlap within the same buffer"); + } } return iree_ok_status(); @@ -571,10 +580,11 @@ iree_status_t iree_hal_command_buffer_push_descriptor_set_validation( // TODO(benvanik): validate set index. // TODO(benvanik): use pipeline layout to derive usage and access bits. + // For now we conservatively say _any_ access may be performed (read/write). iree_hal_buffer_binding_requirements_t requirements = { .required_compatibility = IREE_HAL_BUFFER_COMPATIBILITY_QUEUE_DISPATCH, - // .usage = IREE_HAL_BUFFER_USAGE_DISPATCH_..., - // .access = IREE_HAL_MEMORY_ACCESS_..., + .usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE, + .access = IREE_HAL_MEMORY_ACCESS_ANY, .type = IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE, }; for (iree_host_size_t i = 0; i < binding_count; ++i) { diff --git a/runtime/src/iree/hal/cts/CMakeLists.txt b/runtime/src/iree/hal/cts/CMakeLists.txt index 62e62837e8e9..090a2a635898 100644 --- a/runtime/src/iree/hal/cts/CMakeLists.txt +++ b/runtime/src/iree/hal/cts/CMakeLists.txt @@ -8,8 +8,11 @@ set(IREE_ALL_CTS_TESTS "allocator" "buffer_mapping" "command_buffer" + "command_buffer_copy_buffer" "command_buffer_dispatch" + "command_buffer_fill_buffer" "command_buffer_push_constants" + "command_buffer_update_buffer" "descriptor_set_layout" "driver" "event" @@ -91,6 +94,19 @@ iree_cc_library( TESTONLY ) +iree_cc_library( + NAME + command_buffer_copy_buffer_test_library + HDRS + "command_buffer_copy_buffer_test.h" + DEPS + ::cts_test_base + iree::base + iree::hal + iree::testing::gtest + TESTONLY +) + iree_cc_library( NAME command_buffer_dispatch_test_library @@ -104,6 +120,19 @@ iree_cc_library( TESTONLY ) +iree_cc_library( + NAME + command_buffer_fill_buffer_test_library + HDRS + "command_buffer_fill_buffer_test.h" + DEPS + ::cts_test_base + iree::base + iree::hal + iree::testing::gtest + TESTONLY +) + iree_cc_library( NAME command_buffer_push_constants_test_library @@ -117,6 +146,19 @@ iree_cc_library( TESTONLY ) +iree_cc_library( + NAME + command_buffer_update_buffer_test_library + HDRS + "command_buffer_update_buffer_test.h" + DEPS + ::cts_test_base + iree::base + iree::hal + iree::testing::gtest + TESTONLY +) + iree_cc_library( NAME descriptor_set_layout_test_library diff --git a/runtime/src/iree/hal/cts/allocator_test.h b/runtime/src/iree/hal/cts/allocator_test.h index f8de4149047b..b6a1e50e4d5e 100644 --- a/runtime/src/iree/hal/cts/allocator_test.h +++ b/runtime/src/iree/hal/cts/allocator_test.h @@ -13,17 +13,13 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { namespace { - constexpr iree_device_size_t kAllocationSize = 1024; - } // namespace -class allocator_test : public CTSTestBase<> {}; +class AllocatorTest : public CTSTestBase<> {}; // All allocators must support some baseline capabilities. // @@ -31,7 +27,7 @@ class allocator_test : public CTSTestBase<> {}; // driver implementations or target devices, such as: // IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL // IREE_HAL_BUFFER_USAGE_MAPPING -TEST_F(allocator_test, BaselineBufferCompatibility) { +TEST_F(AllocatorTest, BaselineBufferCompatibility) { // Need at least one way to get data between the host and device. iree_hal_buffer_params_t host_local_params = {0}; host_local_params.type = @@ -80,7 +76,7 @@ TEST_F(allocator_test, BaselineBufferCompatibility) { IREE_HAL_BUFFER_COMPATIBILITY_QUEUE_DISPATCH)); } -TEST_F(allocator_test, AllocateBuffer) { +TEST_F(AllocatorTest, AllocateBuffer) { iree_hal_buffer_params_t params = {0}; params.type = IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL; params.usage = IREE_HAL_BUFFER_USAGE_TRANSFER; @@ -102,7 +98,7 @@ TEST_F(allocator_test, AllocateBuffer) { // While empty allocations aren't particularly useful, they can occur in // practice so we should at least be able to create them without errors. -TEST_F(allocator_test, AllocateEmptyBuffer) { +TEST_F(AllocatorTest, AllocateEmptyBuffer) { iree_hal_buffer_params_t params = {0}; params.type = IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL; params.usage = IREE_HAL_BUFFER_USAGE_TRANSFER; @@ -113,8 +109,6 @@ TEST_F(allocator_test, AllocateEmptyBuffer) { iree_hal_buffer_release(buffer); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_ALLOCATOR_TEST_H_ diff --git a/runtime/src/iree/hal/cts/buffer_mapping_test.h b/runtime/src/iree/hal/cts/buffer_mapping_test.h index c97c1ac1031b..504f3d5e9c40 100644 --- a/runtime/src/iree/hal/cts/buffer_mapping_test.h +++ b/runtime/src/iree/hal/cts/buffer_mapping_test.h @@ -16,9 +16,7 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { using ::testing::ContainerEq; @@ -42,7 +40,7 @@ constexpr iree_device_size_t kDefaultAllocationSize = 1024; // * write with an offset and length // * write into a subspan of a buffer -class buffer_mapping_test : public CTSTestBase<> { +class BufferMappingTest : public CTSTestBase<> { protected: void AllocateUninitializedBuffer(iree_device_size_t buffer_size, iree_hal_buffer_t** out_buffer) { @@ -59,7 +57,7 @@ class buffer_mapping_test : public CTSTestBase<> { } }; -TEST_F(buffer_mapping_test, AllocatorSupportsBufferMapping) { +TEST_F(BufferMappingTest, AllocatorSupportsBufferMapping) { iree_hal_buffer_params_t params = {0}; params.type = IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; params.usage = IREE_HAL_BUFFER_USAGE_MAPPING; @@ -83,7 +81,7 @@ TEST_F(buffer_mapping_test, AllocatorSupportsBufferMapping) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, ZeroWholeBuffer) { +TEST_F(BufferMappingTest, ZeroWholeBuffer) { iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(kDefaultAllocationSize, &buffer); @@ -102,7 +100,7 @@ TEST_F(buffer_mapping_test, ZeroWholeBuffer) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, ZeroWithOffset) { +TEST_F(BufferMappingTest, ZeroWithOffset) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -128,7 +126,7 @@ TEST_F(buffer_mapping_test, ZeroWithOffset) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, ZeroSubspan) { +TEST_F(BufferMappingTest, ZeroSubspan) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -171,7 +169,7 @@ TEST_F(buffer_mapping_test, ZeroSubspan) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, FillEmpty) { +TEST_F(BufferMappingTest, FillEmpty) { iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(kDefaultAllocationSize, &buffer); @@ -195,7 +193,7 @@ TEST_F(buffer_mapping_test, FillEmpty) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, FillWholeBuffer) { +TEST_F(BufferMappingTest, FillWholeBuffer) { iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(kDefaultAllocationSize, &buffer); @@ -217,7 +215,7 @@ TEST_F(buffer_mapping_test, FillWholeBuffer) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, FillWithOffset) { +TEST_F(BufferMappingTest, FillWithOffset) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -244,7 +242,7 @@ TEST_F(buffer_mapping_test, FillWithOffset) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, FillSubspan) { +TEST_F(BufferMappingTest, FillSubspan) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -288,7 +286,7 @@ TEST_F(buffer_mapping_test, FillSubspan) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, ReadData) { +TEST_F(BufferMappingTest, ReadData) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -325,7 +323,7 @@ TEST_F(buffer_mapping_test, ReadData) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, ReadDataSubspan) { +TEST_F(BufferMappingTest, ReadDataSubspan) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -368,7 +366,7 @@ TEST_F(buffer_mapping_test, ReadDataSubspan) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, WriteDataWholeBuffer) { +TEST_F(BufferMappingTest, WriteDataWholeBuffer) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -390,7 +388,7 @@ TEST_F(buffer_mapping_test, WriteDataWholeBuffer) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, WriteDataWithOffset) { +TEST_F(BufferMappingTest, WriteDataWithOffset) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -417,7 +415,7 @@ TEST_F(buffer_mapping_test, WriteDataWithOffset) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, WriteDataSubspan) { +TEST_F(BufferMappingTest, WriteDataSubspan) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -459,7 +457,7 @@ TEST_F(buffer_mapping_test, WriteDataSubspan) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, CopyData) { +TEST_F(BufferMappingTest, CopyData) { iree_hal_buffer_t* buffer_a = NULL; iree_hal_buffer_t* buffer_b = NULL; AllocateUninitializedBuffer(kDefaultAllocationSize, &buffer_a); @@ -490,7 +488,7 @@ TEST_F(buffer_mapping_test, CopyData) { // Maps a buffer range for reading from device -> host. // This is roughly what iree_hal_buffer_map_read does internally. -TEST_F(buffer_mapping_test, MapRangeRead) { +TEST_F(BufferMappingTest, MapRangeRead) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -520,7 +518,7 @@ TEST_F(buffer_mapping_test, MapRangeRead) { // Maps a buffer range for writing from host -> device. // This is roughly what iree_hal_buffer_map_write does internally. -TEST_F(buffer_mapping_test, MapRangeWrite) { +TEST_F(BufferMappingTest, MapRangeWrite) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -550,8 +548,6 @@ TEST_F(buffer_mapping_test, MapRangeWrite) { iree_hal_buffer_release(buffer); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_BUFFER_MAPPING_TEST_H_ diff --git a/runtime/src/iree/hal/cts/command_buffer_copy_buffer_test.h b/runtime/src/iree/hal/cts/command_buffer_copy_buffer_test.h new file mode 100644 index 000000000000..70735c070a27 --- /dev/null +++ b/runtime/src/iree/hal/cts/command_buffer_copy_buffer_test.h @@ -0,0 +1,275 @@ +// Copyright 2019 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef IREE_HAL_CTS_COMMAND_BUFFER_COPY_BUFFER_TEST_H_ +#define IREE_HAL_CTS_COMMAND_BUFFER_COPY_BUFFER_TEST_H_ + +#include +#include + +#include "iree/base/api.h" +#include "iree/hal/api.h" +#include "iree/hal/cts/cts_test_base.h" +#include "iree/testing/gtest.h" +#include "iree/testing/status_matchers.h" + +namespace iree::hal::cts { + +using ::testing::ContainerEq; + +namespace { +constexpr iree_device_size_t kDefaultAllocationSize = 1024; +} // namespace + +class CommandBufferCopyBufferTest : public CTSTestBase<> {}; + +TEST_F(CommandBufferCopyBufferTest, CopyWholeBuffer) { + iree_hal_command_buffer_t* command_buffer = NULL; + IREE_ASSERT_OK(iree_hal_command_buffer_create( + device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, + IREE_HAL_COMMAND_CATEGORY_TRANSFER, IREE_HAL_QUEUE_AFFINITY_ANY, + /*binding_capacity=*/0, &command_buffer)); + + uint8_t i8_val = 0x54; + std::vector reference_buffer(kDefaultAllocationSize); + std::memset(reference_buffer.data(), i8_val, kDefaultAllocationSize); + + // Create and fill a host buffer. + iree_hal_buffer_params_t host_params = {0}; + host_params.type = + IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE; + host_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | + IREE_HAL_BUFFER_USAGE_TRANSFER | + IREE_HAL_BUFFER_USAGE_MAPPING; + iree_hal_buffer_t* host_buffer = nullptr; + IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( + device_allocator_, host_params, kDefaultAllocationSize, &host_buffer)); + IREE_ASSERT_OK(iree_hal_device_transfer_h2d( + device_, reference_buffer.data(), host_buffer, 0, reference_buffer.size(), + IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, iree_infinite_timeout())); + + // Create a device buffer. + iree_hal_buffer_params_t device_params = {0}; + device_params.type = + IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL | IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; + device_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | + IREE_HAL_BUFFER_USAGE_TRANSFER | + IREE_HAL_BUFFER_USAGE_MAPPING; + iree_hal_buffer_t* device_buffer = nullptr; + IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( + device_allocator_, device_params, kDefaultAllocationSize, + &device_buffer)); + + // Copy the host buffer to the device buffer. + IREE_ASSERT_OK(iree_hal_command_buffer_begin(command_buffer)); + IREE_ASSERT_OK(iree_hal_command_buffer_copy_buffer( + command_buffer, /*source_ref=*/ + iree_hal_make_buffer_ref(host_buffer, 0, kDefaultAllocationSize), + /*target_ref=*/ + iree_hal_make_buffer_ref(device_buffer, 0, kDefaultAllocationSize))); + IREE_ASSERT_OK(iree_hal_command_buffer_end(command_buffer)); + + IREE_ASSERT_OK(SubmitCommandBufferAndWait(command_buffer)); + + // Read the device buffer and compare. + std::vector actual_data(kDefaultAllocationSize); + IREE_ASSERT_OK(iree_hal_device_transfer_d2h( + device_, device_buffer, /*source_offset=*/0, + /*target_buffer=*/actual_data.data(), + /*data_length=*/kDefaultAllocationSize, + IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, iree_infinite_timeout())); + EXPECT_THAT(actual_data, ContainerEq(reference_buffer)); + + // Must release the command buffer before resources used by it. + iree_hal_command_buffer_release(command_buffer); + iree_hal_buffer_release(device_buffer); + iree_hal_buffer_release(host_buffer); +} + +TEST_F(CommandBufferCopyBufferTest, CopySubBuffer) { + iree_hal_command_buffer_t* command_buffer = NULL; + IREE_ASSERT_OK(iree_hal_command_buffer_create( + device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, + IREE_HAL_COMMAND_CATEGORY_TRANSFER, IREE_HAL_QUEUE_AFFINITY_ANY, + /*binding_capacity=*/0, &command_buffer)); + + iree_hal_buffer_params_t device_params = {0}; + device_params.type = + IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL | IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; + device_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | + IREE_HAL_BUFFER_USAGE_TRANSFER | + IREE_HAL_BUFFER_USAGE_MAPPING; + iree_hal_buffer_t* device_buffer = NULL; + IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( + device_allocator_, device_params, kDefaultAllocationSize, + &device_buffer)); + + uint8_t i8_val = 0x88; + std::vector reference_buffer(kDefaultAllocationSize); + std::memset(reference_buffer.data() + 8, i8_val, + kDefaultAllocationSize / 2 - 4); + + // Create another host buffer with a smaller size. + iree_hal_buffer_params_t host_params = {0}; + host_params.type = + IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE; + host_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | + IREE_HAL_BUFFER_USAGE_TRANSFER | + IREE_HAL_BUFFER_USAGE_MAPPING; + std::vector host_buffer_data(kDefaultAllocationSize, i8_val); + iree_hal_buffer_t* host_buffer = NULL; + IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( + device_allocator_, host_params, host_buffer_data.size() / 2, + &host_buffer)); + IREE_ASSERT_OK(iree_hal_device_transfer_h2d( + device_, host_buffer_data.data(), host_buffer, 0, + host_buffer_data.size() / 2, IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, + iree_infinite_timeout())); + + // Copy the host buffer to the device buffer; zero fill the untouched bytes. + uint8_t zero_val = 0x0; + IREE_ASSERT_OK(iree_hal_command_buffer_begin(command_buffer)); + IREE_ASSERT_OK(iree_hal_command_buffer_fill_buffer( + command_buffer, + iree_hal_make_buffer_ref(device_buffer, /*target_offset=*/0, + /*length=*/8), + &zero_val, /*pattern_length=*/sizeof(zero_val))); + IREE_ASSERT_OK(iree_hal_command_buffer_copy_buffer( + command_buffer, + iree_hal_make_buffer_ref(/*source_buffer=*/host_buffer, + /*source_offset=*/4, + /*length=*/kDefaultAllocationSize / 2 - 4), + iree_hal_make_buffer_ref(/*target_buffer=*/device_buffer, + /*target_offset=*/8, + /*length=*/kDefaultAllocationSize / 2 - 4))); + IREE_ASSERT_OK(iree_hal_command_buffer_fill_buffer( + command_buffer, + iree_hal_make_buffer_ref( + device_buffer, + /*target_offset=*/8 + kDefaultAllocationSize / 2 - 4, + /*length=*/kDefaultAllocationSize - + (8 + kDefaultAllocationSize / 2 - 4)), + &zero_val, + /*pattern_length=*/sizeof(zero_val))); + IREE_ASSERT_OK(iree_hal_command_buffer_end(command_buffer)); + + IREE_ASSERT_OK(SubmitCommandBufferAndWait(command_buffer)); + + // Read the device buffer and compare. + std::vector actual_data(kDefaultAllocationSize); + IREE_ASSERT_OK(iree_hal_device_transfer_d2h( + device_, device_buffer, /*source_offset=*/0, + /*target_buffer=*/actual_data.data(), + /*data_length=*/kDefaultAllocationSize, + IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, iree_infinite_timeout())); + EXPECT_THAT(actual_data, ContainerEq(reference_buffer)); + + // Must release the command buffer before resources used by it. + iree_hal_command_buffer_release(command_buffer); + iree_hal_buffer_release(device_buffer); + iree_hal_buffer_release(host_buffer); +} + +TEST_F(CommandBufferCopyBufferTest, CopySubBufferIndirect) { + const int kHostBufferSlot = 0; + const int kDeviceBufferSlot = 1; + const int kBindingSlotCount = 2; + iree_hal_command_buffer_t* command_buffer = NULL; + IREE_ASSERT_OK(iree_hal_command_buffer_create( + device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, + IREE_HAL_COMMAND_CATEGORY_TRANSFER, IREE_HAL_QUEUE_AFFINITY_ANY, + kBindingSlotCount, &command_buffer)); + + iree_hal_buffer_params_t device_params = {0}; + device_params.type = + IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL | IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; + device_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | + IREE_HAL_BUFFER_USAGE_TRANSFER | + IREE_HAL_BUFFER_USAGE_MAPPING; + iree_hal_buffer_t* device_buffer = NULL; + IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( + device_allocator_, device_params, kDefaultAllocationSize, + &device_buffer)); + + uint8_t i8_val = 0x88; + std::vector reference_buffer(kDefaultAllocationSize); + std::memset(reference_buffer.data() + 8, i8_val, + kDefaultAllocationSize / 2 - 4); + + // Create another host buffer with a smaller size. + iree_hal_buffer_params_t host_params = {0}; + host_params.type = + IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE; + host_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | + IREE_HAL_BUFFER_USAGE_TRANSFER | + IREE_HAL_BUFFER_USAGE_MAPPING; + std::vector host_buffer_data(kDefaultAllocationSize, i8_val); + iree_hal_buffer_t* host_buffer = NULL; + IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( + device_allocator_, host_params, host_buffer_data.size() / 2, + &host_buffer)); + IREE_ASSERT_OK(iree_hal_device_transfer_h2d( + device_, host_buffer_data.data(), host_buffer, 0, + host_buffer_data.size() / 2, IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, + iree_infinite_timeout())); + + // Copy the host buffer to the device buffer; zero fill the untouched bytes. + uint8_t zero_val = 0x0; + IREE_ASSERT_OK(iree_hal_command_buffer_begin(command_buffer)); + IREE_ASSERT_OK(iree_hal_command_buffer_fill_buffer( + command_buffer, + iree_hal_make_indirect_buffer_ref(kDeviceBufferSlot, /*offset=*/0, + /*length=*/8), + &zero_val, /*pattern_length=*/sizeof(zero_val))); + IREE_ASSERT_OK(iree_hal_command_buffer_copy_buffer( + command_buffer, + iree_hal_make_indirect_buffer_ref( + kHostBufferSlot, + /*offset=*/4, + /*length=*/kDefaultAllocationSize / 2 - 4), + iree_hal_make_indirect_buffer_ref( + kDeviceBufferSlot, + /*offset=*/8, + /*length=*/kDefaultAllocationSize / 2 - 4))); + IREE_ASSERT_OK(iree_hal_command_buffer_fill_buffer( + command_buffer, + iree_hal_make_indirect_buffer_ref( + kDeviceBufferSlot, + /*target_offset=*/8 + kDefaultAllocationSize / 2 - 4, + /*length=*/kDefaultAllocationSize - + (8 + kDefaultAllocationSize / 2 - 4)), + &zero_val, + /*pattern_length=*/sizeof(zero_val))); + IREE_ASSERT_OK(iree_hal_command_buffer_end(command_buffer)); + + const iree_hal_buffer_binding_t bindings[] = { + /*kHostBufferSlot=*/{host_buffer, 0, IREE_WHOLE_BUFFER}, + /*kDeviceBufferSlot=*/{device_buffer, 0, IREE_WHOLE_BUFFER}, + }; + IREE_ASSERT_OK(SubmitCommandBufferAndWait(command_buffer, + iree_hal_buffer_binding_table_t{ + IREE_ARRAYSIZE(bindings), + bindings, + })); + + // Read the device buffer and compare. + std::vector actual_data(kDefaultAllocationSize); + IREE_ASSERT_OK(iree_hal_device_transfer_d2h( + device_, device_buffer, /*source_offset=*/0, + /*target_buffer=*/actual_data.data(), + /*data_length=*/kDefaultAllocationSize, + IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, iree_infinite_timeout())); + EXPECT_THAT(actual_data, ContainerEq(reference_buffer)); + + // Must release the command buffer before resources used by it. + iree_hal_command_buffer_release(command_buffer); + iree_hal_buffer_release(device_buffer); + iree_hal_buffer_release(host_buffer); +} + +} // namespace iree::hal::cts + +#endif // IREE_HAL_CTS_COMMAND_BUFFER_COPY_BUFFER_TEST_H_ diff --git a/runtime/src/iree/hal/cts/command_buffer_dispatch_test.h b/runtime/src/iree/hal/cts/command_buffer_dispatch_test.h index 85f53f2d3347..4d712073fe7f 100644 --- a/runtime/src/iree/hal/cts/command_buffer_dispatch_test.h +++ b/runtime/src/iree/hal/cts/command_buffer_dispatch_test.h @@ -10,15 +10,15 @@ #include "iree/base/api.h" #include "iree/base/string_view.h" #include "iree/hal/api.h" +#include "iree/hal/buffer_view_util.h" #include "iree/hal/cts/cts_test_base.h" #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { -class command_buffer_dispatch_test : public CTSTestBase<> { +class CommandBufferDispatchTest + : public CTSTestBase<::testing::TestWithParam> { protected: void PrepareAbsExecutable() { IREE_ASSERT_OK(iree_hal_executable_cache_create( @@ -76,58 +76,77 @@ class command_buffer_dispatch_test : public CTSTestBase<> { iree_hal_executable_t* executable_ = NULL; }; -TEST_F(command_buffer_dispatch_test, DispatchAbs) { +// Dispatches absf(x) on a subrange (elements 1-2) of a 4 element input buffer. +// input_buffer = [-2.5 -2.5 -2.5 -2.5] +// output_buffer = [-9.0 2.5 2.5 -9.0] +TEST_P(CommandBufferDispatchTest, DispatchAbs) { PrepareAbsExecutable(); - iree_hal_command_buffer_t* command_buffer = NULL; - IREE_ASSERT_OK(iree_hal_command_buffer_create( - device_, - IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT | - IREE_HAL_COMMAND_BUFFER_MODE_ALLOW_INLINE_EXECUTION, - IREE_HAL_COMMAND_CATEGORY_DISPATCH, IREE_HAL_QUEUE_AFFINITY_ANY, - /*binding_capacity=*/0, &command_buffer)); - - IREE_ASSERT_OK(iree_hal_command_buffer_begin(command_buffer)); + // Create input buffer. + iree_hal_buffer_t* input_buffer = NULL; + CreateFilledDeviceBuffer(4 * sizeof(float), -2.5f, &input_buffer); - // Create input and output buffers. - iree_hal_buffer_params_t input_params = {0}; - input_params.type = IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL; - input_params.usage = - IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | IREE_HAL_BUFFER_USAGE_TRANSFER; - iree_hal_buffer_view_t* input_buffer_view = NULL; - float input_data[1] = {-2.5f}; - IREE_ASSERT_OK(iree_hal_buffer_view_allocate_buffer_copy( - device_, device_allocator_, - /*shape_rank=*/0, /*shape=*/NULL, IREE_HAL_ELEMENT_TYPE_FLOAT_32, - IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, input_params, - iree_make_const_byte_span((void*)input_data, sizeof(input_data)), - &input_buffer_view)); - iree_hal_buffer_params_t output_params = {0}; - output_params.type = - IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL | IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; - output_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | - IREE_HAL_BUFFER_USAGE_TRANSFER | - IREE_HAL_BUFFER_USAGE_MAPPING; + // Create output buffer. iree_hal_buffer_t* output_buffer = NULL; - IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( - device_allocator_, output_params, sizeof(float), &output_buffer)); - - iree_hal_buffer_ref_t descriptor_set_bindings[] = { - { + CreateFilledDeviceBuffer(4 * sizeof(float), -9.0f, &output_buffer); + + iree_hal_buffer_ref_t descriptor_set_bindings[2]; + iree_hal_buffer_binding_t bindings[2]; + iree_hal_buffer_binding_table_t binding_table = + iree_hal_buffer_binding_table_empty(); + switch (GetParam()) { + case RecordingType::kDirect: + descriptor_set_bindings[0] = { /*binding=*/0, /*buffer_slot=*/0, - iree_hal_buffer_view_buffer(input_buffer_view), - /*offset=*/0, - iree_hal_buffer_view_byte_length(input_buffer_view), - }, - { + /*buffer=*/input_buffer, + /*offset=*/1 * sizeof(float), + /*length=*/2 * sizeof(float), + }; + descriptor_set_bindings[1] = { /*binding=*/1, /*buffer_slot=*/0, - output_buffer, - iree_hal_buffer_byte_offset(output_buffer), - iree_hal_buffer_byte_length(output_buffer), - }, - }; + /*buffer=*/output_buffer, + /*offset=*/1 * sizeof(float), + /*length=*/2 * sizeof(float), + }; + break; + case RecordingType::kIndirect: + binding_table.count = IREE_ARRAYSIZE(descriptor_set_bindings); + binding_table.bindings = bindings; + bindings[0] = { + /*buffer=*/input_buffer, + /*offset=*/1 * sizeof(float), + /*length=*/2 * sizeof(float), + }; + descriptor_set_bindings[0] = { + /*binding=*/0, + /*buffer_slot=*/0, + /*buffer=*/NULL, + /*offset=*/0, + /*length=*/2 * sizeof(float), + }; + bindings[1] = { + /*buffer=*/output_buffer, + /*offset=*/1 * sizeof(float), + /*length=*/2 * sizeof(float), + }; + descriptor_set_bindings[1] = { + /*binding=*/1, + /*buffer_slot=*/1, + /*buffer=*/NULL, + /*offset=*/0, + /*length=*/2 * sizeof(float), + }; + break; + } + + iree_hal_command_buffer_t* command_buffer = NULL; + IREE_ASSERT_OK(iree_hal_command_buffer_create( + device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, + IREE_HAL_COMMAND_CATEGORY_DISPATCH, IREE_HAL_QUEUE_AFFINITY_ANY, + binding_table.count, &command_buffer)); + IREE_ASSERT_OK(iree_hal_command_buffer_begin(command_buffer)); IREE_ASSERT_OK(iree_hal_command_buffer_push_descriptor_set( command_buffer, pipeline_layout_, /*set=*/0, @@ -149,23 +168,26 @@ TEST_F(command_buffer_dispatch_test, DispatchAbs) { IREE_ASSERT_OK(iree_hal_command_buffer_end(command_buffer)); - IREE_ASSERT_OK(SubmitCommandBufferAndWait(command_buffer)); + IREE_ASSERT_OK(SubmitCommandBufferAndWait(command_buffer, binding_table)); - float output_value = 0.0f; + float output_values[4] = {0.0f}; IREE_ASSERT_OK(iree_hal_device_transfer_d2h( device_, output_buffer, - /*source_offset=*/0, &output_value, sizeof(output_value), + /*source_offset=*/0, output_values, sizeof(output_values), IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, iree_infinite_timeout())); - EXPECT_EQ(2.5f, output_value); + EXPECT_THAT(output_values, ::testing::ElementsAre(-9.0f, 2.5f, 2.5f, -9.0f)); iree_hal_command_buffer_release(command_buffer); iree_hal_buffer_release(output_buffer); - iree_hal_buffer_view_release(input_buffer_view); + iree_hal_buffer_release(input_buffer); CleanupExecutable(); } -} // namespace cts -} // namespace hal -} // namespace iree +INSTANTIATE_TEST_SUITE_P(CommandBufferTest, CommandBufferDispatchTest, + ::testing::Values(RecordingType::kDirect, + RecordingType::kIndirect), + GenerateTestName()); + +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_COMMAND_BUFFER_DISPATCH_TEST_H_ diff --git a/runtime/src/iree/hal/cts/command_buffer_fill_buffer_test.h b/runtime/src/iree/hal/cts/command_buffer_fill_buffer_test.h new file mode 100644 index 000000000000..d5dc0a78b087 --- /dev/null +++ b/runtime/src/iree/hal/cts/command_buffer_fill_buffer_test.h @@ -0,0 +1,331 @@ +// Copyright 2019 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef IREE_HAL_CTS_COMMAND_BUFFER_FILL_BUFFER_TEST_H_ +#define IREE_HAL_CTS_COMMAND_BUFFER_FILL_BUFFER_TEST_H_ + +#include +#include + +#include "iree/base/api.h" +#include "iree/hal/api.h" +#include "iree/hal/cts/cts_test_base.h" +#include "iree/testing/gtest.h" +#include "iree/testing/status_matchers.h" + +namespace iree::hal::cts { + +using ::testing::ContainerEq; + +class CommandBufferFillBufferTest + : public CTSTestBase<::testing::TestWithParam> { + protected: + std::vector RunFillBufferTest(RecordingType recording_type, + iree_device_size_t buffer_size, + iree_device_size_t target_offset, + iree_device_size_t fill_length, + const void* pattern, + iree_host_size_t pattern_length) { + switch (recording_type) { + default: + case RecordingType::kDirect: + return RunFillBufferDirectTest(buffer_size, target_offset, fill_length, + pattern, pattern_length); + case RecordingType::kIndirect: + return RunFillBufferIndirectTest(buffer_size, target_offset, + fill_length, pattern, pattern_length); + } + } + + std::vector RunFillBufferDirectTest( + iree_device_size_t buffer_size, iree_device_size_t target_offset, + iree_device_size_t fill_length, const void* pattern, + iree_host_size_t pattern_length) { + iree_hal_buffer_t* device_buffer = NULL; + CreateZeroedDeviceBuffer(buffer_size, &device_buffer); + + iree_hal_command_buffer_t* command_buffer = NULL; + IREE_CHECK_OK(iree_hal_command_buffer_create( + device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, + IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, + /*binding_capacity=*/0, &command_buffer)); + IREE_CHECK_OK(iree_hal_command_buffer_begin(command_buffer)); + + // Fill the pattern. + IREE_CHECK_OK(iree_hal_command_buffer_fill_buffer( + command_buffer, + iree_hal_make_buffer_ref(device_buffer, target_offset, fill_length), + pattern, pattern_length)); + IREE_CHECK_OK(iree_hal_command_buffer_end(command_buffer)); + IREE_CHECK_OK(SubmitCommandBufferAndWait(command_buffer)); + + // Read data for returning. + std::vector actual_data(buffer_size); + IREE_CHECK_OK(iree_hal_device_transfer_d2h( + device_, device_buffer, /*source_offset=*/0, + /*target_buffer=*/actual_data.data(), + /*data_length=*/buffer_size, IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, + iree_infinite_timeout())); + + // Cleanup and return. + iree_hal_command_buffer_release(command_buffer); + iree_hal_buffer_release(device_buffer); + return actual_data; + } + + std::vector RunFillBufferIndirectTest( + iree_device_size_t buffer_size, iree_device_size_t target_offset, + iree_device_size_t fill_length, const void* pattern, + iree_host_size_t pattern_length) { + iree_hal_buffer_t* device_buffer = NULL; + CreateZeroedDeviceBuffer(buffer_size, &device_buffer); + + iree_hal_command_buffer_t* command_buffer = NULL; + IREE_CHECK_OK(iree_hal_command_buffer_create( + device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, + IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, + /*binding_capacity=*/0, &command_buffer)); + IREE_CHECK_OK(iree_hal_command_buffer_begin(command_buffer)); + + // Fill the pattern. + IREE_CHECK_OK(iree_hal_command_buffer_fill_buffer( + command_buffer, + iree_hal_make_buffer_ref(device_buffer, target_offset, fill_length), + pattern, pattern_length)); + IREE_CHECK_OK(iree_hal_command_buffer_end(command_buffer)); + IREE_CHECK_OK(SubmitCommandBufferAndWait(command_buffer)); + + // Read data for returning. + std::vector actual_data(buffer_size); + IREE_CHECK_OK(iree_hal_device_transfer_d2h( + device_, device_buffer, /*source_offset=*/0, + /*target_buffer=*/actual_data.data(), + /*data_length=*/buffer_size, IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, + iree_infinite_timeout())); + + // Cleanup and return. + iree_hal_command_buffer_release(command_buffer); + iree_hal_buffer_release(device_buffer); + return actual_data; + } +}; + +TEST_P(CommandBufferFillBufferTest, FillBuffer_pattern1_size1_offset0_length1) { + iree_device_size_t buffer_size = 1; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 1; + uint8_t pattern = 0x07; + std::vector reference_buffer{0x07}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, FillBuffer_pattern1_size5_offset0_length5) { + iree_device_size_t buffer_size = 5; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 5; + uint8_t pattern = 0x07; + std::vector reference_buffer{0x07, 0x07, 0x07, 0x07, // + 0x07}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern1_size16_offset0_length1) { + iree_device_size_t buffer_size = 16; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 1; + uint8_t pattern = 0x07; + std::vector reference_buffer{0x07, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern1_size16_offset0_length3) { + iree_device_size_t buffer_size = 16; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 3; + uint8_t pattern = 0x07; + std::vector reference_buffer{0x07, 0x07, 0x07, 0x00, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern1_size16_offset0_length8) { + iree_device_size_t buffer_size = 16; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 8; + uint8_t pattern = 0x07; + std::vector reference_buffer{0x07, 0x07, 0x07, 0x07, // + 0x07, 0x07, 0x07, 0x07, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern1_size16_offset2_length8) { + iree_device_size_t buffer_size = 16; + iree_device_size_t target_offset = 2; + iree_device_size_t fill_length = 8; + uint8_t pattern = 0x07; + std::vector reference_buffer{0x00, 0x00, 0x07, 0x07, // + 0x07, 0x07, 0x07, 0x07, // + 0x07, 0x07, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, FillBuffer_pattern2_size2_offset0_length2) { + iree_device_size_t buffer_size = 2; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 2; + uint16_t pattern = 0xAB23; + std::vector reference_buffer{0x23, 0xAB}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern2_size16_offset0_length8) { + iree_device_size_t buffer_size = 16; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 8; + uint16_t pattern = 0xAB23; + std::vector reference_buffer{0x23, 0xAB, 0x23, 0xAB, // + 0x23, 0xAB, 0x23, 0xAB, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern2_size16_offset0_length10) { + iree_device_size_t buffer_size = 16; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 10; + uint16_t pattern = 0xAB23; + std::vector reference_buffer{0x23, 0xAB, 0x23, 0xAB, // + 0x23, 0xAB, 0x23, 0xAB, // + 0x23, 0xAB, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern2_size16_offset2_length8) { + iree_device_size_t buffer_size = 16; + iree_device_size_t target_offset = 2; + iree_device_size_t fill_length = 8; + uint16_t pattern = 0xAB23; + std::vector reference_buffer{0x00, 0x00, 0x23, 0xAB, // + 0x23, 0xAB, 0x23, 0xAB, // + 0x23, 0xAB, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, FillBuffer_pattern4_size4_offset0_length4) { + iree_device_size_t buffer_size = 4; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 4; + uint32_t pattern = 0xAB23CD45; + std::vector reference_buffer{0x45, 0xCD, 0x23, 0xAB}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern4_size4_offset16_length4) { + iree_device_size_t buffer_size = 20; + iree_device_size_t target_offset = 16; + iree_device_size_t fill_length = 4; + uint32_t pattern = 0xAB23CD45; + std::vector reference_buffer(buffer_size, 0); + *reinterpret_cast(&reference_buffer[target_offset]) = pattern; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern4_size16_offset0_length8) { + iree_device_size_t buffer_size = 16; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 8; + uint32_t pattern = 0xAB23CD45; + std::vector reference_buffer{0x45, 0xCD, 0x23, 0xAB, // + 0x45, 0xCD, 0x23, 0xAB, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern4_size16_offset8_length8) { + iree_device_size_t buffer_size = 16; + iree_device_size_t target_offset = 8; + iree_device_size_t fill_length = 8; + uint32_t pattern = 0xAB23CD45; + std::vector reference_buffer{0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00, // + 0x45, 0xCD, 0x23, 0xAB, // + 0x45, 0xCD, 0x23, 0xAB}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +INSTANTIATE_TEST_SUITE_P(CommandBufferFillBufferTest, + CommandBufferFillBufferTest, + ::testing::Values(RecordingType::kDirect, + RecordingType::kIndirect), + GenerateTestName()); + +} // namespace iree::hal::cts + +#endif // IREE_HAL_CTS_COMMAND_BUFFER_FILL_BUFFER_TEST_H_ diff --git a/runtime/src/iree/hal/cts/command_buffer_push_constants_test.h b/runtime/src/iree/hal/cts/command_buffer_push_constants_test.h index aa719d2e25ea..af99ee1842db 100644 --- a/runtime/src/iree/hal/cts/command_buffer_push_constants_test.h +++ b/runtime/src/iree/hal/cts/command_buffer_push_constants_test.h @@ -14,13 +14,11 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { using ::testing::ContainerEq; -class command_buffer_push_constants_test : public CTSTestBase<> { +class CommandBufferPushConstantsTest : public CTSTestBase<> { protected: void PrepareExecutable() { IREE_ASSERT_OK(iree_hal_executable_cache_create( @@ -77,7 +75,7 @@ class command_buffer_push_constants_test : public CTSTestBase<> { iree_hal_executable_t* executable_ = NULL; }; -TEST_F(command_buffer_push_constants_test, DispatchWithPushConstants) { +TEST_F(CommandBufferPushConstantsTest, DispatchWithPushConstants) { ASSERT_NO_FATAL_FAILURE(PrepareExecutable()); iree_hal_command_buffer_t* command_buffer = NULL; @@ -152,8 +150,6 @@ TEST_F(command_buffer_push_constants_test, DispatchWithPushConstants) { CleanupExecutable(); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_COMMAND_BUFFER_PUSH_CONSTANTS_TEST_H_ diff --git a/runtime/src/iree/hal/cts/command_buffer_test.h b/runtime/src/iree/hal/cts/command_buffer_test.h index dc23c9b77f30..aa7819c6ae1e 100644 --- a/runtime/src/iree/hal/cts/command_buffer_test.h +++ b/runtime/src/iree/hal/cts/command_buffer_test.h @@ -16,74 +16,13 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { using ::testing::ContainerEq; -namespace { -constexpr iree_device_size_t kDefaultAllocationSize = 1024; -} // namespace +class CommandBufferTest : public CTSTestBase<> {}; -class command_buffer_test : public CTSTestBase<> { - protected: - void CreateZeroedDeviceBuffer(iree_device_size_t buffer_size, - iree_hal_buffer_t** out_buffer) { - iree_hal_buffer_params_t params = {0}; - params.type = - IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL | IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; - params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | - IREE_HAL_BUFFER_USAGE_TRANSFER | - IREE_HAL_BUFFER_USAGE_MAPPING; - iree_hal_buffer_t* device_buffer = NULL; - IREE_CHECK_OK(iree_hal_allocator_allocate_buffer( - iree_hal_device_allocator(device_), params, buffer_size, - &device_buffer)); - IREE_ASSERT_OK( - iree_hal_buffer_map_zero(device_buffer, 0, IREE_WHOLE_BUFFER)); - *out_buffer = device_buffer; - } - - std::vector RunFillBufferTest(iree_device_size_t buffer_size, - iree_device_size_t target_offset, - iree_device_size_t fill_length, - const void* pattern, - iree_host_size_t pattern_length) { - iree_hal_buffer_t* device_buffer = NULL; - CreateZeroedDeviceBuffer(buffer_size, &device_buffer); - - iree_hal_command_buffer_t* command_buffer = NULL; - IREE_CHECK_OK(iree_hal_command_buffer_create( - device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, - IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, - /*binding_capacity=*/0, &command_buffer)); - IREE_CHECK_OK(iree_hal_command_buffer_begin(command_buffer)); - - // Fill the pattern. - IREE_CHECK_OK(iree_hal_command_buffer_fill_buffer( - command_buffer, - iree_hal_make_buffer_ref(device_buffer, target_offset, fill_length), - pattern, pattern_length)); - IREE_CHECK_OK(iree_hal_command_buffer_end(command_buffer)); - IREE_CHECK_OK(SubmitCommandBufferAndWait(command_buffer)); - - // Read data for returning. - std::vector actual_data(buffer_size); - IREE_CHECK_OK(iree_hal_device_transfer_d2h( - device_, device_buffer, /*source_offset=*/0, - /*target_buffer=*/actual_data.data(), - /*data_length=*/buffer_size, IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, - iree_infinite_timeout())); - - // Cleanup and return. - iree_hal_command_buffer_release(command_buffer); - iree_hal_buffer_release(device_buffer); - return actual_data; - } -}; - -TEST_F(command_buffer_test, Create) { +TEST_F(CommandBufferTest, Create) { iree_hal_command_buffer_t* command_buffer = NULL; IREE_ASSERT_OK(iree_hal_command_buffer_create( device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, @@ -97,7 +36,7 @@ TEST_F(command_buffer_test, Create) { iree_hal_command_buffer_release(command_buffer); } -TEST_F(command_buffer_test, BeginEnd) { +TEST_F(CommandBufferTest, BeginEnd) { iree_hal_command_buffer_t* command_buffer = NULL; IREE_ASSERT_OK(iree_hal_command_buffer_create( device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, @@ -110,7 +49,7 @@ TEST_F(command_buffer_test, BeginEnd) { iree_hal_command_buffer_release(command_buffer); } -TEST_F(command_buffer_test, SubmitEmpty) { +TEST_F(CommandBufferTest, SubmitEmpty) { iree_hal_command_buffer_t* command_buffer = NULL; IREE_ASSERT_OK(iree_hal_command_buffer_create( device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, @@ -125,489 +64,6 @@ TEST_F(command_buffer_test, SubmitEmpty) { iree_hal_command_buffer_release(command_buffer); } -TEST_F(command_buffer_test, CopyWholeBuffer) { - iree_hal_command_buffer_t* command_buffer = NULL; - IREE_ASSERT_OK(iree_hal_command_buffer_create( - device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, - IREE_HAL_COMMAND_CATEGORY_TRANSFER, IREE_HAL_QUEUE_AFFINITY_ANY, - /*binding_capacity=*/0, &command_buffer)); - - uint8_t i8_val = 0x54; - std::vector reference_buffer(kDefaultAllocationSize); - std::memset(reference_buffer.data(), i8_val, kDefaultAllocationSize); - - // Create and fill a host buffer. - iree_hal_buffer_params_t host_params = {0}; - host_params.type = - IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE; - host_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | - IREE_HAL_BUFFER_USAGE_TRANSFER | - IREE_HAL_BUFFER_USAGE_MAPPING; - iree_hal_buffer_t* host_buffer = nullptr; - IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( - device_allocator_, host_params, kDefaultAllocationSize, &host_buffer)); - IREE_ASSERT_OK(iree_hal_device_transfer_h2d( - device_, reference_buffer.data(), host_buffer, 0, reference_buffer.size(), - IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, iree_infinite_timeout())); - - // Create a device buffer. - iree_hal_buffer_params_t device_params = {0}; - device_params.type = - IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL | IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; - device_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | - IREE_HAL_BUFFER_USAGE_TRANSFER | - IREE_HAL_BUFFER_USAGE_MAPPING; - iree_hal_buffer_t* device_buffer = nullptr; - IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( - device_allocator_, device_params, kDefaultAllocationSize, - &device_buffer)); - - // Copy the host buffer to the device buffer. - IREE_ASSERT_OK(iree_hal_command_buffer_begin(command_buffer)); - IREE_ASSERT_OK(iree_hal_command_buffer_copy_buffer( - command_buffer, /*source_ref=*/ - iree_hal_make_buffer_ref(host_buffer, 0, kDefaultAllocationSize), - /*target_ref=*/ - iree_hal_make_buffer_ref(device_buffer, 0, kDefaultAllocationSize))); - IREE_ASSERT_OK(iree_hal_command_buffer_end(command_buffer)); - - IREE_ASSERT_OK(SubmitCommandBufferAndWait(command_buffer)); - - // Read the device buffer and compare. - std::vector actual_data(kDefaultAllocationSize); - IREE_ASSERT_OK(iree_hal_device_transfer_d2h( - device_, device_buffer, /*source_offset=*/0, - /*target_buffer=*/actual_data.data(), - /*data_length=*/kDefaultAllocationSize, - IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, iree_infinite_timeout())); - EXPECT_THAT(actual_data, ContainerEq(reference_buffer)); - - // Must release the command buffer before resources used by it. - iree_hal_command_buffer_release(command_buffer); - iree_hal_buffer_release(device_buffer); - iree_hal_buffer_release(host_buffer); -} - -TEST_F(command_buffer_test, CopySubBuffer) { - iree_hal_command_buffer_t* command_buffer = NULL; - IREE_ASSERT_OK(iree_hal_command_buffer_create( - device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, - IREE_HAL_COMMAND_CATEGORY_TRANSFER, IREE_HAL_QUEUE_AFFINITY_ANY, - /*binding_capacity=*/0, &command_buffer)); - - iree_hal_buffer_params_t device_params = {0}; - device_params.type = - IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL | IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; - device_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | - IREE_HAL_BUFFER_USAGE_TRANSFER | - IREE_HAL_BUFFER_USAGE_MAPPING; - iree_hal_buffer_t* device_buffer = NULL; - IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( - device_allocator_, device_params, kDefaultAllocationSize, - &device_buffer)); - - uint8_t i8_val = 0x88; - std::vector reference_buffer(kDefaultAllocationSize); - std::memset(reference_buffer.data() + 8, i8_val, - kDefaultAllocationSize / 2 - 4); - - // Create another host buffer with a smaller size. - iree_hal_buffer_params_t host_params = {0}; - host_params.type = - IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE; - host_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | - IREE_HAL_BUFFER_USAGE_TRANSFER | - IREE_HAL_BUFFER_USAGE_MAPPING; - std::vector host_buffer_data(kDefaultAllocationSize, i8_val); - iree_hal_buffer_t* host_buffer = NULL; - IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( - device_allocator_, host_params, host_buffer_data.size() / 2, - &host_buffer)); - IREE_ASSERT_OK(iree_hal_device_transfer_h2d( - device_, host_buffer_data.data(), host_buffer, 0, - host_buffer_data.size() / 2, IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, - iree_infinite_timeout())); - - // Copy the host buffer to the device buffer; zero fill the untouched bytes. - uint8_t zero_val = 0x0; - IREE_ASSERT_OK(iree_hal_command_buffer_begin(command_buffer)); - IREE_ASSERT_OK(iree_hal_command_buffer_fill_buffer( - command_buffer, - iree_hal_make_buffer_ref(device_buffer, /*target_offset=*/0, - /*length=*/8), - &zero_val, /*pattern_length=*/sizeof(zero_val))); - IREE_ASSERT_OK(iree_hal_command_buffer_copy_buffer( - command_buffer, - iree_hal_make_buffer_ref(/*source_buffer=*/host_buffer, - /*source_offset=*/4, - /*length=*/kDefaultAllocationSize / 2 - 4), - iree_hal_make_buffer_ref(/*target_buffer=*/device_buffer, - /*target_offset=*/8, - /*length=*/kDefaultAllocationSize / 2 - 4))); - IREE_ASSERT_OK(iree_hal_command_buffer_fill_buffer( - command_buffer, - iree_hal_make_buffer_ref( - device_buffer, - /*target_offset=*/8 + kDefaultAllocationSize / 2 - 4, - /*length=*/kDefaultAllocationSize - - (8 + kDefaultAllocationSize / 2 - 4)), - &zero_val, - /*pattern_length=*/sizeof(zero_val))); - IREE_ASSERT_OK(iree_hal_command_buffer_end(command_buffer)); - - IREE_ASSERT_OK(SubmitCommandBufferAndWait(command_buffer)); - - // Read the device buffer and compare. - std::vector actual_data(kDefaultAllocationSize); - IREE_ASSERT_OK(iree_hal_device_transfer_d2h( - device_, device_buffer, /*source_offset=*/0, - /*target_buffer=*/actual_data.data(), - /*data_length=*/kDefaultAllocationSize, - IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, iree_infinite_timeout())); - EXPECT_THAT(actual_data, ContainerEq(reference_buffer)); - - // Must release the command buffer before resources used by it. - iree_hal_command_buffer_release(command_buffer); - iree_hal_buffer_release(device_buffer); - iree_hal_buffer_release(host_buffer); -} - -TEST_F(command_buffer_test, FillBuffer_pattern1_size1_offset0_length1) { - iree_device_size_t buffer_size = 1; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 1; - uint8_t pattern = 0x07; - std::vector reference_buffer{0x07}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern1_size5_offset0_length5) { - iree_device_size_t buffer_size = 5; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 5; - uint8_t pattern = 0x07; - std::vector reference_buffer{0x07, 0x07, 0x07, 0x07, // - 0x07}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern1_size16_offset0_length1) { - iree_device_size_t buffer_size = 16; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 1; - uint8_t pattern = 0x07; - std::vector reference_buffer{0x07, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern1_size16_offset0_length3) { - iree_device_size_t buffer_size = 16; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 3; - uint8_t pattern = 0x07; - std::vector reference_buffer{0x07, 0x07, 0x07, 0x00, // - 0x00, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern1_size16_offset0_length8) { - iree_device_size_t buffer_size = 16; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 8; - uint8_t pattern = 0x07; - std::vector reference_buffer{0x07, 0x07, 0x07, 0x07, // - 0x07, 0x07, 0x07, 0x07, // - 0x00, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern1_size16_offset2_length8) { - iree_device_size_t buffer_size = 16; - iree_device_size_t target_offset = 2; - iree_device_size_t fill_length = 8; - uint8_t pattern = 0x07; - std::vector reference_buffer{0x00, 0x00, 0x07, 0x07, // - 0x07, 0x07, 0x07, 0x07, // - 0x07, 0x07, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern2_size2_offset0_length2) { - iree_device_size_t buffer_size = 2; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 2; - uint16_t pattern = 0xAB23; - std::vector reference_buffer{0x23, 0xAB}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern2_size16_offset0_length8) { - iree_device_size_t buffer_size = 16; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 8; - uint16_t pattern = 0xAB23; - std::vector reference_buffer{0x23, 0xAB, 0x23, 0xAB, // - 0x23, 0xAB, 0x23, 0xAB, // - 0x00, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern2_size16_offset0_length10) { - iree_device_size_t buffer_size = 16; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 10; - uint16_t pattern = 0xAB23; - std::vector reference_buffer{0x23, 0xAB, 0x23, 0xAB, // - 0x23, 0xAB, 0x23, 0xAB, // - 0x23, 0xAB, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern2_size16_offset2_length8) { - iree_device_size_t buffer_size = 16; - iree_device_size_t target_offset = 2; - iree_device_size_t fill_length = 8; - uint16_t pattern = 0xAB23; - std::vector reference_buffer{0x00, 0x00, 0x23, 0xAB, // - 0x23, 0xAB, 0x23, 0xAB, // - 0x23, 0xAB, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern4_size4_offset0_length4) { - iree_device_size_t buffer_size = 4; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 4; - uint32_t pattern = 0xAB23CD45; - std::vector reference_buffer{0x45, 0xCD, 0x23, 0xAB}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern4_size4_offset16_length4) { - iree_device_size_t buffer_size = 20; - iree_device_size_t target_offset = 16; - iree_device_size_t fill_length = 4; - uint32_t pattern = 0xAB23CD45; - std::vector reference_buffer(buffer_size, 0); - *reinterpret_cast(&reference_buffer[target_offset]) = pattern; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern4_size16_offset0_length8) { - iree_device_size_t buffer_size = 16; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 8; - uint32_t pattern = 0xAB23CD45; - std::vector reference_buffer{0x45, 0xCD, 0x23, 0xAB, // - 0x45, 0xCD, 0x23, 0xAB, // - 0x00, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern4_size16_offset8_length8) { - iree_device_size_t buffer_size = 16; - iree_device_size_t target_offset = 8; - iree_device_size_t fill_length = 8; - uint32_t pattern = 0xAB23CD45; - std::vector reference_buffer{0x00, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00, // - 0x45, 0xCD, 0x23, 0xAB, // - 0x45, 0xCD, 0x23, 0xAB}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, UpdateBufferWholeBuffer) { - iree_device_size_t target_buffer_size = 16; - std::vector source_buffer{0x01, 0x02, 0x03, 0x04, // - 0x05, 0x06, 0x07, 0x08, // - 0xA1, 0xA2, 0xA3, 0xA4, // - 0xA5, 0xA6, 0xA7, 0xA8}; - - iree_hal_buffer_t* device_buffer = NULL; - CreateZeroedDeviceBuffer(target_buffer_size, &device_buffer); - - iree_hal_command_buffer_t* command_buffer = NULL; - IREE_CHECK_OK(iree_hal_command_buffer_create( - device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, - IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, - /*binding_capacity=*/0, &command_buffer)); - IREE_CHECK_OK(iree_hal_command_buffer_begin(command_buffer)); - - // Issue the update_buffer command. - IREE_CHECK_OK(iree_hal_command_buffer_update_buffer( - command_buffer, - /*source_buffer=*/source_buffer.data(), /*source_offset=*/0, - iree_hal_make_buffer_ref(device_buffer, 0, target_buffer_size))); - IREE_CHECK_OK(iree_hal_command_buffer_end(command_buffer)); - IREE_CHECK_OK(SubmitCommandBufferAndWait(command_buffer)); - - // Check that the contents match what we expect. - std::vector actual_data(target_buffer_size); - IREE_CHECK_OK(iree_hal_device_transfer_d2h( - device_, device_buffer, /*source_offset=*/0, actual_data.data(), - actual_data.size(), IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, - iree_infinite_timeout())); - EXPECT_THAT(actual_data, ContainerEq(source_buffer)); - - iree_hal_command_buffer_release(command_buffer); - iree_hal_buffer_release(device_buffer); -} - -TEST_F(command_buffer_test, UpdateBufferWithOffsets) { - iree_device_size_t target_buffer_size = 16; - std::vector source_buffer{0x01, 0x02, 0x03, 0x04, // - 0x05, 0x06, 0x07, 0x08, // - 0xA1, 0xA2, 0xA3, 0xA4, // - 0xA5, 0xA6, 0xA7, 0xA8}; - - iree_hal_buffer_t* device_buffer = NULL; - CreateZeroedDeviceBuffer(target_buffer_size, &device_buffer); - - iree_hal_command_buffer_t* command_buffer = NULL; - IREE_CHECK_OK(iree_hal_command_buffer_create( - device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, - IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, - /*binding_capacity=*/0, &command_buffer)); - IREE_CHECK_OK(iree_hal_command_buffer_begin(command_buffer)); - - // Issue the update_buffer command. - IREE_CHECK_OK(iree_hal_command_buffer_update_buffer( - command_buffer, - /*source_buffer=*/source_buffer.data(), /*source_offset=*/4, - iree_hal_make_buffer_ref(device_buffer, - /*target_offset=*/4, /*length=*/8))); - IREE_CHECK_OK(iree_hal_command_buffer_end(command_buffer)); - IREE_CHECK_OK(SubmitCommandBufferAndWait(command_buffer)); - - // Check that the contents match what we expect. - std::vector actual_data(target_buffer_size); - IREE_CHECK_OK(iree_hal_device_transfer_d2h( - device_, device_buffer, /*source_offset=*/0, actual_data.data(), - actual_data.size(), IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, - iree_infinite_timeout())); - std::vector reference_buffer{0x00, 0x00, 0x00, 0x00, // - 0x05, 0x06, 0x07, 0x08, // - 0xA1, 0xA2, 0xA3, 0xA4, // - 0x00, 0x00, 0x00, 0x00}; - EXPECT_THAT(actual_data, ContainerEq(reference_buffer)); - - iree_hal_command_buffer_release(command_buffer); - iree_hal_buffer_release(device_buffer); -} - -TEST_F(command_buffer_test, UpdateBufferSubspan) { - iree_device_size_t target_buffer_size = 16; - std::vector source_buffer{0x01, 0x02, 0x03, 0x04, // - 0x05, 0x06, 0x07, 0x08, // - 0xA1, 0xA2, 0xA3, 0xA4, // - 0xA5, 0xA6, 0xA7, 0xA8}; - - iree_hal_buffer_t* device_buffer = NULL; - CreateZeroedDeviceBuffer(target_buffer_size, &device_buffer); - - // Create a subspan. - iree_device_size_t subspan_length = 8; - iree_hal_buffer_t* buffer_subspan; - IREE_ASSERT_OK(iree_hal_buffer_subspan(device_buffer, /*byte_offset=*/4, - subspan_length, &buffer_subspan)); - - iree_hal_command_buffer_t* command_buffer = NULL; - IREE_CHECK_OK(iree_hal_command_buffer_create( - device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, - IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, - /*binding_capacity=*/0, &command_buffer)); - IREE_CHECK_OK(iree_hal_command_buffer_begin(command_buffer)); - - // Issue the update_buffer command. - IREE_CHECK_OK(iree_hal_command_buffer_update_buffer( - command_buffer, - /*source_buffer=*/source_buffer.data(), /*source_offset=*/4, - iree_hal_make_buffer_ref(buffer_subspan, - /*target_offset=*/4, /*length=*/4))); - IREE_CHECK_OK(iree_hal_command_buffer_end(command_buffer)); - IREE_CHECK_OK(SubmitCommandBufferAndWait(command_buffer)); - - // Check that the contents match what we expect. - std::vector actual_data(target_buffer_size); - IREE_ASSERT_OK(iree_hal_device_transfer_d2h( - device_, device_buffer, /*source_offset=*/0, actual_data.data(), - actual_data.size(), IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, - iree_infinite_timeout())); - std::vector reference_buffer{0x00, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00, // - 0x05, 0x06, 0x07, 0x08, // - 0x00, 0x00, 0x00, 0x00}; - EXPECT_THAT(actual_data, ContainerEq(reference_buffer)); - // Also check the subspan. - std::vector actual_data_subspan(subspan_length); - IREE_ASSERT_OK(iree_hal_device_transfer_d2h( - device_, buffer_subspan, /*source_offset=*/0, actual_data_subspan.data(), - actual_data_subspan.size(), IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, - iree_infinite_timeout())); - std::vector reference_buffer_subspan{0x00, 0x00, 0x00, 0x00, // - 0x05, 0x06, 0x07, 0x08}; - EXPECT_THAT(actual_data_subspan, ContainerEq(reference_buffer_subspan)); - - iree_hal_command_buffer_release(command_buffer); - iree_hal_buffer_release(buffer_subspan); - iree_hal_buffer_release(device_buffer); -} - -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_COMMAND_BUFFER_TEST_H_ diff --git a/runtime/src/iree/hal/cts/command_buffer_update_buffer_test.h b/runtime/src/iree/hal/cts/command_buffer_update_buffer_test.h new file mode 100644 index 000000000000..55f9b647a5e0 --- /dev/null +++ b/runtime/src/iree/hal/cts/command_buffer_update_buffer_test.h @@ -0,0 +1,164 @@ +// Copyright 2019 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef IREE_HAL_CTS_COMMAND_BUFFER_UPDATE_BUFFER_TEST_H_ +#define IREE_HAL_CTS_COMMAND_BUFFER_UPDATE_BUFFER_TEST_H_ + +#include +#include + +#include "iree/base/api.h" +#include "iree/hal/api.h" +#include "iree/hal/cts/cts_test_base.h" +#include "iree/testing/gtest.h" +#include "iree/testing/status_matchers.h" + +namespace iree::hal::cts { + +using ::testing::ContainerEq; + +class CommandBufferUpdateBufferTest : public CTSTestBase<> {}; + +TEST_F(CommandBufferUpdateBufferTest, UpdateBufferWholeBuffer) { + iree_device_size_t target_buffer_size = 16; + std::vector source_buffer{0x01, 0x02, 0x03, 0x04, // + 0x05, 0x06, 0x07, 0x08, // + 0xA1, 0xA2, 0xA3, 0xA4, // + 0xA5, 0xA6, 0xA7, 0xA8}; + + iree_hal_buffer_t* device_buffer = NULL; + CreateZeroedDeviceBuffer(target_buffer_size, &device_buffer); + + iree_hal_command_buffer_t* command_buffer = NULL; + IREE_CHECK_OK(iree_hal_command_buffer_create( + device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, + IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, + /*binding_capacity=*/0, &command_buffer)); + IREE_CHECK_OK(iree_hal_command_buffer_begin(command_buffer)); + + // Issue the update_buffer command. + IREE_CHECK_OK(iree_hal_command_buffer_update_buffer( + command_buffer, + /*source_buffer=*/source_buffer.data(), /*source_offset=*/0, + iree_hal_make_buffer_ref(device_buffer, 0, target_buffer_size))); + IREE_CHECK_OK(iree_hal_command_buffer_end(command_buffer)); + IREE_CHECK_OK(SubmitCommandBufferAndWait(command_buffer)); + + // Check that the contents match what we expect. + std::vector actual_data(target_buffer_size); + IREE_CHECK_OK(iree_hal_device_transfer_d2h( + device_, device_buffer, /*source_offset=*/0, actual_data.data(), + actual_data.size(), IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, + iree_infinite_timeout())); + EXPECT_THAT(actual_data, ContainerEq(source_buffer)); + + iree_hal_command_buffer_release(command_buffer); + iree_hal_buffer_release(device_buffer); +} + +TEST_F(CommandBufferUpdateBufferTest, UpdateBufferWithOffsets) { + iree_device_size_t target_buffer_size = 16; + std::vector source_buffer{0x01, 0x02, 0x03, 0x04, // + 0x05, 0x06, 0x07, 0x08, // + 0xA1, 0xA2, 0xA3, 0xA4, // + 0xA5, 0xA6, 0xA7, 0xA8}; + + iree_hal_buffer_t* device_buffer = NULL; + CreateZeroedDeviceBuffer(target_buffer_size, &device_buffer); + + iree_hal_command_buffer_t* command_buffer = NULL; + IREE_CHECK_OK(iree_hal_command_buffer_create( + device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, + IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, + /*binding_capacity=*/0, &command_buffer)); + IREE_CHECK_OK(iree_hal_command_buffer_begin(command_buffer)); + + // Issue the update_buffer command. + IREE_CHECK_OK(iree_hal_command_buffer_update_buffer( + command_buffer, + /*source_buffer=*/source_buffer.data(), /*source_offset=*/4, + iree_hal_make_buffer_ref(device_buffer, + /*target_offset=*/4, /*length=*/8))); + IREE_CHECK_OK(iree_hal_command_buffer_end(command_buffer)); + IREE_CHECK_OK(SubmitCommandBufferAndWait(command_buffer)); + + // Check that the contents match what we expect. + std::vector actual_data(target_buffer_size); + IREE_CHECK_OK(iree_hal_device_transfer_d2h( + device_, device_buffer, /*source_offset=*/0, actual_data.data(), + actual_data.size(), IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, + iree_infinite_timeout())); + std::vector reference_buffer{0x00, 0x00, 0x00, 0x00, // + 0x05, 0x06, 0x07, 0x08, // + 0xA1, 0xA2, 0xA3, 0xA4, // + 0x00, 0x00, 0x00, 0x00}; + EXPECT_THAT(actual_data, ContainerEq(reference_buffer)); + + iree_hal_command_buffer_release(command_buffer); + iree_hal_buffer_release(device_buffer); +} + +TEST_F(CommandBufferUpdateBufferTest, UpdateBufferSubspan) { + iree_device_size_t target_buffer_size = 16; + std::vector source_buffer{0x01, 0x02, 0x03, 0x04, // + 0x05, 0x06, 0x07, 0x08, // + 0xA1, 0xA2, 0xA3, 0xA4, // + 0xA5, 0xA6, 0xA7, 0xA8}; + + iree_hal_buffer_t* device_buffer = NULL; + CreateZeroedDeviceBuffer(target_buffer_size, &device_buffer); + + // Create a subspan. + iree_device_size_t subspan_length = 8; + iree_hal_buffer_t* buffer_subspan; + IREE_ASSERT_OK(iree_hal_buffer_subspan(device_buffer, /*byte_offset=*/4, + subspan_length, &buffer_subspan)); + + iree_hal_command_buffer_t* command_buffer = NULL; + IREE_CHECK_OK(iree_hal_command_buffer_create( + device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, + IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, + /*binding_capacity=*/0, &command_buffer)); + IREE_CHECK_OK(iree_hal_command_buffer_begin(command_buffer)); + + // Issue the update_buffer command. + IREE_CHECK_OK(iree_hal_command_buffer_update_buffer( + command_buffer, + /*source_buffer=*/source_buffer.data(), /*source_offset=*/4, + iree_hal_make_buffer_ref(buffer_subspan, + /*target_offset=*/4, /*length=*/4))); + IREE_CHECK_OK(iree_hal_command_buffer_end(command_buffer)); + IREE_CHECK_OK(SubmitCommandBufferAndWait(command_buffer)); + + // Check that the contents match what we expect. + std::vector actual_data(target_buffer_size); + IREE_ASSERT_OK(iree_hal_device_transfer_d2h( + device_, device_buffer, /*source_offset=*/0, actual_data.data(), + actual_data.size(), IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, + iree_infinite_timeout())); + std::vector reference_buffer{0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00, // + 0x05, 0x06, 0x07, 0x08, // + 0x00, 0x00, 0x00, 0x00}; + EXPECT_THAT(actual_data, ContainerEq(reference_buffer)); + // Also check the subspan. + std::vector actual_data_subspan(subspan_length); + IREE_ASSERT_OK(iree_hal_device_transfer_d2h( + device_, buffer_subspan, /*source_offset=*/0, actual_data_subspan.data(), + actual_data_subspan.size(), IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, + iree_infinite_timeout())); + std::vector reference_buffer_subspan{0x00, 0x00, 0x00, 0x00, // + 0x05, 0x06, 0x07, 0x08}; + EXPECT_THAT(actual_data_subspan, ContainerEq(reference_buffer_subspan)); + + iree_hal_command_buffer_release(command_buffer); + iree_hal_buffer_release(buffer_subspan); + iree_hal_buffer_release(device_buffer); +} + +} // namespace iree::hal::cts + +#endif // IREE_HAL_CTS_COMMAND_BUFFER_UPDATE_BUFFER_TEST_H_ diff --git a/runtime/src/iree/hal/cts/cts_test_base.h b/runtime/src/iree/hal/cts/cts_test_base.h index bdef0285105e..919933e0d616 100644 --- a/runtime/src/iree/hal/cts/cts_test_base.h +++ b/runtime/src/iree/hal/cts/cts_test_base.h @@ -36,53 +36,103 @@ const char* get_test_executable_format(); // Leaf test binaries must implement this function. iree_const_byte_span_t get_test_executable_data(iree_string_view_t file_name); +enum class RecordingType { + kDirect = 0, + kIndirect, +}; + +struct GenerateTestName { + template + std::string operator()( + const ::testing::TestParamInfo& info) const { + switch (info.param) { + default: + return ""; + case RecordingType::kDirect: + return "direct"; + case RecordingType::kIndirect: + return "indirect"; + } + } +}; + +// Gets a HAL driver with the provided name, if available. +static iree_status_t TryGetDriver(const std::string& driver_name, + iree_hal_driver_t** out_driver) { + static std::set unavailable_driver_names; + + // If creation failed before, don't try again. + if (unavailable_driver_names.find(driver_name) != + unavailable_driver_names.end()) { + return iree_make_status(IREE_STATUS_UNAVAILABLE, "driver unavailable"); + } + + // No existing driver, attempt to create. + iree_hal_driver_t* driver = NULL; + iree_status_t status = iree_hal_driver_registry_try_create( + iree_hal_driver_registry_default(), + iree_make_string_view(driver_name.data(), driver_name.size()), + iree_allocator_system(), &driver); + if (iree_status_is_unavailable(status)) { + unavailable_driver_names.insert(driver_name); + } + if (iree_status_is_ok(status)) { + *out_driver = driver; + } + return status; +} + +// Statics available in CTSTestBase without template magic. +// Note that this header is intended to be included in a single .cc so we can +// define the static member storage here. +class CTSTestResources { + public: + static iree_hal_driver_t* driver_; + static iree_hal_device_t* device_; + static iree_hal_allocator_t* device_allocator_; +}; +/*static*/ iree_hal_driver_t* CTSTestResources::driver_ = NULL; +/*static*/ iree_hal_device_t* CTSTestResources::device_ = NULL; +/*static*/ iree_hal_allocator_t* CTSTestResources::device_allocator_ = NULL; + // Common setup for tests parameterized on driver names. template -class CTSTestBase : public BaseType { - protected: +class CTSTestBase : public BaseType, public CTSTestResources { + public: static void SetUpTestSuite() { iree_status_t status = register_test_driver(iree_hal_driver_registry_default()); if (iree_status_is_already_exists(status)) { - return; + status = iree_status_ignore(status); } IREE_CHECK_OK(status); - } - - virtual void SetUp() { - const std::string& driver_name = get_test_driver_name(); // Get driver with the given name and create its default device. // Skip drivers that are (gracefully) unavailable, fail if creation fails. iree_hal_driver_t* driver = NULL; - iree_status_t status = TryGetDriver(driver_name, &driver); + status = TryGetDriver(get_test_driver_name(), &driver); if (iree_status_is_unavailable(status)) { - iree_status_free(status); - GTEST_SKIP() << "Skipping test as '" << driver_name - << "' driver is unavailable"; + iree_status_ignore(status); return; } - IREE_ASSERT_OK(status); + IREE_CHECK_OK(status); driver_ = driver; iree_hal_device_t* device = NULL; status = iree_hal_driver_create_default_device( driver_, iree_allocator_system(), &device); if (iree_status_is_unavailable(status)) { - iree_status_free(status); - GTEST_SKIP() << "Skipping test as default device for '" << driver_name - << "' driver is unavailable"; + iree_status_ignore(status); return; } - IREE_ASSERT_OK(status); - iree_status_free(status); + IREE_CHECK_OK(status); device_ = device; device_allocator_ = iree_hal_device_allocator(device_); iree_hal_allocator_retain(device_allocator_); } - virtual void TearDown() { + static void TearDownTestSuite() { if (device_allocator_) { iree_hal_allocator_release(device_allocator_); device_allocator_ = NULL; @@ -97,18 +147,84 @@ class CTSTestBase : public BaseType { } } + virtual void SetUp() { + if (!driver_) { + GTEST_SKIP() << "Skipping test as '" << get_test_driver_name() + << "' driver is unavailable"; + return; + } + if (!device_) { + GTEST_SKIP() << "Skipping test as default device for '" + << get_test_driver_name() << "' driver is unavailable"; + return; + } + } + + virtual void TearDown() {} + + void CreateUninitializedDeviceBuffer(iree_device_size_t buffer_size, + iree_hal_buffer_t** out_buffer) { + iree_hal_buffer_params_t params = {0}; + params.type = + IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL | IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; + params.usage = + IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | IREE_HAL_BUFFER_USAGE_TRANSFER; + iree_hal_buffer_t* device_buffer = NULL; + IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( + iree_hal_device_allocator(device_), params, buffer_size, out_buffer)); + } + + void CreateZeroedDeviceBuffer(iree_device_size_t buffer_size, + iree_hal_buffer_t** out_buffer) { + iree_hal_buffer_params_t params = {0}; + params.type = + IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL | IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; + params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | + IREE_HAL_BUFFER_USAGE_TRANSFER | + IREE_HAL_BUFFER_USAGE_MAPPING; + iree_hal_buffer_t* device_buffer = NULL; + IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( + iree_hal_device_allocator(device_), params, buffer_size, + &device_buffer)); + IREE_ASSERT_OK( + iree_hal_buffer_map_zero(device_buffer, 0, IREE_WHOLE_BUFFER)); + *out_buffer = device_buffer; + } + + template + void CreateFilledDeviceBuffer(iree_device_size_t buffer_size, + PatternType pattern, + iree_hal_buffer_t** out_buffer) { + iree_hal_buffer_params_t params = {0}; + params.type = IREE_HAL_MEMORY_TYPE_OPTIMAL_FOR_DEVICE | + IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; + params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | + IREE_HAL_BUFFER_USAGE_TRANSFER | + IREE_HAL_BUFFER_USAGE_MAPPING; + iree_hal_buffer_t* device_buffer = NULL; + IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( + iree_hal_device_allocator(device_), params, buffer_size, + &device_buffer)); + IREE_ASSERT_OK(iree_hal_buffer_map_fill(device_buffer, 0, IREE_WHOLE_BUFFER, + &pattern, sizeof(pattern))); + *out_buffer = device_buffer; + } + // Submits |command_buffer| to the device and waits for it to complete before // returning. iree_status_t SubmitCommandBufferAndWait( - iree_hal_command_buffer_t* command_buffer) { - return SubmitCommandBuffersAndWait(1, &command_buffer); + iree_hal_command_buffer_t* command_buffer, + iree_hal_buffer_binding_table_t binding_table = + iree_hal_buffer_binding_table_empty()) { + return SubmitCommandBuffersAndWait(1, &command_buffer, &binding_table); } // Submits |command_buffers| to the device and waits for them to complete // before returning. iree_status_t SubmitCommandBuffersAndWait( iree_host_size_t command_buffer_count, - iree_hal_command_buffer_t** command_buffers) { + iree_hal_command_buffer_t** command_buffers, + const iree_hal_buffer_binding_table_t* binding_tables = nullptr) { // No wait semaphores. iree_hal_semaphore_list_t wait_semaphores = iree_hal_semaphore_list_empty(); @@ -126,7 +242,7 @@ class CTSTestBase : public BaseType { iree_status_t status = iree_hal_device_queue_execute( device_, IREE_HAL_QUEUE_AFFINITY_ANY, wait_semaphores, signal_semaphores, command_buffer_count, command_buffers, - /*binding_tables=*/NULL); + binding_tables); if (iree_status_is_ok(status)) { status = iree_hal_semaphore_wait(signal_semaphore, target_payload_value, iree_infinite_timeout()); @@ -136,12 +252,13 @@ class CTSTestBase : public BaseType { return status; } - iree_hal_command_buffer_t* CreateEmptyCommandBuffer() { + iree_hal_command_buffer_t* CreateEmptyCommandBuffer( + iree_host_size_t binding_capacity = 0) { iree_hal_command_buffer_t* command_buffer = NULL; IREE_EXPECT_OK(iree_hal_command_buffer_create( device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, IREE_HAL_COMMAND_CATEGORY_DISPATCH, IREE_HAL_QUEUE_AFFINITY_ANY, - /*binding_capacity=*/0, &command_buffer)); + binding_capacity, &command_buffer)); IREE_EXPECT_OK(iree_hal_command_buffer_begin(command_buffer)); IREE_EXPECT_OK(iree_hal_command_buffer_end(command_buffer)); return command_buffer; @@ -159,37 +276,6 @@ class CTSTestBase : public BaseType { IREE_EXPECT_OK(iree_hal_semaphore_query(semaphore, &value)); EXPECT_EQ(expected_value, value); } - - iree_hal_driver_t* driver_ = NULL; - iree_hal_device_t* device_ = NULL; - iree_hal_allocator_t* device_allocator_ = NULL; - - private: - // Gets a HAL driver with the provided name, if available. - static iree_status_t TryGetDriver(const std::string& driver_name, - iree_hal_driver_t** out_driver) { - static std::set unavailable_driver_names; - - // If creation failed before, don't try again. - if (unavailable_driver_names.find(driver_name) != - unavailable_driver_names.end()) { - return iree_make_status(IREE_STATUS_UNAVAILABLE, "driver unavailable"); - } - - // No existing driver, attempt to create. - iree_hal_driver_t* driver = NULL; - iree_status_t status = iree_hal_driver_registry_try_create( - iree_hal_driver_registry_default(), - iree_make_string_view(driver_name.data(), driver_name.size()), - iree_allocator_system(), &driver); - if (iree_status_is_unavailable(status)) { - unavailable_driver_names.insert(driver_name); - } - if (iree_status_is_ok(status)) { - *out_driver = driver; - } - return status; - } }; } // namespace cts diff --git a/runtime/src/iree/hal/cts/descriptor_set_layout_test.h b/runtime/src/iree/hal/cts/descriptor_set_layout_test.h index 401220a12f08..ffeb03c1861f 100644 --- a/runtime/src/iree/hal/cts/descriptor_set_layout_test.h +++ b/runtime/src/iree/hal/cts/descriptor_set_layout_test.h @@ -13,15 +13,13 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { -class descriptor_set_layout_test : public CTSTestBase<> {}; +class DescriptorSetLayoutTest : public CTSTestBase<> {}; // Note: bindingCount == 0 is valid in VkDescriptorSetLayoutCreateInfo: // https://www.khronos.org/registry/vulkan/specs/1.2-extensions/man/html/VkDescriptorSetLayoutCreateInfo.html -TEST_F(descriptor_set_layout_test, CreateWithNoBindings) { +TEST_F(DescriptorSetLayoutTest, CreateWithNoBindings) { iree_hal_descriptor_set_layout_t* descriptor_set_layout = NULL; IREE_ASSERT_OK(iree_hal_descriptor_set_layout_create( device_, IREE_HAL_DESCRIPTOR_SET_LAYOUT_FLAG_NONE, @@ -30,7 +28,7 @@ TEST_F(descriptor_set_layout_test, CreateWithNoBindings) { iree_hal_descriptor_set_layout_release(descriptor_set_layout); } -TEST_F(descriptor_set_layout_test, CreateWithOneBinding) { +TEST_F(DescriptorSetLayoutTest, CreateWithOneBinding) { iree_hal_descriptor_set_layout_t* descriptor_set_layout = NULL; iree_hal_descriptor_set_layout_binding_t descriptor_set_layout_bindings[] = { { @@ -46,7 +44,7 @@ TEST_F(descriptor_set_layout_test, CreateWithOneBinding) { iree_hal_descriptor_set_layout_release(descriptor_set_layout); } -TEST_F(descriptor_set_layout_test, CreateWithTwoBindings) { +TEST_F(DescriptorSetLayoutTest, CreateWithTwoBindings) { iree_hal_descriptor_set_layout_t* descriptor_set_layout = NULL; iree_hal_descriptor_set_layout_binding_t descriptor_set_layout_bindings[] = { { @@ -67,7 +65,7 @@ TEST_F(descriptor_set_layout_test, CreateWithTwoBindings) { iree_hal_descriptor_set_layout_release(descriptor_set_layout); } -TEST_F(descriptor_set_layout_test, CreateWithPushDescriptorType) { +TEST_F(DescriptorSetLayoutTest, CreateWithPushDescriptorType) { iree_hal_descriptor_set_layout_t* descriptor_set_layout = NULL; iree_hal_descriptor_set_layout_binding_t descriptor_set_layout_bindings[] = { { @@ -88,8 +86,6 @@ TEST_F(descriptor_set_layout_test, CreateWithPushDescriptorType) { iree_hal_descriptor_set_layout_release(descriptor_set_layout); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_DESCRIPTOR_SET_LAYOUT_TEST_H_ diff --git a/runtime/src/iree/hal/cts/driver_test.h b/runtime/src/iree/hal/cts/driver_test.h index 092dd75a7df8..515244fbb6c7 100644 --- a/runtime/src/iree/hal/cts/driver_test.h +++ b/runtime/src/iree/hal/cts/driver_test.h @@ -16,19 +16,44 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { -class driver_test : public CTSTestBase<> { +// NOTE: does not use CTSTestBase as we don't want the automatically created +// driver/device it provides. +class DriverTest : public ::testing::Test { protected: + using DriverPtr = + std::unique_ptr; + DriverPtr CreateDriver() { + iree_status_t status = + register_test_driver(iree_hal_driver_registry_default()); + if (iree_status_is_already_exists(status)) { + status = iree_status_ignore(status); + } + IREE_CHECK_OK(status); + + // Get driver with the given name and create its default device. + // Skip drivers that are (gracefully) unavailable, fail if creation fails. + iree_hal_driver_t* driver = NULL; + status = TryGetDriver(get_test_driver_name(), &driver); + if (iree_status_is_unavailable(status)) { + iree_status_ignore(status); + return DriverPtr{nullptr, iree_hal_driver_release}; + } + IREE_CHECK_OK(status); + + return DriverPtr{driver, iree_hal_driver_release}; + } + void CheckCreateDeviceViaPath(iree_string_view_t name, iree_string_view_t path) { + auto driver = CreateDriver(); + std::cout << " Creating device '" << std::string(name.data, name.size) << "' with path '" << std::string(path.data, path.size) << "'\n"; iree_hal_device_t* device = NULL; iree_status_t status = iree_hal_driver_create_device_by_path( - driver_, name, path, /*param_count=*/0, /*params=*/NULL, + driver.get(), name, path, /*param_count=*/0, /*params=*/NULL, iree_allocator_system(), &device); // Creation via path is HAL driver specific. Allow unimplemented cases. @@ -46,11 +71,14 @@ class driver_test : public CTSTestBase<> { } }; -TEST_F(driver_test, QueryAndCreateAvailableDevicesByID) { +TEST_F(DriverTest, QueryAndCreateAvailableDevicesByID) { + auto driver = CreateDriver(); + iree_host_size_t device_info_count = 0; iree_hal_device_info_t* device_infos = NULL; IREE_ASSERT_OK(iree_hal_driver_query_available_devices( - driver_, iree_allocator_system(), &device_info_count, &device_infos)); + driver.get(), iree_allocator_system(), &device_info_count, + &device_infos)); std::cout << "Driver has " << device_info_count << " device(s)\n"; for (iree_host_size_t i = 0; i < device_info_count; ++i) { @@ -60,8 +88,8 @@ TEST_F(driver_test, QueryAndCreateAvailableDevicesByID) { << "'\n"; iree_hal_device_t* device = NULL; IREE_ASSERT_OK(iree_hal_driver_create_device_by_id( - driver_, device_infos[i].device_id, /*param_count=*/0, /*params=*/NULL, - iree_allocator_system(), &device)); + driver.get(), device_infos[i].device_id, /*param_count=*/0, + /*params=*/NULL, iree_allocator_system(), &device)); iree_string_view_t device_id = iree_hal_device_id(device); std::cout << " Created device with id: '" << std::string(device_id.data, device_id.size) << "'\n"; @@ -71,11 +99,14 @@ TEST_F(driver_test, QueryAndCreateAvailableDevicesByID) { iree_allocator_free(iree_allocator_system(), device_infos); } -TEST_F(driver_test, QueryAndCreateAvailableDevicesByOrdinal) { +TEST_F(DriverTest, QueryAndCreateAvailableDevicesByOrdinal) { + auto driver = CreateDriver(); + iree_host_size_t device_info_count = 0; iree_hal_device_info_t* device_infos = NULL; IREE_ASSERT_OK(iree_hal_driver_query_available_devices( - driver_, iree_allocator_system(), &device_info_count, &device_infos)); + driver.get(), iree_allocator_system(), &device_info_count, + &device_infos)); std::cout << "Driver has " << device_info_count << " device(s)\n"; for (iree_host_size_t i = 0; i < device_info_count; ++i) { @@ -85,8 +116,8 @@ TEST_F(driver_test, QueryAndCreateAvailableDevicesByOrdinal) { << "'\n"; iree_hal_device_t* device = NULL; IREE_ASSERT_OK(iree_hal_driver_create_device_by_ordinal( - driver_, i, /*param_count=*/0, /*params=*/NULL, iree_allocator_system(), - &device)); + driver.get(), i, /*param_count=*/0, /*params=*/NULL, + iree_allocator_system(), &device)); iree_string_view_t device_id = iree_hal_device_id(device); std::cout << " Created device with id: '" << std::string(device_id.data, device_id.size) << "'\n"; @@ -96,11 +127,14 @@ TEST_F(driver_test, QueryAndCreateAvailableDevicesByOrdinal) { iree_allocator_free(iree_allocator_system(), device_infos); } -TEST_F(driver_test, QueryAndCreateAvailableDevicesByPath) { +TEST_F(DriverTest, QueryAndCreateAvailableDevicesByPath) { + auto driver = CreateDriver(); + iree_host_size_t device_info_count = 0; iree_hal_device_info_t* device_infos = NULL; IREE_ASSERT_OK(iree_hal_driver_query_available_devices( - driver_, iree_allocator_system(), &device_info_count, &device_infos)); + driver.get(), iree_allocator_system(), &device_info_count, + &device_infos)); std::cout << "Driver has " << device_info_count << " device(s)\n"; if (device_info_count == 0) GTEST_SKIP() << "No available devices"; @@ -119,8 +153,6 @@ TEST_F(driver_test, QueryAndCreateAvailableDevicesByPath) { iree_allocator_free(iree_allocator_system(), device_infos); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_DRIVER_TEST_H_ diff --git a/runtime/src/iree/hal/cts/event_test.h b/runtime/src/iree/hal/cts/event_test.h index 47808a094304..6ddb4db41bff 100644 --- a/runtime/src/iree/hal/cts/event_test.h +++ b/runtime/src/iree/hal/cts/event_test.h @@ -15,19 +15,17 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { -class event_test : public CTSTestBase<> {}; +class EventTest : public CTSTestBase<> {}; -TEST_F(event_test, Create) { +TEST_F(EventTest, Create) { iree_hal_event_t* event = NULL; IREE_ASSERT_OK(iree_hal_event_create(device_, &event)); iree_hal_event_release(event); } -TEST_F(event_test, SignalAndReset) { +TEST_F(EventTest, SignalAndReset) { iree_hal_event_t* event = NULL; IREE_ASSERT_OK(iree_hal_event_create(device_, &event)); @@ -50,7 +48,7 @@ TEST_F(event_test, SignalAndReset) { iree_hal_command_buffer_release(command_buffer); } -TEST_F(event_test, SubmitWithChainedCommandBuffers) { +TEST_F(EventTest, SubmitWithChainedCommandBuffers) { iree_hal_event_t* event = NULL; IREE_ASSERT_OK(iree_hal_event_create(device_, &event)); @@ -96,8 +94,6 @@ TEST_F(event_test, SubmitWithChainedCommandBuffers) { iree_hal_event_release(event); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_EVENT_TEST_H_ diff --git a/runtime/src/iree/hal/cts/executable_cache_test.h b/runtime/src/iree/hal/cts/executable_cache_test.h index cb130e4e2b8e..8792fd97560e 100644 --- a/runtime/src/iree/hal/cts/executable_cache_test.h +++ b/runtime/src/iree/hal/cts/executable_cache_test.h @@ -14,13 +14,11 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { -class executable_cache_test : public CTSTestBase<> {}; +class ExecutableCacheTest : public CTSTestBase<> {}; -TEST_F(executable_cache_test, Create) { +TEST_F(ExecutableCacheTest, Create) { iree_status_t loop_status = iree_ok_status(); iree_hal_executable_cache_t* executable_cache = NULL; IREE_ASSERT_OK(iree_hal_executable_cache_create( @@ -31,7 +29,7 @@ TEST_F(executable_cache_test, Create) { IREE_ASSERT_OK(loop_status); } -TEST_F(executable_cache_test, CantPrepareUnknownFormat) { +TEST_F(ExecutableCacheTest, CantPrepareUnknownFormat) { iree_status_t loop_status = iree_ok_status(); iree_hal_executable_cache_t* executable_cache = NULL; IREE_ASSERT_OK(iree_hal_executable_cache_create( @@ -45,7 +43,7 @@ TEST_F(executable_cache_test, CantPrepareUnknownFormat) { IREE_ASSERT_OK(loop_status); } -TEST_F(executable_cache_test, PrepareExecutable) { +TEST_F(ExecutableCacheTest, PrepareExecutable) { iree_status_t loop_status = iree_ok_status(); iree_hal_executable_cache_t* executable_cache = NULL; IREE_ASSERT_OK(iree_hal_executable_cache_create( @@ -97,8 +95,6 @@ TEST_F(executable_cache_test, PrepareExecutable) { IREE_ASSERT_OK(loop_status); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_EXECUTABLE_CACHE_TEST_H_ diff --git a/runtime/src/iree/hal/cts/file_test.h b/runtime/src/iree/hal/cts/file_test.h index b59e61a71abd..b7db5f4c96b7 100644 --- a/runtime/src/iree/hal/cts/file_test.h +++ b/runtime/src/iree/hal/cts/file_test.h @@ -16,9 +16,7 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { using ::testing::ContainerEq; @@ -26,7 +24,7 @@ namespace { constexpr iree_device_size_t kMinimumAlignment = 128; } // namespace -class file_test : public CTSTestBase<> { +class FileTest : public CTSTestBase<> { protected: void CreatePatternedDeviceBuffer(iree_device_size_t buffer_size, uint8_t pattern, @@ -86,7 +84,7 @@ class file_test : public CTSTestBase<> { }; // Reads the entire file into a buffer and check the contents match. -TEST_F(file_test, ReadEntireFile) { +TEST_F(FileTest, ReadEntireFile) { iree_device_size_t file_size = 128; iree_hal_file_t* file = NULL; CreatePatternedMemoryFile(IREE_HAL_MEMORY_ACCESS_READ, file_size, 0xDEu, @@ -133,8 +131,6 @@ TEST_F(file_test, ReadEntireFile) { iree_hal_file_release(file); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_FILE_TEST_H_ diff --git a/runtime/src/iree/hal/cts/pipeline_layout_test.h b/runtime/src/iree/hal/cts/pipeline_layout_test.h index 1cb02cf99a08..4342c6a0ee62 100644 --- a/runtime/src/iree/hal/cts/pipeline_layout_test.h +++ b/runtime/src/iree/hal/cts/pipeline_layout_test.h @@ -13,13 +13,11 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { -class pipeline_layout_test : public CTSTestBase<> {}; +class PipelineLayoutTest : public CTSTestBase<> {}; -TEST_F(pipeline_layout_test, CreateWithNoLayouts) { +TEST_F(PipelineLayoutTest, CreateWithNoLayouts) { iree_hal_pipeline_layout_t* pipeline_layout = NULL; IREE_ASSERT_OK(iree_hal_pipeline_layout_create(device_, /*push_constants=*/0, /*set_layout_count=*/0, NULL, @@ -28,7 +26,7 @@ TEST_F(pipeline_layout_test, CreateWithNoLayouts) { iree_hal_pipeline_layout_release(pipeline_layout); } -TEST_F(pipeline_layout_test, CreateWithPushConstants) { +TEST_F(PipelineLayoutTest, CreateWithPushConstants) { iree_hal_pipeline_layout_t* pipeline_layout = NULL; // Note: The Vulkan maxPushConstantsSize limit must be at least 128 bytes: // https://www.khronos.org/registry/vulkan/specs/1.2/html/vkspec.html#limits-minmax @@ -39,7 +37,7 @@ TEST_F(pipeline_layout_test, CreateWithPushConstants) { iree_hal_pipeline_layout_release(pipeline_layout); } -TEST_F(pipeline_layout_test, CreateWithOneLayout) { +TEST_F(PipelineLayoutTest, CreateWithOneLayout) { iree_hal_descriptor_set_layout_t* descriptor_set_layout = NULL; iree_hal_descriptor_set_layout_binding_t descriptor_set_layout_bindings[] = { { @@ -67,7 +65,7 @@ TEST_F(pipeline_layout_test, CreateWithOneLayout) { iree_hal_descriptor_set_layout_release(descriptor_set_layout); } -TEST_F(pipeline_layout_test, CreateWithTwoLayouts) { +TEST_F(PipelineLayoutTest, CreateWithTwoLayouts) { iree_hal_descriptor_set_layout_t* descriptor_set_layouts[2] = {NULL}; iree_hal_descriptor_set_layout_binding_t layout_bindings_0[] = { { @@ -118,8 +116,6 @@ TEST_F(pipeline_layout_test, CreateWithTwoLayouts) { iree_hal_descriptor_set_layout_release(descriptor_set_layouts[1]); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_PIPELINE_LAYOUT_TEST_H_ diff --git a/runtime/src/iree/hal/cts/semaphore_submission_test.h b/runtime/src/iree/hal/cts/semaphore_submission_test.h index 824471eb6b23..40d0a0f878e5 100644 --- a/runtime/src/iree/hal/cts/semaphore_submission_test.h +++ b/runtime/src/iree/hal/cts/semaphore_submission_test.h @@ -17,13 +17,11 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { -class semaphore_submission_test : public CTSTestBase<> {}; +class SemaphoreSubmissionTest : public CTSTestBase<> {}; -TEST_F(semaphore_submission_test, SubmitWithNoCommandBuffers) { +TEST_F(SemaphoreSubmissionTest, SubmitWithNoCommandBuffers) { // No waits, one signal which we immediately wait on after submit. iree_hal_semaphore_t* signal_semaphore = CreateSemaphore(); uint64_t signal_payload_values[] = {1}; @@ -43,7 +41,7 @@ TEST_F(semaphore_submission_test, SubmitWithNoCommandBuffers) { iree_hal_semaphore_release(signal_semaphore); } -TEST_F(semaphore_submission_test, SubmitAndSignal) { +TEST_F(SemaphoreSubmissionTest, SubmitAndSignal) { iree_hal_command_buffer_t* command_buffer = CreateEmptyCommandBuffer(); // No waits, one signal which we immediately wait on after submit. @@ -66,7 +64,7 @@ TEST_F(semaphore_submission_test, SubmitAndSignal) { iree_hal_semaphore_release(signal_semaphore); } -TEST_F(semaphore_submission_test, SubmitWithWait) { +TEST_F(SemaphoreSubmissionTest, SubmitWithWait) { // Empty command buffer. iree_hal_command_buffer_t* command_buffer = CreateEmptyCommandBuffer(); @@ -105,7 +103,7 @@ TEST_F(semaphore_submission_test, SubmitWithWait) { iree_hal_semaphore_release(signal_semaphore); } -TEST_F(semaphore_submission_test, SubmitWithMultipleSemaphores) { +TEST_F(SemaphoreSubmissionTest, SubmitWithMultipleSemaphores) { iree_hal_command_buffer_t* command_buffer = CreateEmptyCommandBuffer(); iree_hal_semaphore_t* wait_semaphore_1 = CreateSemaphore(); @@ -154,7 +152,7 @@ TEST_F(semaphore_submission_test, SubmitWithMultipleSemaphores) { } // Tests we can wait on both host and device semaphore to singal. -TEST_F(semaphore_submission_test, WaitAllHostAndDeviceSemaphores) { +TEST_F(SemaphoreSubmissionTest, WaitAllHostAndDeviceSemaphores) { iree_hal_command_buffer_t* command_buffer = CreateEmptyCommandBuffer(); // Create two semaphores, one for the host thread to wait on, and one for the @@ -217,7 +215,7 @@ TEST_F(semaphore_submission_test, WaitAllHostAndDeviceSemaphores) { // Tests that we can wait on any host and device semaphore to singal, // and device signals. -TEST_F(semaphore_submission_test, +TEST_F(SemaphoreSubmissionTest, WaitAnyHostAndDeviceSemaphoresAndDeviceSignals) { iree_hal_command_buffer_t* command_buffer = CreateEmptyCommandBuffer(); @@ -286,8 +284,7 @@ TEST_F(semaphore_submission_test, // Tests we can wait on any host and device semaphore to singal, // and host signals. -TEST_F(semaphore_submission_test, - WaitAnyHostAndDeviceSemaphoresAndHostSignals) { +TEST_F(SemaphoreSubmissionTest, WaitAnyHostAndDeviceSemaphoresAndHostSignals) { iree_hal_command_buffer_t* command_buffer = CreateEmptyCommandBuffer(); // Create two semaphores, one for the host thread to wait on, and one for the @@ -359,7 +356,7 @@ TEST_F(semaphore_submission_test, // Test device -> device synchronization: submit two batches with a // semaphore signal -> wait dependency. -TEST_F(semaphore_submission_test, IntermediateSemaphoreBetweenDeviceBatches) { +TEST_F(SemaphoreSubmissionTest, IntermediateSemaphoreBetweenDeviceBatches) { // The signaling relationship is // command_buffer1 -> semaphore1 -> command_buffer2 -> semaphore2 @@ -419,7 +416,7 @@ TEST_F(semaphore_submission_test, IntermediateSemaphoreBetweenDeviceBatches) { // Test device -> device synchronization: submit multiple batches with // multiple later batches waiting on the same signaling from a former batch. -TEST_F(semaphore_submission_test, TwoBatchesWaitingOn1FormerBatchAmongst2) { +TEST_F(SemaphoreSubmissionTest, TwoBatchesWaitingOn1FormerBatchAmongst2) { // The signaling-wait relation is: // command_buffer11 command_buffer12 // ↓ @@ -502,7 +499,7 @@ TEST_F(semaphore_submission_test, TwoBatchesWaitingOn1FormerBatchAmongst2) { // Test device -> device synchronization: submit multiple batches with // a former batch signaling a value greater than all other batches' (different) // wait values. -TEST_F(semaphore_submission_test, TwoBatchesWaitingOnDifferentSemaphoreValues) { +TEST_F(SemaphoreSubmissionTest, TwoBatchesWaitingOnDifferentSemaphoreValues) { // The signal-wait relation is // // command_buffer11 @@ -590,7 +587,7 @@ TEST_F(semaphore_submission_test, TwoBatchesWaitingOnDifferentSemaphoreValues) { // Test host + device -> device synchronization: submit two batches // with a later batch waiting on both a host and device signal to proceed. -TEST_F(semaphore_submission_test, BatchWaitingOnAnotherAndHostSignal) { +TEST_F(SemaphoreSubmissionTest, BatchWaitingOnAnotherAndHostSignal) { // Signal/wait relation: // // command_buffer1 @@ -665,7 +662,7 @@ TEST_F(semaphore_submission_test, BatchWaitingOnAnotherAndHostSignal) { // Test device -> host + device synchronization: submit two batches // with a former batch signaling to enable both host and device to proceed. -TEST_F(semaphore_submission_test, DeviceBatchSignalAnotherAndHost) { +TEST_F(SemaphoreSubmissionTest, DeviceBatchSignalAnotherAndHost) { // Signal-wait relation: // // command_buffer1 @@ -753,7 +750,7 @@ TEST_F(semaphore_submission_test, DeviceBatchSignalAnotherAndHost) { // Test signaling a larger value before enqueuing waiting a smaller // value to the device. -TEST_F(semaphore_submission_test, BatchWaitingOnSmallerValueAfterSignaled) { +TEST_F(SemaphoreSubmissionTest, BatchWaitingOnSmallerValueAfterSignaled) { // signal-wait relation: // // signal value 2 @@ -797,7 +794,7 @@ TEST_F(semaphore_submission_test, BatchWaitingOnSmallerValueAfterSignaled) { // Test signaling a larger value after enqueuing waiting a smaller // value to the device. -TEST_F(semaphore_submission_test, BatchWaitingOnSmallerValueBeforeSignaled) { +TEST_F(SemaphoreSubmissionTest, BatchWaitingOnSmallerValueBeforeSignaled) { // signal-wait relation: // // signal value 2 @@ -844,8 +841,6 @@ TEST_F(semaphore_submission_test, BatchWaitingOnSmallerValueBeforeSignaled) { iree_hal_command_buffer_release(command_buffer); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_SEMAPHORE_SUBMISSION_TEST_H_ diff --git a/runtime/src/iree/hal/cts/semaphore_test.h b/runtime/src/iree/hal/cts/semaphore_test.h index da2f297d069c..2f84b6f9f4d3 100644 --- a/runtime/src/iree/hal/cts/semaphore_test.h +++ b/runtime/src/iree/hal/cts/semaphore_test.h @@ -17,15 +17,14 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { + using namespace std::chrono_literals; -class semaphore_test : public CTSTestBase<> {}; +class SemaphoreTest : public CTSTestBase<> {}; // Tests that a semaphore that is unused properly cleans itself up. -TEST_F(semaphore_test, NoOp) { +TEST_F(SemaphoreTest, NoOp) { iree_hal_semaphore_t* semaphore = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 123ull, &semaphore)); @@ -37,7 +36,7 @@ TEST_F(semaphore_test, NoOp) { } // Tests that a semaphore will accept new values as it is signaled. -TEST_F(semaphore_test, NormalSignaling) { +TEST_F(SemaphoreTest, NormalSignaling) { iree_hal_semaphore_t* semaphore = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 2ull, &semaphore)); @@ -59,7 +58,7 @@ TEST_F(semaphore_test, NormalSignaling) { // while others may accept the new, decreasing, values. // Tests semaphore failure handling. -TEST_F(semaphore_test, Failure) { +TEST_F(SemaphoreTest, Failure) { iree_hal_semaphore_t* semaphore = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 2ull, &semaphore)); @@ -80,7 +79,7 @@ TEST_F(semaphore_test, Failure) { } // Tests waiting on no semaphores. -TEST_F(semaphore_test, EmptyWait) { +TEST_F(SemaphoreTest, EmptyWait) { IREE_ASSERT_OK(iree_hal_device_wait_semaphores( device_, IREE_HAL_WAIT_MODE_ANY, iree_hal_semaphore_list_empty(), iree_make_deadline(IREE_TIME_INFINITE_FUTURE))); @@ -97,7 +96,7 @@ TEST_F(semaphore_test, EmptyWait) { } // Tests waiting on a semaphore that has already been signaled. -TEST_F(semaphore_test, WaitAlreadySignaled) { +TEST_F(SemaphoreTest, WaitAlreadySignaled) { iree_hal_semaphore_t* semaphore = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 2ull, &semaphore)); @@ -116,7 +115,7 @@ TEST_F(semaphore_test, WaitAlreadySignaled) { } // Tests waiting on a semaphore that has not been signaled. -TEST_F(semaphore_test, WaitUnsignaled) { +TEST_F(SemaphoreTest, WaitUnsignaled) { iree_hal_semaphore_t* semaphore = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 2ull, &semaphore)); @@ -130,7 +129,7 @@ TEST_F(semaphore_test, WaitUnsignaled) { } // Tests waiting on a semaphore that has signals past the desired value. -TEST_F(semaphore_test, WaitLaterSignaledBeyond) { +TEST_F(SemaphoreTest, WaitLaterSignaledBeyond) { iree_hal_semaphore_t* semaphore = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 2ull, &semaphore)); @@ -152,7 +151,7 @@ TEST_F(semaphore_test, WaitLaterSignaledBeyond) { // return UnknownError while others may succeed. // Tests IREE_HAL_WAIT_MODE_ALL when not all are signaled. -TEST_F(semaphore_test, WaitAllButNotAllSignaled) { +TEST_F(SemaphoreTest, WaitAllButNotAllSignaled) { iree_hal_semaphore_t* semaphore_a = NULL; iree_hal_semaphore_t* semaphore_b = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 0ull, &semaphore_a)); @@ -177,7 +176,7 @@ TEST_F(semaphore_test, WaitAllButNotAllSignaled) { } // Tests IREE_HAL_WAIT_MODE_ALL when all are signaled. -TEST_F(semaphore_test, WaitAllAndAllSignaled) { +TEST_F(SemaphoreTest, WaitAllAndAllSignaled) { iree_hal_semaphore_t* semaphore_a = NULL; iree_hal_semaphore_t* semaphore_b = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 1ull, &semaphore_a)); @@ -202,7 +201,7 @@ TEST_F(semaphore_test, WaitAllAndAllSignaled) { } // Tests IREE_HAL_WAIT_MODE_ANY. -TEST_F(semaphore_test, WaitAnyAlreadySignaled) { +TEST_F(SemaphoreTest, WaitAnyAlreadySignaled) { iree_hal_semaphore_t* semaphore_a = NULL; iree_hal_semaphore_t* semaphore_b = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 0ull, &semaphore_a)); @@ -223,7 +222,7 @@ TEST_F(semaphore_test, WaitAnyAlreadySignaled) { iree_hal_semaphore_release(semaphore_b); } -TEST_F(semaphore_test, WaitAnyLaterSignaled) { +TEST_F(SemaphoreTest, WaitAnyLaterSignaled) { iree_hal_semaphore_t* semaphore_a = NULL; iree_hal_semaphore_t* semaphore_b = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 0ull, &semaphore_a)); @@ -253,7 +252,7 @@ TEST_F(semaphore_test, WaitAnyLaterSignaled) { // Tests threading behavior by ping-ponging between the test main thread and // a little thread. -TEST_F(semaphore_test, PingPong) { +TEST_F(SemaphoreTest, PingPong) { iree_hal_semaphore_t* a2b = NULL; iree_hal_semaphore_t* b2a = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 0ull, &a2b)); @@ -278,7 +277,7 @@ TEST_F(semaphore_test, PingPong) { } // Waiting the same value multiple times. -TEST_F(semaphore_test, WaitOnTheSameValueMultipleTimes) { +TEST_F(SemaphoreTest, WaitOnTheSameValueMultipleTimes) { iree_hal_semaphore_t* semaphore = CreateSemaphore(); std::thread thread( [&]() { IREE_ASSERT_OK(iree_hal_semaphore_signal(semaphore, 1)); }); @@ -297,7 +296,7 @@ TEST_F(semaphore_test, WaitOnTheSameValueMultipleTimes) { } // Waiting for a finite amount of time. -TEST_F(semaphore_test, WaitForFiniteTime) { +TEST_F(SemaphoreTest, WaitForFiniteTime) { auto generic_test_fn = [this](auto wait_fn) { iree_hal_semaphore_t* semaphore = this->CreateSemaphore(); @@ -341,7 +340,7 @@ TEST_F(semaphore_test, WaitForFiniteTime) { } // Wait on all semaphores on multiple places simultaneously. -TEST_F(semaphore_test, SimultaneousMultiWaitAll) { +TEST_F(SemaphoreTest, SimultaneousMultiWaitAll) { iree_hal_semaphore_t* semaphore1 = this->CreateSemaphore(); iree_hal_semaphore_t* semaphore2 = this->CreateSemaphore(); @@ -378,8 +377,6 @@ TEST_F(semaphore_test, SimultaneousMultiWaitAll) { iree_hal_semaphore_release(semaphore2); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_SEMAPHORE_TEST_H_ diff --git a/runtime/src/iree/hal/cts/testdata/command_buffer_dispatch_test.mlir b/runtime/src/iree/hal/cts/testdata/command_buffer_dispatch_test.mlir index 366a9f34bb0e..0067230abaf7 100644 --- a/runtime/src/iree/hal/cts/testdata/command_buffer_dispatch_test.mlir +++ b/runtime/src/iree/hal/cts/testdata/command_buffer_dispatch_test.mlir @@ -1,8 +1,8 @@ // Bootstrapped from this source IR: // -// func.func @abs(%input : tensor) -> (tensor) { -// %result = math.absf %input : tensor -// return %result : tensor +// func.func @abs(%input : tensor<2xf32>) -> (tensor<2xf32>) { +// %result = math.absf %input : tensor<2xf32> +// return %result : tensor<2xf32> // } #pipeline_layout = #hal.pipeline.layout - %1 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(32) offset(%c0) : !flow.dispatch.tensor + %0 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(4) offset(%c0) : !flow.dispatch.tensor> + %1 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(4) offset(%c0) : !flow.dispatch.tensor> - %2 = flow.dispatch.tensor.load %0, offsets = [], sizes = [], strides = [] : !flow.dispatch.tensor -> tensor - %3 = tensor.empty() : tensor - %4 = linalg.generic {indexing_maps = [affine_map<() -> ()>, affine_map<() -> ()>], iterator_types = []} ins(%2 : tensor) outs(%3 : tensor) { + %2 = flow.dispatch.tensor.load %0, offsets = [0], sizes = [2], strides = [1] : !flow.dispatch.tensor> -> tensor<2xf32> + %3 = tensor.empty() : tensor<2xf32> + %4 = linalg.generic {indexing_maps = [affine_map<(d0) -> (d0)>, affine_map<(d0) -> (d0)>], iterator_types = ["parallel"]} ins(%2 : tensor<2xf32>) outs(%3 : tensor<2xf32>) { ^bb0(%arg0: f32, %arg1: f32): %5 = math.absf %arg0 : f32 linalg.yield %5 : f32 - } -> tensor - flow.dispatch.tensor.store %4, %1, offsets = [], sizes = [], strides = [] : tensor -> !flow.dispatch.tensor + } -> tensor<2xf32> + flow.dispatch.tensor.store %4, %1, offsets = [0], sizes = [2], strides = [1] : tensor<2xf32> -> !flow.dispatch.tensor> return } diff --git a/runtime/src/iree/hal/drivers/cuda/pending_queue_actions.c b/runtime/src/iree/hal/drivers/cuda/pending_queue_actions.c index 717932d94457..86d5762379f2 100644 --- a/runtime/src/iree/hal/drivers/cuda/pending_queue_actions.c +++ b/runtime/src/iree/hal/drivers/cuda/pending_queue_actions.c @@ -738,9 +738,14 @@ static iree_status_t iree_hal_cuda_pending_queue_actions_issue_execution( } else { iree_hal_command_buffer_t* stream_command_buffer = NULL; iree_hal_command_buffer_mode_t mode = + iree_hal_command_buffer_mode(command_buffer) | IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT | IREE_HAL_COMMAND_BUFFER_MODE_ALLOW_INLINE_EXECUTION | - IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED; + // NOTE: we need to validate if a binding table is provided as the + // bindings were not known when it was originally recorded. + (iree_hal_buffer_binding_table_is_empty(binding_table) + ? IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED + : 0); IREE_RETURN_AND_END_ZONE_IF_ERROR( z0, iree_hal_cuda_device_create_stream_command_buffer( action->device, mode, IREE_HAL_COMMAND_CATEGORY_ANY, diff --git a/runtime/src/iree/hal/drivers/hip/cts/CMakeLists.txt b/runtime/src/iree/hal/drivers/hip/cts/CMakeLists.txt index 5d7c10943430..1b5b5b7c3ae1 100644 --- a/runtime/src/iree/hal/drivers/hip/cts/CMakeLists.txt +++ b/runtime/src/iree/hal/drivers/hip/cts/CMakeLists.txt @@ -60,12 +60,13 @@ iree_hal_cts_test_suite( DEPS iree::hal::drivers::hip::registration EXCLUDED_TESTS - # These tests fail with: - # UNAVAILABLE; missing hipDrvGraphAddMemcpyNode symbol; - # cannot use graph-based command buffer - "command_buffer" - "command_buffer_dispatch" - "file" + # These tests fail with: + # UNAVAILABLE; missing hipDrvGraphAddMemcpyNode symbol; + # cannot use graph-based command buffer + "command_buffer_copy_buffer" + "command_buffer_dispatch" + "command_buffer_update_buffer" + "file" # HAL event is unimplemented for now. "event" LABELS diff --git a/runtime/src/iree/hal/drivers/hip/hip_device.c b/runtime/src/iree/hal/drivers/hip/hip_device.c index 6a10fcbf60b2..3378758c5971 100644 --- a/runtime/src/iree/hal/drivers/hip/hip_device.c +++ b/runtime/src/iree/hal/drivers/hip/hip_device.c @@ -549,11 +549,21 @@ static iree_status_t iree_hal_hip_device_create_command_buffer( } switch (device->params.command_buffer_mode) { case IREE_HAL_HIP_COMMAND_BUFFER_MODE_GRAPH: - return iree_hal_hip_graph_command_buffer_create( - iree_hal_device_allocator(base_device), device->hip_symbols, - device->tracing_context, device->hip_context, mode, - command_categories, queue_affinity, binding_capacity, - &device->block_pool, device->host_allocator, out_command_buffer); + // TODO(indirect-cmd): when we can record indirect graphs we won't need + // to use deferred command buffers - this is here to emulate indirect + // command buffers. + if (binding_capacity > 0) { + return iree_hal_deferred_command_buffer_create( + iree_hal_device_allocator(base_device), mode, command_categories, + binding_capacity, &device->block_pool, + iree_hal_device_host_allocator(base_device), out_command_buffer); + } else { + return iree_hal_hip_graph_command_buffer_create( + iree_hal_device_allocator(base_device), device->hip_symbols, + device->tracing_context, device->hip_context, mode, + command_categories, queue_affinity, binding_capacity, + &device->block_pool, device->host_allocator, out_command_buffer); + } case IREE_HAL_HIP_COMMAND_BUFFER_MODE_STREAM: return iree_hal_deferred_command_buffer_create( iree_hal_device_allocator(base_device), mode, command_categories, diff --git a/runtime/src/iree/hal/drivers/hip/pending_queue_actions.c b/runtime/src/iree/hal/drivers/hip/pending_queue_actions.c index 7e1dcff66c9e..7fe40b5601c0 100644 --- a/runtime/src/iree/hal/drivers/hip/pending_queue_actions.c +++ b/runtime/src/iree/hal/drivers/hip/pending_queue_actions.c @@ -743,9 +743,14 @@ static iree_status_t iree_hal_hip_pending_queue_actions_issue_execution( } else { iree_hal_command_buffer_t* stream_command_buffer = NULL; iree_hal_command_buffer_mode_t mode = + iree_hal_command_buffer_mode(command_buffer) | IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT | IREE_HAL_COMMAND_BUFFER_MODE_ALLOW_INLINE_EXECUTION | - IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED; + // NOTE: we need to validate if a binding table is provided as the + // bindings were not known when it was originally recorded. + (iree_hal_buffer_binding_table_is_empty(binding_table) + ? IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED + : 0); IREE_RETURN_AND_END_ZONE_IF_ERROR( z0, iree_hal_hip_device_create_stream_command_buffer( action->device, mode, IREE_HAL_COMMAND_CATEGORY_ANY, diff --git a/runtime/src/iree/hal/drivers/local_sync/sync_device.c b/runtime/src/iree/hal/drivers/local_sync/sync_device.c index e3a384c65693..b29174378328 100644 --- a/runtime/src/iree/hal/drivers/local_sync/sync_device.c +++ b/runtime/src/iree/hal/drivers/local_sync/sync_device.c @@ -398,11 +398,27 @@ static iree_status_t iree_hal_sync_device_apply_deferred_command_buffers( // Stack allocate storage for an inline command buffer we'll use to replay // the deferred command buffers. We want to reset it between each apply so // that we don't get state carrying across. - iree_host_size_t storage_size = iree_hal_inline_command_buffer_size( - IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED, - /*binding_capacity=*/0); + iree_host_size_t max_storage_size = 0; + for (iree_host_size_t i = 0; i < command_buffer_count; ++i) { + iree_hal_command_buffer_t* command_buffer = command_buffers[i]; + iree_hal_buffer_binding_table_t binding_table = + binding_tables ? binding_tables[i] + : iree_hal_buffer_binding_table_empty(); + max_storage_size = iree_max( + max_storage_size, + iree_hal_inline_command_buffer_size( + iree_hal_command_buffer_mode(command_buffer) | + IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT | + IREE_HAL_COMMAND_BUFFER_MODE_ALLOW_INLINE_EXECUTION | + // NOTE: we need to validate if a binding table is provided as + // the bindings were not known when it was originally recorded. + (iree_hal_buffer_binding_table_is_empty(binding_table) + ? IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED + : 0), + /*binding_capacity=*/0)); + } iree_byte_span_t storage = - iree_make_byte_span(iree_alloca(storage_size), storage_size); + iree_make_byte_span(iree_alloca(max_storage_size), max_storage_size); // NOTE: we ignore any inline command buffers that may be passed in as they've // already executed during recording. The caller is probably in for a bad time @@ -419,9 +435,15 @@ static iree_status_t iree_hal_sync_device_apply_deferred_command_buffers( IREE_RETURN_IF_ERROR(iree_hal_inline_command_buffer_initialize( device->device_allocator, iree_hal_command_buffer_mode(command_buffer) | + IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT | IREE_HAL_COMMAND_BUFFER_MODE_ALLOW_INLINE_EXECUTION | - IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED, - IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, + // NOTE: we need to validate if a binding table is provided as the + // bindings were not known when it was originally recorded. + (iree_hal_buffer_binding_table_is_empty(binding_table) + ? IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED + : 0), + iree_hal_command_buffer_allowed_categories(command_buffer), + IREE_HAL_QUEUE_AFFINITY_ANY, /*binding_capacity=*/0, device->host_allocator, storage, &inline_command_buffer)); iree_status_t status = iree_hal_deferred_command_buffer_apply( diff --git a/runtime/src/iree/hal/drivers/local_task/task_queue.c b/runtime/src/iree/hal/drivers/local_task/task_queue.c index ebf093d9eb31..dde76b5ae9ab 100644 --- a/runtime/src/iree/hal/drivers/local_task/task_queue.c +++ b/runtime/src/iree/hal/drivers/local_task/task_queue.c @@ -189,6 +189,8 @@ typedef struct iree_hal_task_queue_issue_cmd_t { iree_hal_task_queue_t* queue; // A resource set containing all binding table buffers. + // Owned by the retire command and any resources added will be retained until + // the submission has completed (or failed). iree_hal_resource_set_t* resource_set; // Command buffers to be issued in the order they appeared in the submission. @@ -209,14 +211,21 @@ static iree_status_t iree_hal_task_queue_issue_cmd_deferred( // they may not run immediately. iree_hal_command_buffer_t* task_command_buffer = NULL; IREE_RETURN_AND_END_ZONE_IF_ERROR( - z0, iree_hal_task_command_buffer_create( - cmd->queue->device_allocator, &cmd->queue->scope, - iree_hal_command_buffer_mode(command_buffer), - iree_hal_command_buffer_allowed_categories(command_buffer), - cmd->queue->affinity, /*binding_capacity=*/0, - cmd->queue->large_block_pool, - iree_hal_allocator_host_allocator(cmd->queue->device_allocator), - &task_command_buffer)); + z0, + iree_hal_task_command_buffer_create( + cmd->queue->device_allocator, &cmd->queue->scope, + iree_hal_command_buffer_mode(command_buffer) | + IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT | + // NOTE: we need to validate if a binding table is provided as the + // bindings were not known when it was originally recorded. + (iree_hal_buffer_binding_table_is_empty(binding_table) + ? IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED + : 0), + iree_hal_command_buffer_allowed_categories(command_buffer), + cmd->queue->affinity, /*binding_capacity=*/0, + cmd->queue->large_block_pool, + iree_hal_allocator_host_allocator(cmd->queue->device_allocator), + &task_command_buffer)); // Keep the command buffer live until the queue operation completes. iree_status_t status = @@ -242,6 +251,9 @@ static iree_status_t iree_hal_task_queue_issue_cmd_deferred( cmd->task.header.completion_task, cmd->arena, pending_submission)); + // Still retained in the resource set until retirement. + iree_hal_command_buffer_release(task_command_buffer); + IREE_TRACE_ZONE_END(z0); return iree_ok_status(); } @@ -287,24 +299,11 @@ static iree_status_t iree_hal_task_queue_issue_cmd( return status; } -// Cleanup for iree_hal_task_queue_issue_cmd_t that releases the retained -// semaphores. -static void iree_hal_task_queue_issue_cmd_cleanup( - iree_task_t* task, iree_status_code_t status_code) { - iree_hal_task_queue_issue_cmd_t* cmd = (iree_hal_task_queue_issue_cmd_t*)task; - if (cmd->resource_set) { - IREE_TRACE_ZONE_BEGIN(z0); - iree_hal_resource_set_free(cmd->resource_set); - IREE_TRACE_ZONE_END(z0); - } -} - // Allocates and initializes a iree_hal_task_queue_issue_cmd_t task. static iree_status_t iree_hal_task_queue_issue_cmd_allocate( void* user_data, iree_task_scope_t* scope, iree_hal_task_queue_t* queue, - iree_host_size_t resource_count, iree_hal_resource_t* const* resources, iree_task_t* retire_task, iree_arena_allocator_t* arena, - iree_task_t** out_issue_task) { + iree_hal_resource_set_t* resource_set, iree_task_t** out_issue_task) { iree_hal_task_submission_batch_t* batch = (iree_hal_task_submission_batch_t*)user_data; @@ -329,30 +328,15 @@ static iree_status_t iree_hal_task_queue_issue_cmd_allocate( iree_task_call_initialize( scope, iree_task_make_call_closure(iree_hal_task_queue_issue_cmd, 0), &cmd->task); - iree_task_set_cleanup_fn(&cmd->task.header, - iree_hal_task_queue_issue_cmd_cleanup); iree_task_set_completion_task(&cmd->task.header, retire_task); cmd->arena = arena; cmd->queue = queue; + cmd->resource_set = resource_set; cmd->command_buffer_count = batch->command_buffer_count; cmd->command_buffers = (iree_hal_command_buffer_t**)((uint8_t*)cmd + sizeof(*cmd)); - bool has_any_deferred = false; - for (iree_host_size_t i = 0; i < batch->command_buffer_count; ++i) { - iree_hal_command_buffer_t* command_buffer = batch->command_buffers[i]; - cmd->command_buffers[i] = batch->command_buffers[i]; - has_any_deferred |= iree_hal_deferred_command_buffer_isa(command_buffer); - } - - // Only create a resource set if we know we need it. - // NOTE: if this fails we'll unwind and release the cmd arena in the caller. - if (has_any_deferred || binding_table_elements_size > 0) { - IREE_RETURN_IF_ERROR( - iree_hal_resource_set_allocate(arena->block_pool, &cmd->resource_set)); - } else { - cmd->resource_set = NULL; - } + memcpy(cmd->command_buffers, batch->command_buffers, command_buffers_size); // Binding tables are optional and we only need this extra work if there were // any non-empty binding tables provided during submission. @@ -363,7 +347,8 @@ static iree_status_t iree_hal_task_queue_issue_cmd_allocate( (iree_hal_buffer_binding_table_t*)((uint8_t*)cmd->command_buffers + command_buffers_size); iree_hal_buffer_binding_t* binding_element_ptr = - (iree_hal_buffer_binding_t*)(cmd->binding_tables + binding_tables_size); + (iree_hal_buffer_binding_t*)((uint8_t*)cmd->binding_tables + + binding_tables_size); for (iree_host_size_t i = 0; i < batch->command_buffer_count; ++i) { iree_host_size_t element_count = batch->binding_tables[i].count; cmd->binding_tables[i].count = element_count; @@ -388,9 +373,6 @@ static iree_status_t iree_hal_task_queue_issue_cmd_allocate( if (iree_status_is_ok(status)) { *out_issue_task = &cmd->task.header; - } else { - iree_hal_resource_set_free(cmd->resource_set); - cmd->resource_set = NULL; } return status; } @@ -418,8 +400,10 @@ typedef struct iree_hal_task_queue_retire_cmd_t { // Resources retained until all have retired. // We could release them earlier but that would require tracking individual // resource-level completion. - iree_host_size_t resource_count; - iree_hal_resource_t* resources[]; + // + // This resource set is allocated from the small block pool and is expected to + // only have a small number of resources (command buffers, etc). + iree_hal_resource_set_t* resource_set; } iree_hal_task_queue_retire_cmd_t; // Retires a submission by signaling semaphores to their desired value and @@ -431,13 +415,11 @@ static iree_status_t iree_hal_task_queue_retire_cmd( (iree_hal_task_queue_retire_cmd_t*)task; IREE_TRACE_ZONE_BEGIN(z0); - // Release command buffers now that all are known to have retired. + // Release retained resources (command buffers, etc). // We do this before signaling so that waiting threads can immediately reuse // resources that are released. - for (iree_host_size_t i = 0; i < cmd->resource_count; ++i) { - iree_hal_resource_release(cmd->resources[i]); - cmd->resources[i] = NULL; - } + iree_hal_resource_set_free(cmd->resource_set); + cmd->resource_set = NULL; // Signal all semaphores to their new values. // Note that if any signal fails then the whole command will fail and all @@ -466,11 +448,9 @@ static void iree_hal_task_queue_retire_cmd_cleanup( // Release resources now that all are known to have retired. // In success cases we try to do this eagerly to allow for more potential // reuse but during full/partial failures they may still be live here. - for (iree_host_size_t i = 0; i < cmd->resource_count; ++i) { - if (cmd->resources[i]) { - iree_hal_resource_release(cmd->resources[i]); - cmd->resources[i] = NULL; - } + if (!cmd->resource_set) { + iree_hal_resource_set_free(cmd->resource_set); + cmd->resource_set = NULL; } // If the command failed then fail all semaphores to ensure future @@ -497,8 +477,7 @@ static void iree_hal_task_queue_retire_cmd_cleanup( // The command will own an arena that can be used for other submission-related // allocations. static iree_status_t iree_hal_task_queue_retire_cmd_allocate( - iree_task_scope_t* scope, iree_host_size_t resource_count, - iree_hal_resource_t* const* resources, + iree_task_scope_t* scope, const iree_hal_semaphore_list_t* signal_semaphores, iree_arena_block_pool_t* block_pool, iree_hal_task_queue_retire_cmd_t** out_cmd) { @@ -508,18 +487,21 @@ static iree_status_t iree_hal_task_queue_retire_cmd_allocate( // Allocate the command from the arena. iree_hal_task_queue_retire_cmd_t* cmd = NULL; - iree_host_size_t total_cmd_size = - sizeof(*cmd) + resource_count * sizeof(*cmd->resources); iree_status_t status = - iree_arena_allocate(&arena, total_cmd_size, (void**)&cmd); - if (iree_status_is_ok(status)) { - iree_task_call_initialize( - scope, iree_task_make_call_closure(iree_hal_task_queue_retire_cmd, 0), - &cmd->task); - iree_task_set_cleanup_fn(&cmd->task.header, - iree_hal_task_queue_retire_cmd_cleanup); + iree_arena_allocate(&arena, sizeof(*cmd), (void**)&cmd); + if (!iree_status_is_ok(status)) { + iree_arena_deinitialize(&arena); + return status; } + iree_task_call_initialize( + scope, iree_task_make_call_closure(iree_hal_task_queue_retire_cmd, 0), + &cmd->task); + iree_task_set_cleanup_fn(&cmd->task.header, + iree_hal_task_queue_retire_cmd_cleanup); + cmd->signal_semaphores = iree_hal_semaphore_list_empty(); + cmd->resource_set = NULL; + // Clone the signal semaphores from the batch - we retain them and their // payloads. if (iree_status_is_ok(status)) { @@ -527,20 +509,22 @@ static iree_status_t iree_hal_task_queue_retire_cmd_allocate( &cmd->signal_semaphores); } + // Create a lightweight resource set to retain any resources used by the + // command. Note that this is coming from the small block pool and is intended + // only for a small number of resources. + if (iree_status_is_ok(status)) { + status = iree_hal_resource_set_allocate(block_pool, &cmd->resource_set); + } + if (iree_status_is_ok(status)) { // Transfer ownership of the arena to command. memcpy(&cmd->arena, &arena, sizeof(cmd->arena)); - - // Retain command buffers. - cmd->resource_count = resource_count; - for (iree_host_size_t i = 0; i < resource_count; ++i) { - iree_hal_resource_t* resource = resources[i]; - iree_hal_resource_retain(resource); - cmd->resources[i] = resource; - } - *out_cmd = cmd; } else { + if (cmd) { + iree_hal_resource_set_free(cmd->resource_set); + iree_hal_semaphore_list_release(&cmd->signal_semaphores); + } iree_arena_deinitialize(&arena); } return status; @@ -599,9 +583,8 @@ void iree_hal_task_queue_trim(iree_hal_task_queue_t* queue) { typedef iree_status_t(IREE_API_PTR* iree_hal_task_queue_issue_t)( void* user_data, iree_task_scope_t* scope, iree_hal_task_queue_t* queue, - iree_host_size_t resource_count, iree_hal_resource_t* const* resources, iree_task_t* retire_task, iree_arena_allocator_t* arena, - iree_task_t** out_issue_task); + iree_hal_resource_set_t* resource_set, iree_task_t** out_issue_task); static iree_status_t iree_hal_task_queue_submit( iree_hal_task_queue_t* queue, iree_hal_semaphore_list_t wait_semaphores, @@ -613,18 +596,27 @@ static iree_status_t iree_hal_task_queue_submit( // arena which we will use to allocate all other commands. iree_hal_task_queue_retire_cmd_t* retire_cmd = NULL; IREE_RETURN_IF_ERROR(iree_hal_task_queue_retire_cmd_allocate( - &queue->scope, resource_count, resources, &signal_semaphores, - queue->small_block_pool, &retire_cmd)); + &queue->scope, &signal_semaphores, queue->small_block_pool, &retire_cmd)); + // If the caller provided any resources they wanted to retain we add them to + // the resource set for them. This is just a helper to avoid needing to pass + // too much state back to issue callbacks. + // // NOTE: if we fail from here on we must drop the retire_cmd arena. iree_status_t status = iree_ok_status(); + if (resource_count > 0) { + status = iree_hal_resource_set_insert(retire_cmd->resource_set, + resource_count, resources); + } // A fence we'll use to detect when the entire submission has completed. // TODO(benvanik): fold into the retire command. iree_task_fence_t* fence = NULL; - status = - iree_task_executor_acquire_fence(queue->executor, &queue->scope, &fence); - iree_task_set_completion_task(&retire_cmd->task.header, &fence->header); + if (iree_status_is_ok(status)) { + status = iree_task_executor_acquire_fence(queue->executor, &queue->scope, + &fence); + iree_task_set_completion_task(&retire_cmd->task.header, &fence->header); + } // Task to fork and wait for unsatisfied semaphore dependencies. // This is optional and only required if we have previous submissions still @@ -640,8 +632,8 @@ static iree_status_t iree_hal_task_queue_submit( // completed and the issued commands may complete in any order. iree_task_t* issue_cmd = NULL; if (iree_status_is_ok(status) && issue != NULL) { - status = issue(user_data, &queue->scope, queue, resource_count, resources, - &retire_cmd->task.header, &retire_cmd->arena, &issue_cmd); + status = issue(user_data, &queue->scope, queue, &retire_cmd->task.header, + &retire_cmd->arena, retire_cmd->resource_set, &issue_cmd); } // Last chance for failure - from here on we are submitting. @@ -715,9 +707,8 @@ iree_status_t iree_hal_task_queue_submit_commands( static iree_status_t iree_hal_task_queue_callback_cmd_allocate( void* user_data, iree_task_scope_t* scope, iree_hal_task_queue_t* queue, - iree_host_size_t resource_count, iree_hal_resource_t* const* resources, iree_task_t* retire_task, iree_arena_allocator_t* arena, - iree_task_t** out_issue_task) { + iree_hal_resource_set_t* resource_set, iree_task_t** out_issue_task) { iree_task_call_closure_t callback = *(iree_task_call_closure_t*)user_data; iree_task_call_t* cmd = NULL; diff --git a/runtime/src/iree/hal/drivers/vulkan/vulkan_device.cc b/runtime/src/iree/hal/drivers/vulkan/vulkan_device.cc index bc7d64ee58aa..a156d1ee36fe 100644 --- a/runtime/src/iree/hal/drivers/vulkan/vulkan_device.cc +++ b/runtime/src/iree/hal/drivers/vulkan/vulkan_device.cc @@ -1747,13 +1747,23 @@ static iree_status_t iree_hal_vulkan_device_queue_execute( iree_hal_command_buffer_t* command_buffer = command_buffers[i]; if (iree_hal_deferred_command_buffer_isa(command_buffers[i])) { iree_hal_command_buffer_t* translated_command_buffer = NULL; + iree_hal_buffer_binding_table_t binding_table = + binding_tables ? binding_tables[i] + : iree_hal_buffer_binding_table_empty(); status = iree_hal_vulkan_device_create_command_buffer( - base_device, iree_hal_command_buffer_mode(command_buffer), + base_device, + iree_hal_command_buffer_mode(command_buffer) | + IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT | + // NOTE: we need to validate if a binding table is provided as the + // bindings were not known when it was originally recorded. + (iree_hal_buffer_binding_table_is_empty(binding_table) + ? IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED + : 0), iree_hal_command_buffer_allowed_categories(command_buffer), queue_affinity, /*binding_capacity=*/0, &translated_command_buffer); if (iree_status_is_ok(status)) { status = iree_hal_deferred_command_buffer_apply( - command_buffer, translated_command_buffer, binding_tables[i]); + command_buffer, translated_command_buffer, binding_table); } translated_command_buffers[i] = translated_command_buffer; } else { @@ -1772,16 +1782,18 @@ static iree_status_t iree_hal_vulkan_device_queue_execute( status = queue->Submit(1, &batch); } - for (iree_host_size_t i = 0; i < command_buffer_count; ++i) { - iree_hal_command_buffer_release(translated_command_buffers[i]); - } - // HACK: we don't track async resource lifetimes so we have to block. if (iree_status_is_ok(status)) { status = iree_hal_semaphore_list_wait(signal_semaphore_list, iree_infinite_timeout()); } + // TODO(indirect-cmd): when async these need to be retained until the + // submission completes. + for (iree_host_size_t i = 0; i < command_buffer_count; ++i) { + iree_hal_command_buffer_release(translated_command_buffers[i]); + } + return status; }