diff --git a/runtime/src/iree/hal/command_buffer.h b/runtime/src/iree/hal/command_buffer.h index 91fb184218e0..38ac03d05f9b 100644 --- a/runtime/src/iree/hal/command_buffer.h +++ b/runtime/src/iree/hal/command_buffer.h @@ -451,6 +451,11 @@ iree_hal_buffer_binding_table_empty(void) { return table; } +static inline bool iree_hal_buffer_binding_table_is_empty( + iree_hal_buffer_binding_table_t binding_table) { + return binding_table.count == 0; +} + // Returns an unretained buffer specified in |buffer_ref| or from // |binding_table| with the slot specified if indirect. If the caller needs to // preserve the buffer for longer than the (known) lifetime of the binding table diff --git a/runtime/src/iree/hal/command_buffer_validation.c b/runtime/src/iree/hal/command_buffer_validation.c index 9086303aeeb6..87f3a4bb9e67 100644 --- a/runtime/src/iree/hal/command_buffer_validation.c +++ b/runtime/src/iree/hal/command_buffer_validation.c @@ -123,15 +123,16 @@ static iree_status_t iree_hal_command_buffer_validate_binding_requirements( // it are in range. if (requirements.max_byte_offset > 0) { iree_device_size_t end = binding.offset + requirements.max_byte_offset; - if (IREE_UNLIKELY(end > binding.length)) { - return iree_make_status(IREE_STATUS_OUT_OF_RANGE, - "at least one command attempted to access an " - "address outside of the valid bound buffer " - "range (length=%" PRIdsz ", end(inc)=%" PRIdsz - ", binding offset=%" PRIdsz - ", binding length=%" PRIdsz ")", - requirements.max_byte_offset, end - 1, - binding.offset, binding.length); + if (IREE_UNLIKELY(end > binding.offset + binding.length)) { + return iree_make_status( + IREE_STATUS_OUT_OF_RANGE, + "at least one command attempted to access an " + "address outside of the valid bound buffer " + "range (length=%" PRIdsz ", end(inc)=%" PRIdsz + ", binding offset=%" PRIdsz ", binding length=%" PRIdsz + ", binding end(inc)=%" PRIdsz ")", + requirements.max_byte_offset, end - 1, binding.offset, binding.length, + binding.offset + binding.length - 1); } } @@ -188,8 +189,11 @@ static iree_status_t iree_hal_command_buffer_validate_buffer_requirements( table_requirements->type |= requirements.type; table_requirements->max_byte_offset = iree_max( table_requirements->max_byte_offset, requirements.max_byte_offset); - table_requirements->min_byte_alignment = iree_device_size_lcm( - table_requirements->min_byte_alignment, requirements.min_byte_alignment); + if (requirements.min_byte_alignment) { + table_requirements->min_byte_alignment = + iree_device_size_lcm(table_requirements->min_byte_alignment, + requirements.min_byte_alignment); + } return iree_ok_status(); } @@ -430,14 +434,19 @@ iree_status_t iree_hal_command_buffer_copy_buffer_validation( // Check for overlap - just like memcpy we don't handle that. // Note that it's only undefined behavior if violated so we are ok if tricky // situations (subspans of subspans of binding table subranges etc) make it - // through. - if (iree_hal_buffer_test_overlap(source_ref.buffer, source_ref.offset, - source_ref.length, target_ref.buffer, - target_ref.offset, target_ref.length) != - IREE_HAL_BUFFER_OVERLAP_DISJOINT) { - return iree_make_status( - IREE_STATUS_INVALID_ARGUMENT, - "source and target ranges overlap within the same buffer"); + // through. This is only possible if both buffers are directly referenced - + // we _could_ try to catch this for indirect references by stashing the + // overlap check metadata for validation when the binding table is available + // but that's too costly to be worth it. + if (source_ref.buffer && target_ref.buffer) { + if (iree_hal_buffer_test_overlap(source_ref.buffer, source_ref.offset, + source_ref.length, target_ref.buffer, + target_ref.offset, target_ref.length) != + IREE_HAL_BUFFER_OVERLAP_DISJOINT) { + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "source and target ranges overlap within the same buffer"); + } } return iree_ok_status(); @@ -571,10 +580,11 @@ iree_status_t iree_hal_command_buffer_push_descriptor_set_validation( // TODO(benvanik): validate set index. // TODO(benvanik): use pipeline layout to derive usage and access bits. + // For now we conservatively say _any_ access may be performed (read/write). iree_hal_buffer_binding_requirements_t requirements = { .required_compatibility = IREE_HAL_BUFFER_COMPATIBILITY_QUEUE_DISPATCH, - // .usage = IREE_HAL_BUFFER_USAGE_DISPATCH_..., - // .access = IREE_HAL_MEMORY_ACCESS_..., + .usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE, + .access = IREE_HAL_MEMORY_ACCESS_ANY, .type = IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE, }; for (iree_host_size_t i = 0; i < binding_count; ++i) { diff --git a/runtime/src/iree/hal/cts/CMakeLists.txt b/runtime/src/iree/hal/cts/CMakeLists.txt index 62e62837e8e9..090a2a635898 100644 --- a/runtime/src/iree/hal/cts/CMakeLists.txt +++ b/runtime/src/iree/hal/cts/CMakeLists.txt @@ -8,8 +8,11 @@ set(IREE_ALL_CTS_TESTS "allocator" "buffer_mapping" "command_buffer" + "command_buffer_copy_buffer" "command_buffer_dispatch" + "command_buffer_fill_buffer" "command_buffer_push_constants" + "command_buffer_update_buffer" "descriptor_set_layout" "driver" "event" @@ -91,6 +94,19 @@ iree_cc_library( TESTONLY ) +iree_cc_library( + NAME + command_buffer_copy_buffer_test_library + HDRS + "command_buffer_copy_buffer_test.h" + DEPS + ::cts_test_base + iree::base + iree::hal + iree::testing::gtest + TESTONLY +) + iree_cc_library( NAME command_buffer_dispatch_test_library @@ -104,6 +120,19 @@ iree_cc_library( TESTONLY ) +iree_cc_library( + NAME + command_buffer_fill_buffer_test_library + HDRS + "command_buffer_fill_buffer_test.h" + DEPS + ::cts_test_base + iree::base + iree::hal + iree::testing::gtest + TESTONLY +) + iree_cc_library( NAME command_buffer_push_constants_test_library @@ -117,6 +146,19 @@ iree_cc_library( TESTONLY ) +iree_cc_library( + NAME + command_buffer_update_buffer_test_library + HDRS + "command_buffer_update_buffer_test.h" + DEPS + ::cts_test_base + iree::base + iree::hal + iree::testing::gtest + TESTONLY +) + iree_cc_library( NAME descriptor_set_layout_test_library diff --git a/runtime/src/iree/hal/cts/allocator_test.h b/runtime/src/iree/hal/cts/allocator_test.h index f8de4149047b..b6a1e50e4d5e 100644 --- a/runtime/src/iree/hal/cts/allocator_test.h +++ b/runtime/src/iree/hal/cts/allocator_test.h @@ -13,17 +13,13 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { namespace { - constexpr iree_device_size_t kAllocationSize = 1024; - } // namespace -class allocator_test : public CTSTestBase<> {}; +class AllocatorTest : public CTSTestBase<> {}; // All allocators must support some baseline capabilities. // @@ -31,7 +27,7 @@ class allocator_test : public CTSTestBase<> {}; // driver implementations or target devices, such as: // IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL // IREE_HAL_BUFFER_USAGE_MAPPING -TEST_F(allocator_test, BaselineBufferCompatibility) { +TEST_F(AllocatorTest, BaselineBufferCompatibility) { // Need at least one way to get data between the host and device. iree_hal_buffer_params_t host_local_params = {0}; host_local_params.type = @@ -80,7 +76,7 @@ TEST_F(allocator_test, BaselineBufferCompatibility) { IREE_HAL_BUFFER_COMPATIBILITY_QUEUE_DISPATCH)); } -TEST_F(allocator_test, AllocateBuffer) { +TEST_F(AllocatorTest, AllocateBuffer) { iree_hal_buffer_params_t params = {0}; params.type = IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL; params.usage = IREE_HAL_BUFFER_USAGE_TRANSFER; @@ -102,7 +98,7 @@ TEST_F(allocator_test, AllocateBuffer) { // While empty allocations aren't particularly useful, they can occur in // practice so we should at least be able to create them without errors. -TEST_F(allocator_test, AllocateEmptyBuffer) { +TEST_F(AllocatorTest, AllocateEmptyBuffer) { iree_hal_buffer_params_t params = {0}; params.type = IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL; params.usage = IREE_HAL_BUFFER_USAGE_TRANSFER; @@ -113,8 +109,6 @@ TEST_F(allocator_test, AllocateEmptyBuffer) { iree_hal_buffer_release(buffer); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_ALLOCATOR_TEST_H_ diff --git a/runtime/src/iree/hal/cts/buffer_mapping_test.h b/runtime/src/iree/hal/cts/buffer_mapping_test.h index c97c1ac1031b..504f3d5e9c40 100644 --- a/runtime/src/iree/hal/cts/buffer_mapping_test.h +++ b/runtime/src/iree/hal/cts/buffer_mapping_test.h @@ -16,9 +16,7 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { using ::testing::ContainerEq; @@ -42,7 +40,7 @@ constexpr iree_device_size_t kDefaultAllocationSize = 1024; // * write with an offset and length // * write into a subspan of a buffer -class buffer_mapping_test : public CTSTestBase<> { +class BufferMappingTest : public CTSTestBase<> { protected: void AllocateUninitializedBuffer(iree_device_size_t buffer_size, iree_hal_buffer_t** out_buffer) { @@ -59,7 +57,7 @@ class buffer_mapping_test : public CTSTestBase<> { } }; -TEST_F(buffer_mapping_test, AllocatorSupportsBufferMapping) { +TEST_F(BufferMappingTest, AllocatorSupportsBufferMapping) { iree_hal_buffer_params_t params = {0}; params.type = IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; params.usage = IREE_HAL_BUFFER_USAGE_MAPPING; @@ -83,7 +81,7 @@ TEST_F(buffer_mapping_test, AllocatorSupportsBufferMapping) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, ZeroWholeBuffer) { +TEST_F(BufferMappingTest, ZeroWholeBuffer) { iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(kDefaultAllocationSize, &buffer); @@ -102,7 +100,7 @@ TEST_F(buffer_mapping_test, ZeroWholeBuffer) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, ZeroWithOffset) { +TEST_F(BufferMappingTest, ZeroWithOffset) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -128,7 +126,7 @@ TEST_F(buffer_mapping_test, ZeroWithOffset) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, ZeroSubspan) { +TEST_F(BufferMappingTest, ZeroSubspan) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -171,7 +169,7 @@ TEST_F(buffer_mapping_test, ZeroSubspan) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, FillEmpty) { +TEST_F(BufferMappingTest, FillEmpty) { iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(kDefaultAllocationSize, &buffer); @@ -195,7 +193,7 @@ TEST_F(buffer_mapping_test, FillEmpty) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, FillWholeBuffer) { +TEST_F(BufferMappingTest, FillWholeBuffer) { iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(kDefaultAllocationSize, &buffer); @@ -217,7 +215,7 @@ TEST_F(buffer_mapping_test, FillWholeBuffer) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, FillWithOffset) { +TEST_F(BufferMappingTest, FillWithOffset) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -244,7 +242,7 @@ TEST_F(buffer_mapping_test, FillWithOffset) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, FillSubspan) { +TEST_F(BufferMappingTest, FillSubspan) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -288,7 +286,7 @@ TEST_F(buffer_mapping_test, FillSubspan) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, ReadData) { +TEST_F(BufferMappingTest, ReadData) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -325,7 +323,7 @@ TEST_F(buffer_mapping_test, ReadData) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, ReadDataSubspan) { +TEST_F(BufferMappingTest, ReadDataSubspan) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -368,7 +366,7 @@ TEST_F(buffer_mapping_test, ReadDataSubspan) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, WriteDataWholeBuffer) { +TEST_F(BufferMappingTest, WriteDataWholeBuffer) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -390,7 +388,7 @@ TEST_F(buffer_mapping_test, WriteDataWholeBuffer) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, WriteDataWithOffset) { +TEST_F(BufferMappingTest, WriteDataWithOffset) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -417,7 +415,7 @@ TEST_F(buffer_mapping_test, WriteDataWithOffset) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, WriteDataSubspan) { +TEST_F(BufferMappingTest, WriteDataSubspan) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -459,7 +457,7 @@ TEST_F(buffer_mapping_test, WriteDataSubspan) { iree_hal_buffer_release(buffer); } -TEST_F(buffer_mapping_test, CopyData) { +TEST_F(BufferMappingTest, CopyData) { iree_hal_buffer_t* buffer_a = NULL; iree_hal_buffer_t* buffer_b = NULL; AllocateUninitializedBuffer(kDefaultAllocationSize, &buffer_a); @@ -490,7 +488,7 @@ TEST_F(buffer_mapping_test, CopyData) { // Maps a buffer range for reading from device -> host. // This is roughly what iree_hal_buffer_map_read does internally. -TEST_F(buffer_mapping_test, MapRangeRead) { +TEST_F(BufferMappingTest, MapRangeRead) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -520,7 +518,7 @@ TEST_F(buffer_mapping_test, MapRangeRead) { // Maps a buffer range for writing from host -> device. // This is roughly what iree_hal_buffer_map_write does internally. -TEST_F(buffer_mapping_test, MapRangeWrite) { +TEST_F(BufferMappingTest, MapRangeWrite) { iree_device_size_t buffer_size = 16; iree_hal_buffer_t* buffer = NULL; AllocateUninitializedBuffer(buffer_size, &buffer); @@ -550,8 +548,6 @@ TEST_F(buffer_mapping_test, MapRangeWrite) { iree_hal_buffer_release(buffer); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_BUFFER_MAPPING_TEST_H_ diff --git a/runtime/src/iree/hal/cts/command_buffer_copy_buffer_test.h b/runtime/src/iree/hal/cts/command_buffer_copy_buffer_test.h new file mode 100644 index 000000000000..70735c070a27 --- /dev/null +++ b/runtime/src/iree/hal/cts/command_buffer_copy_buffer_test.h @@ -0,0 +1,275 @@ +// Copyright 2019 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef IREE_HAL_CTS_COMMAND_BUFFER_COPY_BUFFER_TEST_H_ +#define IREE_HAL_CTS_COMMAND_BUFFER_COPY_BUFFER_TEST_H_ + +#include +#include + +#include "iree/base/api.h" +#include "iree/hal/api.h" +#include "iree/hal/cts/cts_test_base.h" +#include "iree/testing/gtest.h" +#include "iree/testing/status_matchers.h" + +namespace iree::hal::cts { + +using ::testing::ContainerEq; + +namespace { +constexpr iree_device_size_t kDefaultAllocationSize = 1024; +} // namespace + +class CommandBufferCopyBufferTest : public CTSTestBase<> {}; + +TEST_F(CommandBufferCopyBufferTest, CopyWholeBuffer) { + iree_hal_command_buffer_t* command_buffer = NULL; + IREE_ASSERT_OK(iree_hal_command_buffer_create( + device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, + IREE_HAL_COMMAND_CATEGORY_TRANSFER, IREE_HAL_QUEUE_AFFINITY_ANY, + /*binding_capacity=*/0, &command_buffer)); + + uint8_t i8_val = 0x54; + std::vector reference_buffer(kDefaultAllocationSize); + std::memset(reference_buffer.data(), i8_val, kDefaultAllocationSize); + + // Create and fill a host buffer. + iree_hal_buffer_params_t host_params = {0}; + host_params.type = + IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE; + host_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | + IREE_HAL_BUFFER_USAGE_TRANSFER | + IREE_HAL_BUFFER_USAGE_MAPPING; + iree_hal_buffer_t* host_buffer = nullptr; + IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( + device_allocator_, host_params, kDefaultAllocationSize, &host_buffer)); + IREE_ASSERT_OK(iree_hal_device_transfer_h2d( + device_, reference_buffer.data(), host_buffer, 0, reference_buffer.size(), + IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, iree_infinite_timeout())); + + // Create a device buffer. + iree_hal_buffer_params_t device_params = {0}; + device_params.type = + IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL | IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; + device_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | + IREE_HAL_BUFFER_USAGE_TRANSFER | + IREE_HAL_BUFFER_USAGE_MAPPING; + iree_hal_buffer_t* device_buffer = nullptr; + IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( + device_allocator_, device_params, kDefaultAllocationSize, + &device_buffer)); + + // Copy the host buffer to the device buffer. + IREE_ASSERT_OK(iree_hal_command_buffer_begin(command_buffer)); + IREE_ASSERT_OK(iree_hal_command_buffer_copy_buffer( + command_buffer, /*source_ref=*/ + iree_hal_make_buffer_ref(host_buffer, 0, kDefaultAllocationSize), + /*target_ref=*/ + iree_hal_make_buffer_ref(device_buffer, 0, kDefaultAllocationSize))); + IREE_ASSERT_OK(iree_hal_command_buffer_end(command_buffer)); + + IREE_ASSERT_OK(SubmitCommandBufferAndWait(command_buffer)); + + // Read the device buffer and compare. + std::vector actual_data(kDefaultAllocationSize); + IREE_ASSERT_OK(iree_hal_device_transfer_d2h( + device_, device_buffer, /*source_offset=*/0, + /*target_buffer=*/actual_data.data(), + /*data_length=*/kDefaultAllocationSize, + IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, iree_infinite_timeout())); + EXPECT_THAT(actual_data, ContainerEq(reference_buffer)); + + // Must release the command buffer before resources used by it. + iree_hal_command_buffer_release(command_buffer); + iree_hal_buffer_release(device_buffer); + iree_hal_buffer_release(host_buffer); +} + +TEST_F(CommandBufferCopyBufferTest, CopySubBuffer) { + iree_hal_command_buffer_t* command_buffer = NULL; + IREE_ASSERT_OK(iree_hal_command_buffer_create( + device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, + IREE_HAL_COMMAND_CATEGORY_TRANSFER, IREE_HAL_QUEUE_AFFINITY_ANY, + /*binding_capacity=*/0, &command_buffer)); + + iree_hal_buffer_params_t device_params = {0}; + device_params.type = + IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL | IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; + device_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | + IREE_HAL_BUFFER_USAGE_TRANSFER | + IREE_HAL_BUFFER_USAGE_MAPPING; + iree_hal_buffer_t* device_buffer = NULL; + IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( + device_allocator_, device_params, kDefaultAllocationSize, + &device_buffer)); + + uint8_t i8_val = 0x88; + std::vector reference_buffer(kDefaultAllocationSize); + std::memset(reference_buffer.data() + 8, i8_val, + kDefaultAllocationSize / 2 - 4); + + // Create another host buffer with a smaller size. + iree_hal_buffer_params_t host_params = {0}; + host_params.type = + IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE; + host_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | + IREE_HAL_BUFFER_USAGE_TRANSFER | + IREE_HAL_BUFFER_USAGE_MAPPING; + std::vector host_buffer_data(kDefaultAllocationSize, i8_val); + iree_hal_buffer_t* host_buffer = NULL; + IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( + device_allocator_, host_params, host_buffer_data.size() / 2, + &host_buffer)); + IREE_ASSERT_OK(iree_hal_device_transfer_h2d( + device_, host_buffer_data.data(), host_buffer, 0, + host_buffer_data.size() / 2, IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, + iree_infinite_timeout())); + + // Copy the host buffer to the device buffer; zero fill the untouched bytes. + uint8_t zero_val = 0x0; + IREE_ASSERT_OK(iree_hal_command_buffer_begin(command_buffer)); + IREE_ASSERT_OK(iree_hal_command_buffer_fill_buffer( + command_buffer, + iree_hal_make_buffer_ref(device_buffer, /*target_offset=*/0, + /*length=*/8), + &zero_val, /*pattern_length=*/sizeof(zero_val))); + IREE_ASSERT_OK(iree_hal_command_buffer_copy_buffer( + command_buffer, + iree_hal_make_buffer_ref(/*source_buffer=*/host_buffer, + /*source_offset=*/4, + /*length=*/kDefaultAllocationSize / 2 - 4), + iree_hal_make_buffer_ref(/*target_buffer=*/device_buffer, + /*target_offset=*/8, + /*length=*/kDefaultAllocationSize / 2 - 4))); + IREE_ASSERT_OK(iree_hal_command_buffer_fill_buffer( + command_buffer, + iree_hal_make_buffer_ref( + device_buffer, + /*target_offset=*/8 + kDefaultAllocationSize / 2 - 4, + /*length=*/kDefaultAllocationSize - + (8 + kDefaultAllocationSize / 2 - 4)), + &zero_val, + /*pattern_length=*/sizeof(zero_val))); + IREE_ASSERT_OK(iree_hal_command_buffer_end(command_buffer)); + + IREE_ASSERT_OK(SubmitCommandBufferAndWait(command_buffer)); + + // Read the device buffer and compare. + std::vector actual_data(kDefaultAllocationSize); + IREE_ASSERT_OK(iree_hal_device_transfer_d2h( + device_, device_buffer, /*source_offset=*/0, + /*target_buffer=*/actual_data.data(), + /*data_length=*/kDefaultAllocationSize, + IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, iree_infinite_timeout())); + EXPECT_THAT(actual_data, ContainerEq(reference_buffer)); + + // Must release the command buffer before resources used by it. + iree_hal_command_buffer_release(command_buffer); + iree_hal_buffer_release(device_buffer); + iree_hal_buffer_release(host_buffer); +} + +TEST_F(CommandBufferCopyBufferTest, CopySubBufferIndirect) { + const int kHostBufferSlot = 0; + const int kDeviceBufferSlot = 1; + const int kBindingSlotCount = 2; + iree_hal_command_buffer_t* command_buffer = NULL; + IREE_ASSERT_OK(iree_hal_command_buffer_create( + device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, + IREE_HAL_COMMAND_CATEGORY_TRANSFER, IREE_HAL_QUEUE_AFFINITY_ANY, + kBindingSlotCount, &command_buffer)); + + iree_hal_buffer_params_t device_params = {0}; + device_params.type = + IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL | IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; + device_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | + IREE_HAL_BUFFER_USAGE_TRANSFER | + IREE_HAL_BUFFER_USAGE_MAPPING; + iree_hal_buffer_t* device_buffer = NULL; + IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( + device_allocator_, device_params, kDefaultAllocationSize, + &device_buffer)); + + uint8_t i8_val = 0x88; + std::vector reference_buffer(kDefaultAllocationSize); + std::memset(reference_buffer.data() + 8, i8_val, + kDefaultAllocationSize / 2 - 4); + + // Create another host buffer with a smaller size. + iree_hal_buffer_params_t host_params = {0}; + host_params.type = + IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE; + host_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | + IREE_HAL_BUFFER_USAGE_TRANSFER | + IREE_HAL_BUFFER_USAGE_MAPPING; + std::vector host_buffer_data(kDefaultAllocationSize, i8_val); + iree_hal_buffer_t* host_buffer = NULL; + IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( + device_allocator_, host_params, host_buffer_data.size() / 2, + &host_buffer)); + IREE_ASSERT_OK(iree_hal_device_transfer_h2d( + device_, host_buffer_data.data(), host_buffer, 0, + host_buffer_data.size() / 2, IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, + iree_infinite_timeout())); + + // Copy the host buffer to the device buffer; zero fill the untouched bytes. + uint8_t zero_val = 0x0; + IREE_ASSERT_OK(iree_hal_command_buffer_begin(command_buffer)); + IREE_ASSERT_OK(iree_hal_command_buffer_fill_buffer( + command_buffer, + iree_hal_make_indirect_buffer_ref(kDeviceBufferSlot, /*offset=*/0, + /*length=*/8), + &zero_val, /*pattern_length=*/sizeof(zero_val))); + IREE_ASSERT_OK(iree_hal_command_buffer_copy_buffer( + command_buffer, + iree_hal_make_indirect_buffer_ref( + kHostBufferSlot, + /*offset=*/4, + /*length=*/kDefaultAllocationSize / 2 - 4), + iree_hal_make_indirect_buffer_ref( + kDeviceBufferSlot, + /*offset=*/8, + /*length=*/kDefaultAllocationSize / 2 - 4))); + IREE_ASSERT_OK(iree_hal_command_buffer_fill_buffer( + command_buffer, + iree_hal_make_indirect_buffer_ref( + kDeviceBufferSlot, + /*target_offset=*/8 + kDefaultAllocationSize / 2 - 4, + /*length=*/kDefaultAllocationSize - + (8 + kDefaultAllocationSize / 2 - 4)), + &zero_val, + /*pattern_length=*/sizeof(zero_val))); + IREE_ASSERT_OK(iree_hal_command_buffer_end(command_buffer)); + + const iree_hal_buffer_binding_t bindings[] = { + /*kHostBufferSlot=*/{host_buffer, 0, IREE_WHOLE_BUFFER}, + /*kDeviceBufferSlot=*/{device_buffer, 0, IREE_WHOLE_BUFFER}, + }; + IREE_ASSERT_OK(SubmitCommandBufferAndWait(command_buffer, + iree_hal_buffer_binding_table_t{ + IREE_ARRAYSIZE(bindings), + bindings, + })); + + // Read the device buffer and compare. + std::vector actual_data(kDefaultAllocationSize); + IREE_ASSERT_OK(iree_hal_device_transfer_d2h( + device_, device_buffer, /*source_offset=*/0, + /*target_buffer=*/actual_data.data(), + /*data_length=*/kDefaultAllocationSize, + IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, iree_infinite_timeout())); + EXPECT_THAT(actual_data, ContainerEq(reference_buffer)); + + // Must release the command buffer before resources used by it. + iree_hal_command_buffer_release(command_buffer); + iree_hal_buffer_release(device_buffer); + iree_hal_buffer_release(host_buffer); +} + +} // namespace iree::hal::cts + +#endif // IREE_HAL_CTS_COMMAND_BUFFER_COPY_BUFFER_TEST_H_ diff --git a/runtime/src/iree/hal/cts/command_buffer_dispatch_test.h b/runtime/src/iree/hal/cts/command_buffer_dispatch_test.h index 85f53f2d3347..4d712073fe7f 100644 --- a/runtime/src/iree/hal/cts/command_buffer_dispatch_test.h +++ b/runtime/src/iree/hal/cts/command_buffer_dispatch_test.h @@ -10,15 +10,15 @@ #include "iree/base/api.h" #include "iree/base/string_view.h" #include "iree/hal/api.h" +#include "iree/hal/buffer_view_util.h" #include "iree/hal/cts/cts_test_base.h" #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { -class command_buffer_dispatch_test : public CTSTestBase<> { +class CommandBufferDispatchTest + : public CTSTestBase<::testing::TestWithParam> { protected: void PrepareAbsExecutable() { IREE_ASSERT_OK(iree_hal_executable_cache_create( @@ -76,58 +76,77 @@ class command_buffer_dispatch_test : public CTSTestBase<> { iree_hal_executable_t* executable_ = NULL; }; -TEST_F(command_buffer_dispatch_test, DispatchAbs) { +// Dispatches absf(x) on a subrange (elements 1-2) of a 4 element input buffer. +// input_buffer = [-2.5 -2.5 -2.5 -2.5] +// output_buffer = [-9.0 2.5 2.5 -9.0] +TEST_P(CommandBufferDispatchTest, DispatchAbs) { PrepareAbsExecutable(); - iree_hal_command_buffer_t* command_buffer = NULL; - IREE_ASSERT_OK(iree_hal_command_buffer_create( - device_, - IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT | - IREE_HAL_COMMAND_BUFFER_MODE_ALLOW_INLINE_EXECUTION, - IREE_HAL_COMMAND_CATEGORY_DISPATCH, IREE_HAL_QUEUE_AFFINITY_ANY, - /*binding_capacity=*/0, &command_buffer)); - - IREE_ASSERT_OK(iree_hal_command_buffer_begin(command_buffer)); + // Create input buffer. + iree_hal_buffer_t* input_buffer = NULL; + CreateFilledDeviceBuffer(4 * sizeof(float), -2.5f, &input_buffer); - // Create input and output buffers. - iree_hal_buffer_params_t input_params = {0}; - input_params.type = IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL; - input_params.usage = - IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | IREE_HAL_BUFFER_USAGE_TRANSFER; - iree_hal_buffer_view_t* input_buffer_view = NULL; - float input_data[1] = {-2.5f}; - IREE_ASSERT_OK(iree_hal_buffer_view_allocate_buffer_copy( - device_, device_allocator_, - /*shape_rank=*/0, /*shape=*/NULL, IREE_HAL_ELEMENT_TYPE_FLOAT_32, - IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, input_params, - iree_make_const_byte_span((void*)input_data, sizeof(input_data)), - &input_buffer_view)); - iree_hal_buffer_params_t output_params = {0}; - output_params.type = - IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL | IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; - output_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | - IREE_HAL_BUFFER_USAGE_TRANSFER | - IREE_HAL_BUFFER_USAGE_MAPPING; + // Create output buffer. iree_hal_buffer_t* output_buffer = NULL; - IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( - device_allocator_, output_params, sizeof(float), &output_buffer)); - - iree_hal_buffer_ref_t descriptor_set_bindings[] = { - { + CreateFilledDeviceBuffer(4 * sizeof(float), -9.0f, &output_buffer); + + iree_hal_buffer_ref_t descriptor_set_bindings[2]; + iree_hal_buffer_binding_t bindings[2]; + iree_hal_buffer_binding_table_t binding_table = + iree_hal_buffer_binding_table_empty(); + switch (GetParam()) { + case RecordingType::kDirect: + descriptor_set_bindings[0] = { /*binding=*/0, /*buffer_slot=*/0, - iree_hal_buffer_view_buffer(input_buffer_view), - /*offset=*/0, - iree_hal_buffer_view_byte_length(input_buffer_view), - }, - { + /*buffer=*/input_buffer, + /*offset=*/1 * sizeof(float), + /*length=*/2 * sizeof(float), + }; + descriptor_set_bindings[1] = { /*binding=*/1, /*buffer_slot=*/0, - output_buffer, - iree_hal_buffer_byte_offset(output_buffer), - iree_hal_buffer_byte_length(output_buffer), - }, - }; + /*buffer=*/output_buffer, + /*offset=*/1 * sizeof(float), + /*length=*/2 * sizeof(float), + }; + break; + case RecordingType::kIndirect: + binding_table.count = IREE_ARRAYSIZE(descriptor_set_bindings); + binding_table.bindings = bindings; + bindings[0] = { + /*buffer=*/input_buffer, + /*offset=*/1 * sizeof(float), + /*length=*/2 * sizeof(float), + }; + descriptor_set_bindings[0] = { + /*binding=*/0, + /*buffer_slot=*/0, + /*buffer=*/NULL, + /*offset=*/0, + /*length=*/2 * sizeof(float), + }; + bindings[1] = { + /*buffer=*/output_buffer, + /*offset=*/1 * sizeof(float), + /*length=*/2 * sizeof(float), + }; + descriptor_set_bindings[1] = { + /*binding=*/1, + /*buffer_slot=*/1, + /*buffer=*/NULL, + /*offset=*/0, + /*length=*/2 * sizeof(float), + }; + break; + } + + iree_hal_command_buffer_t* command_buffer = NULL; + IREE_ASSERT_OK(iree_hal_command_buffer_create( + device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, + IREE_HAL_COMMAND_CATEGORY_DISPATCH, IREE_HAL_QUEUE_AFFINITY_ANY, + binding_table.count, &command_buffer)); + IREE_ASSERT_OK(iree_hal_command_buffer_begin(command_buffer)); IREE_ASSERT_OK(iree_hal_command_buffer_push_descriptor_set( command_buffer, pipeline_layout_, /*set=*/0, @@ -149,23 +168,26 @@ TEST_F(command_buffer_dispatch_test, DispatchAbs) { IREE_ASSERT_OK(iree_hal_command_buffer_end(command_buffer)); - IREE_ASSERT_OK(SubmitCommandBufferAndWait(command_buffer)); + IREE_ASSERT_OK(SubmitCommandBufferAndWait(command_buffer, binding_table)); - float output_value = 0.0f; + float output_values[4] = {0.0f}; IREE_ASSERT_OK(iree_hal_device_transfer_d2h( device_, output_buffer, - /*source_offset=*/0, &output_value, sizeof(output_value), + /*source_offset=*/0, output_values, sizeof(output_values), IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, iree_infinite_timeout())); - EXPECT_EQ(2.5f, output_value); + EXPECT_THAT(output_values, ::testing::ElementsAre(-9.0f, 2.5f, 2.5f, -9.0f)); iree_hal_command_buffer_release(command_buffer); iree_hal_buffer_release(output_buffer); - iree_hal_buffer_view_release(input_buffer_view); + iree_hal_buffer_release(input_buffer); CleanupExecutable(); } -} // namespace cts -} // namespace hal -} // namespace iree +INSTANTIATE_TEST_SUITE_P(CommandBufferTest, CommandBufferDispatchTest, + ::testing::Values(RecordingType::kDirect, + RecordingType::kIndirect), + GenerateTestName()); + +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_COMMAND_BUFFER_DISPATCH_TEST_H_ diff --git a/runtime/src/iree/hal/cts/command_buffer_fill_buffer_test.h b/runtime/src/iree/hal/cts/command_buffer_fill_buffer_test.h new file mode 100644 index 000000000000..d5dc0a78b087 --- /dev/null +++ b/runtime/src/iree/hal/cts/command_buffer_fill_buffer_test.h @@ -0,0 +1,331 @@ +// Copyright 2019 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef IREE_HAL_CTS_COMMAND_BUFFER_FILL_BUFFER_TEST_H_ +#define IREE_HAL_CTS_COMMAND_BUFFER_FILL_BUFFER_TEST_H_ + +#include +#include + +#include "iree/base/api.h" +#include "iree/hal/api.h" +#include "iree/hal/cts/cts_test_base.h" +#include "iree/testing/gtest.h" +#include "iree/testing/status_matchers.h" + +namespace iree::hal::cts { + +using ::testing::ContainerEq; + +class CommandBufferFillBufferTest + : public CTSTestBase<::testing::TestWithParam> { + protected: + std::vector RunFillBufferTest(RecordingType recording_type, + iree_device_size_t buffer_size, + iree_device_size_t target_offset, + iree_device_size_t fill_length, + const void* pattern, + iree_host_size_t pattern_length) { + switch (recording_type) { + default: + case RecordingType::kDirect: + return RunFillBufferDirectTest(buffer_size, target_offset, fill_length, + pattern, pattern_length); + case RecordingType::kIndirect: + return RunFillBufferIndirectTest(buffer_size, target_offset, + fill_length, pattern, pattern_length); + } + } + + std::vector RunFillBufferDirectTest( + iree_device_size_t buffer_size, iree_device_size_t target_offset, + iree_device_size_t fill_length, const void* pattern, + iree_host_size_t pattern_length) { + iree_hal_buffer_t* device_buffer = NULL; + CreateZeroedDeviceBuffer(buffer_size, &device_buffer); + + iree_hal_command_buffer_t* command_buffer = NULL; + IREE_CHECK_OK(iree_hal_command_buffer_create( + device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, + IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, + /*binding_capacity=*/0, &command_buffer)); + IREE_CHECK_OK(iree_hal_command_buffer_begin(command_buffer)); + + // Fill the pattern. + IREE_CHECK_OK(iree_hal_command_buffer_fill_buffer( + command_buffer, + iree_hal_make_buffer_ref(device_buffer, target_offset, fill_length), + pattern, pattern_length)); + IREE_CHECK_OK(iree_hal_command_buffer_end(command_buffer)); + IREE_CHECK_OK(SubmitCommandBufferAndWait(command_buffer)); + + // Read data for returning. + std::vector actual_data(buffer_size); + IREE_CHECK_OK(iree_hal_device_transfer_d2h( + device_, device_buffer, /*source_offset=*/0, + /*target_buffer=*/actual_data.data(), + /*data_length=*/buffer_size, IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, + iree_infinite_timeout())); + + // Cleanup and return. + iree_hal_command_buffer_release(command_buffer); + iree_hal_buffer_release(device_buffer); + return actual_data; + } + + std::vector RunFillBufferIndirectTest( + iree_device_size_t buffer_size, iree_device_size_t target_offset, + iree_device_size_t fill_length, const void* pattern, + iree_host_size_t pattern_length) { + iree_hal_buffer_t* device_buffer = NULL; + CreateZeroedDeviceBuffer(buffer_size, &device_buffer); + + iree_hal_command_buffer_t* command_buffer = NULL; + IREE_CHECK_OK(iree_hal_command_buffer_create( + device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, + IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, + /*binding_capacity=*/0, &command_buffer)); + IREE_CHECK_OK(iree_hal_command_buffer_begin(command_buffer)); + + // Fill the pattern. + IREE_CHECK_OK(iree_hal_command_buffer_fill_buffer( + command_buffer, + iree_hal_make_buffer_ref(device_buffer, target_offset, fill_length), + pattern, pattern_length)); + IREE_CHECK_OK(iree_hal_command_buffer_end(command_buffer)); + IREE_CHECK_OK(SubmitCommandBufferAndWait(command_buffer)); + + // Read data for returning. + std::vector actual_data(buffer_size); + IREE_CHECK_OK(iree_hal_device_transfer_d2h( + device_, device_buffer, /*source_offset=*/0, + /*target_buffer=*/actual_data.data(), + /*data_length=*/buffer_size, IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, + iree_infinite_timeout())); + + // Cleanup and return. + iree_hal_command_buffer_release(command_buffer); + iree_hal_buffer_release(device_buffer); + return actual_data; + } +}; + +TEST_P(CommandBufferFillBufferTest, FillBuffer_pattern1_size1_offset0_length1) { + iree_device_size_t buffer_size = 1; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 1; + uint8_t pattern = 0x07; + std::vector reference_buffer{0x07}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, FillBuffer_pattern1_size5_offset0_length5) { + iree_device_size_t buffer_size = 5; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 5; + uint8_t pattern = 0x07; + std::vector reference_buffer{0x07, 0x07, 0x07, 0x07, // + 0x07}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern1_size16_offset0_length1) { + iree_device_size_t buffer_size = 16; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 1; + uint8_t pattern = 0x07; + std::vector reference_buffer{0x07, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern1_size16_offset0_length3) { + iree_device_size_t buffer_size = 16; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 3; + uint8_t pattern = 0x07; + std::vector reference_buffer{0x07, 0x07, 0x07, 0x00, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern1_size16_offset0_length8) { + iree_device_size_t buffer_size = 16; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 8; + uint8_t pattern = 0x07; + std::vector reference_buffer{0x07, 0x07, 0x07, 0x07, // + 0x07, 0x07, 0x07, 0x07, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern1_size16_offset2_length8) { + iree_device_size_t buffer_size = 16; + iree_device_size_t target_offset = 2; + iree_device_size_t fill_length = 8; + uint8_t pattern = 0x07; + std::vector reference_buffer{0x00, 0x00, 0x07, 0x07, // + 0x07, 0x07, 0x07, 0x07, // + 0x07, 0x07, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, FillBuffer_pattern2_size2_offset0_length2) { + iree_device_size_t buffer_size = 2; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 2; + uint16_t pattern = 0xAB23; + std::vector reference_buffer{0x23, 0xAB}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern2_size16_offset0_length8) { + iree_device_size_t buffer_size = 16; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 8; + uint16_t pattern = 0xAB23; + std::vector reference_buffer{0x23, 0xAB, 0x23, 0xAB, // + 0x23, 0xAB, 0x23, 0xAB, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern2_size16_offset0_length10) { + iree_device_size_t buffer_size = 16; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 10; + uint16_t pattern = 0xAB23; + std::vector reference_buffer{0x23, 0xAB, 0x23, 0xAB, // + 0x23, 0xAB, 0x23, 0xAB, // + 0x23, 0xAB, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern2_size16_offset2_length8) { + iree_device_size_t buffer_size = 16; + iree_device_size_t target_offset = 2; + iree_device_size_t fill_length = 8; + uint16_t pattern = 0xAB23; + std::vector reference_buffer{0x00, 0x00, 0x23, 0xAB, // + 0x23, 0xAB, 0x23, 0xAB, // + 0x23, 0xAB, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, FillBuffer_pattern4_size4_offset0_length4) { + iree_device_size_t buffer_size = 4; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 4; + uint32_t pattern = 0xAB23CD45; + std::vector reference_buffer{0x45, 0xCD, 0x23, 0xAB}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern4_size4_offset16_length4) { + iree_device_size_t buffer_size = 20; + iree_device_size_t target_offset = 16; + iree_device_size_t fill_length = 4; + uint32_t pattern = 0xAB23CD45; + std::vector reference_buffer(buffer_size, 0); + *reinterpret_cast(&reference_buffer[target_offset]) = pattern; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern4_size16_offset0_length8) { + iree_device_size_t buffer_size = 16; + iree_device_size_t target_offset = 0; + iree_device_size_t fill_length = 8; + uint32_t pattern = 0xAB23CD45; + std::vector reference_buffer{0x45, 0xCD, 0x23, 0xAB, // + 0x45, 0xCD, 0x23, 0xAB, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +TEST_P(CommandBufferFillBufferTest, + FillBuffer_pattern4_size16_offset8_length8) { + iree_device_size_t buffer_size = 16; + iree_device_size_t target_offset = 8; + iree_device_size_t fill_length = 8; + uint32_t pattern = 0xAB23CD45; + std::vector reference_buffer{0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00, // + 0x45, 0xCD, 0x23, 0xAB, // + 0x45, 0xCD, 0x23, 0xAB}; + std::vector actual_buffer = + RunFillBufferTest(GetParam(), buffer_size, target_offset, fill_length, + (void*)&pattern, sizeof(pattern)); + EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); +} + +INSTANTIATE_TEST_SUITE_P(CommandBufferFillBufferTest, + CommandBufferFillBufferTest, + ::testing::Values(RecordingType::kDirect, + RecordingType::kIndirect), + GenerateTestName()); + +} // namespace iree::hal::cts + +#endif // IREE_HAL_CTS_COMMAND_BUFFER_FILL_BUFFER_TEST_H_ diff --git a/runtime/src/iree/hal/cts/command_buffer_push_constants_test.h b/runtime/src/iree/hal/cts/command_buffer_push_constants_test.h index aa719d2e25ea..af99ee1842db 100644 --- a/runtime/src/iree/hal/cts/command_buffer_push_constants_test.h +++ b/runtime/src/iree/hal/cts/command_buffer_push_constants_test.h @@ -14,13 +14,11 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { using ::testing::ContainerEq; -class command_buffer_push_constants_test : public CTSTestBase<> { +class CommandBufferPushConstantsTest : public CTSTestBase<> { protected: void PrepareExecutable() { IREE_ASSERT_OK(iree_hal_executable_cache_create( @@ -77,7 +75,7 @@ class command_buffer_push_constants_test : public CTSTestBase<> { iree_hal_executable_t* executable_ = NULL; }; -TEST_F(command_buffer_push_constants_test, DispatchWithPushConstants) { +TEST_F(CommandBufferPushConstantsTest, DispatchWithPushConstants) { ASSERT_NO_FATAL_FAILURE(PrepareExecutable()); iree_hal_command_buffer_t* command_buffer = NULL; @@ -152,8 +150,6 @@ TEST_F(command_buffer_push_constants_test, DispatchWithPushConstants) { CleanupExecutable(); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_COMMAND_BUFFER_PUSH_CONSTANTS_TEST_H_ diff --git a/runtime/src/iree/hal/cts/command_buffer_test.h b/runtime/src/iree/hal/cts/command_buffer_test.h index dc23c9b77f30..aa7819c6ae1e 100644 --- a/runtime/src/iree/hal/cts/command_buffer_test.h +++ b/runtime/src/iree/hal/cts/command_buffer_test.h @@ -16,74 +16,13 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { using ::testing::ContainerEq; -namespace { -constexpr iree_device_size_t kDefaultAllocationSize = 1024; -} // namespace +class CommandBufferTest : public CTSTestBase<> {}; -class command_buffer_test : public CTSTestBase<> { - protected: - void CreateZeroedDeviceBuffer(iree_device_size_t buffer_size, - iree_hal_buffer_t** out_buffer) { - iree_hal_buffer_params_t params = {0}; - params.type = - IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL | IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; - params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | - IREE_HAL_BUFFER_USAGE_TRANSFER | - IREE_HAL_BUFFER_USAGE_MAPPING; - iree_hal_buffer_t* device_buffer = NULL; - IREE_CHECK_OK(iree_hal_allocator_allocate_buffer( - iree_hal_device_allocator(device_), params, buffer_size, - &device_buffer)); - IREE_ASSERT_OK( - iree_hal_buffer_map_zero(device_buffer, 0, IREE_WHOLE_BUFFER)); - *out_buffer = device_buffer; - } - - std::vector RunFillBufferTest(iree_device_size_t buffer_size, - iree_device_size_t target_offset, - iree_device_size_t fill_length, - const void* pattern, - iree_host_size_t pattern_length) { - iree_hal_buffer_t* device_buffer = NULL; - CreateZeroedDeviceBuffer(buffer_size, &device_buffer); - - iree_hal_command_buffer_t* command_buffer = NULL; - IREE_CHECK_OK(iree_hal_command_buffer_create( - device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, - IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, - /*binding_capacity=*/0, &command_buffer)); - IREE_CHECK_OK(iree_hal_command_buffer_begin(command_buffer)); - - // Fill the pattern. - IREE_CHECK_OK(iree_hal_command_buffer_fill_buffer( - command_buffer, - iree_hal_make_buffer_ref(device_buffer, target_offset, fill_length), - pattern, pattern_length)); - IREE_CHECK_OK(iree_hal_command_buffer_end(command_buffer)); - IREE_CHECK_OK(SubmitCommandBufferAndWait(command_buffer)); - - // Read data for returning. - std::vector actual_data(buffer_size); - IREE_CHECK_OK(iree_hal_device_transfer_d2h( - device_, device_buffer, /*source_offset=*/0, - /*target_buffer=*/actual_data.data(), - /*data_length=*/buffer_size, IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, - iree_infinite_timeout())); - - // Cleanup and return. - iree_hal_command_buffer_release(command_buffer); - iree_hal_buffer_release(device_buffer); - return actual_data; - } -}; - -TEST_F(command_buffer_test, Create) { +TEST_F(CommandBufferTest, Create) { iree_hal_command_buffer_t* command_buffer = NULL; IREE_ASSERT_OK(iree_hal_command_buffer_create( device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, @@ -97,7 +36,7 @@ TEST_F(command_buffer_test, Create) { iree_hal_command_buffer_release(command_buffer); } -TEST_F(command_buffer_test, BeginEnd) { +TEST_F(CommandBufferTest, BeginEnd) { iree_hal_command_buffer_t* command_buffer = NULL; IREE_ASSERT_OK(iree_hal_command_buffer_create( device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, @@ -110,7 +49,7 @@ TEST_F(command_buffer_test, BeginEnd) { iree_hal_command_buffer_release(command_buffer); } -TEST_F(command_buffer_test, SubmitEmpty) { +TEST_F(CommandBufferTest, SubmitEmpty) { iree_hal_command_buffer_t* command_buffer = NULL; IREE_ASSERT_OK(iree_hal_command_buffer_create( device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, @@ -125,489 +64,6 @@ TEST_F(command_buffer_test, SubmitEmpty) { iree_hal_command_buffer_release(command_buffer); } -TEST_F(command_buffer_test, CopyWholeBuffer) { - iree_hal_command_buffer_t* command_buffer = NULL; - IREE_ASSERT_OK(iree_hal_command_buffer_create( - device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, - IREE_HAL_COMMAND_CATEGORY_TRANSFER, IREE_HAL_QUEUE_AFFINITY_ANY, - /*binding_capacity=*/0, &command_buffer)); - - uint8_t i8_val = 0x54; - std::vector reference_buffer(kDefaultAllocationSize); - std::memset(reference_buffer.data(), i8_val, kDefaultAllocationSize); - - // Create and fill a host buffer. - iree_hal_buffer_params_t host_params = {0}; - host_params.type = - IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE; - host_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | - IREE_HAL_BUFFER_USAGE_TRANSFER | - IREE_HAL_BUFFER_USAGE_MAPPING; - iree_hal_buffer_t* host_buffer = nullptr; - IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( - device_allocator_, host_params, kDefaultAllocationSize, &host_buffer)); - IREE_ASSERT_OK(iree_hal_device_transfer_h2d( - device_, reference_buffer.data(), host_buffer, 0, reference_buffer.size(), - IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, iree_infinite_timeout())); - - // Create a device buffer. - iree_hal_buffer_params_t device_params = {0}; - device_params.type = - IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL | IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; - device_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | - IREE_HAL_BUFFER_USAGE_TRANSFER | - IREE_HAL_BUFFER_USAGE_MAPPING; - iree_hal_buffer_t* device_buffer = nullptr; - IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( - device_allocator_, device_params, kDefaultAllocationSize, - &device_buffer)); - - // Copy the host buffer to the device buffer. - IREE_ASSERT_OK(iree_hal_command_buffer_begin(command_buffer)); - IREE_ASSERT_OK(iree_hal_command_buffer_copy_buffer( - command_buffer, /*source_ref=*/ - iree_hal_make_buffer_ref(host_buffer, 0, kDefaultAllocationSize), - /*target_ref=*/ - iree_hal_make_buffer_ref(device_buffer, 0, kDefaultAllocationSize))); - IREE_ASSERT_OK(iree_hal_command_buffer_end(command_buffer)); - - IREE_ASSERT_OK(SubmitCommandBufferAndWait(command_buffer)); - - // Read the device buffer and compare. - std::vector actual_data(kDefaultAllocationSize); - IREE_ASSERT_OK(iree_hal_device_transfer_d2h( - device_, device_buffer, /*source_offset=*/0, - /*target_buffer=*/actual_data.data(), - /*data_length=*/kDefaultAllocationSize, - IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, iree_infinite_timeout())); - EXPECT_THAT(actual_data, ContainerEq(reference_buffer)); - - // Must release the command buffer before resources used by it. - iree_hal_command_buffer_release(command_buffer); - iree_hal_buffer_release(device_buffer); - iree_hal_buffer_release(host_buffer); -} - -TEST_F(command_buffer_test, CopySubBuffer) { - iree_hal_command_buffer_t* command_buffer = NULL; - IREE_ASSERT_OK(iree_hal_command_buffer_create( - device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, - IREE_HAL_COMMAND_CATEGORY_TRANSFER, IREE_HAL_QUEUE_AFFINITY_ANY, - /*binding_capacity=*/0, &command_buffer)); - - iree_hal_buffer_params_t device_params = {0}; - device_params.type = - IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL | IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; - device_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | - IREE_HAL_BUFFER_USAGE_TRANSFER | - IREE_HAL_BUFFER_USAGE_MAPPING; - iree_hal_buffer_t* device_buffer = NULL; - IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( - device_allocator_, device_params, kDefaultAllocationSize, - &device_buffer)); - - uint8_t i8_val = 0x88; - std::vector reference_buffer(kDefaultAllocationSize); - std::memset(reference_buffer.data() + 8, i8_val, - kDefaultAllocationSize / 2 - 4); - - // Create another host buffer with a smaller size. - iree_hal_buffer_params_t host_params = {0}; - host_params.type = - IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE; - host_params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | - IREE_HAL_BUFFER_USAGE_TRANSFER | - IREE_HAL_BUFFER_USAGE_MAPPING; - std::vector host_buffer_data(kDefaultAllocationSize, i8_val); - iree_hal_buffer_t* host_buffer = NULL; - IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( - device_allocator_, host_params, host_buffer_data.size() / 2, - &host_buffer)); - IREE_ASSERT_OK(iree_hal_device_transfer_h2d( - device_, host_buffer_data.data(), host_buffer, 0, - host_buffer_data.size() / 2, IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, - iree_infinite_timeout())); - - // Copy the host buffer to the device buffer; zero fill the untouched bytes. - uint8_t zero_val = 0x0; - IREE_ASSERT_OK(iree_hal_command_buffer_begin(command_buffer)); - IREE_ASSERT_OK(iree_hal_command_buffer_fill_buffer( - command_buffer, - iree_hal_make_buffer_ref(device_buffer, /*target_offset=*/0, - /*length=*/8), - &zero_val, /*pattern_length=*/sizeof(zero_val))); - IREE_ASSERT_OK(iree_hal_command_buffer_copy_buffer( - command_buffer, - iree_hal_make_buffer_ref(/*source_buffer=*/host_buffer, - /*source_offset=*/4, - /*length=*/kDefaultAllocationSize / 2 - 4), - iree_hal_make_buffer_ref(/*target_buffer=*/device_buffer, - /*target_offset=*/8, - /*length=*/kDefaultAllocationSize / 2 - 4))); - IREE_ASSERT_OK(iree_hal_command_buffer_fill_buffer( - command_buffer, - iree_hal_make_buffer_ref( - device_buffer, - /*target_offset=*/8 + kDefaultAllocationSize / 2 - 4, - /*length=*/kDefaultAllocationSize - - (8 + kDefaultAllocationSize / 2 - 4)), - &zero_val, - /*pattern_length=*/sizeof(zero_val))); - IREE_ASSERT_OK(iree_hal_command_buffer_end(command_buffer)); - - IREE_ASSERT_OK(SubmitCommandBufferAndWait(command_buffer)); - - // Read the device buffer and compare. - std::vector actual_data(kDefaultAllocationSize); - IREE_ASSERT_OK(iree_hal_device_transfer_d2h( - device_, device_buffer, /*source_offset=*/0, - /*target_buffer=*/actual_data.data(), - /*data_length=*/kDefaultAllocationSize, - IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, iree_infinite_timeout())); - EXPECT_THAT(actual_data, ContainerEq(reference_buffer)); - - // Must release the command buffer before resources used by it. - iree_hal_command_buffer_release(command_buffer); - iree_hal_buffer_release(device_buffer); - iree_hal_buffer_release(host_buffer); -} - -TEST_F(command_buffer_test, FillBuffer_pattern1_size1_offset0_length1) { - iree_device_size_t buffer_size = 1; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 1; - uint8_t pattern = 0x07; - std::vector reference_buffer{0x07}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern1_size5_offset0_length5) { - iree_device_size_t buffer_size = 5; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 5; - uint8_t pattern = 0x07; - std::vector reference_buffer{0x07, 0x07, 0x07, 0x07, // - 0x07}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern1_size16_offset0_length1) { - iree_device_size_t buffer_size = 16; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 1; - uint8_t pattern = 0x07; - std::vector reference_buffer{0x07, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern1_size16_offset0_length3) { - iree_device_size_t buffer_size = 16; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 3; - uint8_t pattern = 0x07; - std::vector reference_buffer{0x07, 0x07, 0x07, 0x00, // - 0x00, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern1_size16_offset0_length8) { - iree_device_size_t buffer_size = 16; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 8; - uint8_t pattern = 0x07; - std::vector reference_buffer{0x07, 0x07, 0x07, 0x07, // - 0x07, 0x07, 0x07, 0x07, // - 0x00, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern1_size16_offset2_length8) { - iree_device_size_t buffer_size = 16; - iree_device_size_t target_offset = 2; - iree_device_size_t fill_length = 8; - uint8_t pattern = 0x07; - std::vector reference_buffer{0x00, 0x00, 0x07, 0x07, // - 0x07, 0x07, 0x07, 0x07, // - 0x07, 0x07, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern2_size2_offset0_length2) { - iree_device_size_t buffer_size = 2; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 2; - uint16_t pattern = 0xAB23; - std::vector reference_buffer{0x23, 0xAB}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern2_size16_offset0_length8) { - iree_device_size_t buffer_size = 16; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 8; - uint16_t pattern = 0xAB23; - std::vector reference_buffer{0x23, 0xAB, 0x23, 0xAB, // - 0x23, 0xAB, 0x23, 0xAB, // - 0x00, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern2_size16_offset0_length10) { - iree_device_size_t buffer_size = 16; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 10; - uint16_t pattern = 0xAB23; - std::vector reference_buffer{0x23, 0xAB, 0x23, 0xAB, // - 0x23, 0xAB, 0x23, 0xAB, // - 0x23, 0xAB, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern2_size16_offset2_length8) { - iree_device_size_t buffer_size = 16; - iree_device_size_t target_offset = 2; - iree_device_size_t fill_length = 8; - uint16_t pattern = 0xAB23; - std::vector reference_buffer{0x00, 0x00, 0x23, 0xAB, // - 0x23, 0xAB, 0x23, 0xAB, // - 0x23, 0xAB, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern4_size4_offset0_length4) { - iree_device_size_t buffer_size = 4; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 4; - uint32_t pattern = 0xAB23CD45; - std::vector reference_buffer{0x45, 0xCD, 0x23, 0xAB}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern4_size4_offset16_length4) { - iree_device_size_t buffer_size = 20; - iree_device_size_t target_offset = 16; - iree_device_size_t fill_length = 4; - uint32_t pattern = 0xAB23CD45; - std::vector reference_buffer(buffer_size, 0); - *reinterpret_cast(&reference_buffer[target_offset]) = pattern; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern4_size16_offset0_length8) { - iree_device_size_t buffer_size = 16; - iree_device_size_t target_offset = 0; - iree_device_size_t fill_length = 8; - uint32_t pattern = 0xAB23CD45; - std::vector reference_buffer{0x45, 0xCD, 0x23, 0xAB, // - 0x45, 0xCD, 0x23, 0xAB, // - 0x00, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, FillBuffer_pattern4_size16_offset8_length8) { - iree_device_size_t buffer_size = 16; - iree_device_size_t target_offset = 8; - iree_device_size_t fill_length = 8; - uint32_t pattern = 0xAB23CD45; - std::vector reference_buffer{0x00, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00, // - 0x45, 0xCD, 0x23, 0xAB, // - 0x45, 0xCD, 0x23, 0xAB}; - std::vector actual_buffer = - RunFillBufferTest(buffer_size, target_offset, fill_length, - (void*)&pattern, sizeof(pattern)); - EXPECT_THAT(actual_buffer, ContainerEq(reference_buffer)); -} - -TEST_F(command_buffer_test, UpdateBufferWholeBuffer) { - iree_device_size_t target_buffer_size = 16; - std::vector source_buffer{0x01, 0x02, 0x03, 0x04, // - 0x05, 0x06, 0x07, 0x08, // - 0xA1, 0xA2, 0xA3, 0xA4, // - 0xA5, 0xA6, 0xA7, 0xA8}; - - iree_hal_buffer_t* device_buffer = NULL; - CreateZeroedDeviceBuffer(target_buffer_size, &device_buffer); - - iree_hal_command_buffer_t* command_buffer = NULL; - IREE_CHECK_OK(iree_hal_command_buffer_create( - device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, - IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, - /*binding_capacity=*/0, &command_buffer)); - IREE_CHECK_OK(iree_hal_command_buffer_begin(command_buffer)); - - // Issue the update_buffer command. - IREE_CHECK_OK(iree_hal_command_buffer_update_buffer( - command_buffer, - /*source_buffer=*/source_buffer.data(), /*source_offset=*/0, - iree_hal_make_buffer_ref(device_buffer, 0, target_buffer_size))); - IREE_CHECK_OK(iree_hal_command_buffer_end(command_buffer)); - IREE_CHECK_OK(SubmitCommandBufferAndWait(command_buffer)); - - // Check that the contents match what we expect. - std::vector actual_data(target_buffer_size); - IREE_CHECK_OK(iree_hal_device_transfer_d2h( - device_, device_buffer, /*source_offset=*/0, actual_data.data(), - actual_data.size(), IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, - iree_infinite_timeout())); - EXPECT_THAT(actual_data, ContainerEq(source_buffer)); - - iree_hal_command_buffer_release(command_buffer); - iree_hal_buffer_release(device_buffer); -} - -TEST_F(command_buffer_test, UpdateBufferWithOffsets) { - iree_device_size_t target_buffer_size = 16; - std::vector source_buffer{0x01, 0x02, 0x03, 0x04, // - 0x05, 0x06, 0x07, 0x08, // - 0xA1, 0xA2, 0xA3, 0xA4, // - 0xA5, 0xA6, 0xA7, 0xA8}; - - iree_hal_buffer_t* device_buffer = NULL; - CreateZeroedDeviceBuffer(target_buffer_size, &device_buffer); - - iree_hal_command_buffer_t* command_buffer = NULL; - IREE_CHECK_OK(iree_hal_command_buffer_create( - device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, - IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, - /*binding_capacity=*/0, &command_buffer)); - IREE_CHECK_OK(iree_hal_command_buffer_begin(command_buffer)); - - // Issue the update_buffer command. - IREE_CHECK_OK(iree_hal_command_buffer_update_buffer( - command_buffer, - /*source_buffer=*/source_buffer.data(), /*source_offset=*/4, - iree_hal_make_buffer_ref(device_buffer, - /*target_offset=*/4, /*length=*/8))); - IREE_CHECK_OK(iree_hal_command_buffer_end(command_buffer)); - IREE_CHECK_OK(SubmitCommandBufferAndWait(command_buffer)); - - // Check that the contents match what we expect. - std::vector actual_data(target_buffer_size); - IREE_CHECK_OK(iree_hal_device_transfer_d2h( - device_, device_buffer, /*source_offset=*/0, actual_data.data(), - actual_data.size(), IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, - iree_infinite_timeout())); - std::vector reference_buffer{0x00, 0x00, 0x00, 0x00, // - 0x05, 0x06, 0x07, 0x08, // - 0xA1, 0xA2, 0xA3, 0xA4, // - 0x00, 0x00, 0x00, 0x00}; - EXPECT_THAT(actual_data, ContainerEq(reference_buffer)); - - iree_hal_command_buffer_release(command_buffer); - iree_hal_buffer_release(device_buffer); -} - -TEST_F(command_buffer_test, UpdateBufferSubspan) { - iree_device_size_t target_buffer_size = 16; - std::vector source_buffer{0x01, 0x02, 0x03, 0x04, // - 0x05, 0x06, 0x07, 0x08, // - 0xA1, 0xA2, 0xA3, 0xA4, // - 0xA5, 0xA6, 0xA7, 0xA8}; - - iree_hal_buffer_t* device_buffer = NULL; - CreateZeroedDeviceBuffer(target_buffer_size, &device_buffer); - - // Create a subspan. - iree_device_size_t subspan_length = 8; - iree_hal_buffer_t* buffer_subspan; - IREE_ASSERT_OK(iree_hal_buffer_subspan(device_buffer, /*byte_offset=*/4, - subspan_length, &buffer_subspan)); - - iree_hal_command_buffer_t* command_buffer = NULL; - IREE_CHECK_OK(iree_hal_command_buffer_create( - device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, - IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, - /*binding_capacity=*/0, &command_buffer)); - IREE_CHECK_OK(iree_hal_command_buffer_begin(command_buffer)); - - // Issue the update_buffer command. - IREE_CHECK_OK(iree_hal_command_buffer_update_buffer( - command_buffer, - /*source_buffer=*/source_buffer.data(), /*source_offset=*/4, - iree_hal_make_buffer_ref(buffer_subspan, - /*target_offset=*/4, /*length=*/4))); - IREE_CHECK_OK(iree_hal_command_buffer_end(command_buffer)); - IREE_CHECK_OK(SubmitCommandBufferAndWait(command_buffer)); - - // Check that the contents match what we expect. - std::vector actual_data(target_buffer_size); - IREE_ASSERT_OK(iree_hal_device_transfer_d2h( - device_, device_buffer, /*source_offset=*/0, actual_data.data(), - actual_data.size(), IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, - iree_infinite_timeout())); - std::vector reference_buffer{0x00, 0x00, 0x00, 0x00, // - 0x00, 0x00, 0x00, 0x00, // - 0x05, 0x06, 0x07, 0x08, // - 0x00, 0x00, 0x00, 0x00}; - EXPECT_THAT(actual_data, ContainerEq(reference_buffer)); - // Also check the subspan. - std::vector actual_data_subspan(subspan_length); - IREE_ASSERT_OK(iree_hal_device_transfer_d2h( - device_, buffer_subspan, /*source_offset=*/0, actual_data_subspan.data(), - actual_data_subspan.size(), IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, - iree_infinite_timeout())); - std::vector reference_buffer_subspan{0x00, 0x00, 0x00, 0x00, // - 0x05, 0x06, 0x07, 0x08}; - EXPECT_THAT(actual_data_subspan, ContainerEq(reference_buffer_subspan)); - - iree_hal_command_buffer_release(command_buffer); - iree_hal_buffer_release(buffer_subspan); - iree_hal_buffer_release(device_buffer); -} - -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_COMMAND_BUFFER_TEST_H_ diff --git a/runtime/src/iree/hal/cts/command_buffer_update_buffer_test.h b/runtime/src/iree/hal/cts/command_buffer_update_buffer_test.h new file mode 100644 index 000000000000..55f9b647a5e0 --- /dev/null +++ b/runtime/src/iree/hal/cts/command_buffer_update_buffer_test.h @@ -0,0 +1,164 @@ +// Copyright 2019 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef IREE_HAL_CTS_COMMAND_BUFFER_UPDATE_BUFFER_TEST_H_ +#define IREE_HAL_CTS_COMMAND_BUFFER_UPDATE_BUFFER_TEST_H_ + +#include +#include + +#include "iree/base/api.h" +#include "iree/hal/api.h" +#include "iree/hal/cts/cts_test_base.h" +#include "iree/testing/gtest.h" +#include "iree/testing/status_matchers.h" + +namespace iree::hal::cts { + +using ::testing::ContainerEq; + +class CommandBufferUpdateBufferTest : public CTSTestBase<> {}; + +TEST_F(CommandBufferUpdateBufferTest, UpdateBufferWholeBuffer) { + iree_device_size_t target_buffer_size = 16; + std::vector source_buffer{0x01, 0x02, 0x03, 0x04, // + 0x05, 0x06, 0x07, 0x08, // + 0xA1, 0xA2, 0xA3, 0xA4, // + 0xA5, 0xA6, 0xA7, 0xA8}; + + iree_hal_buffer_t* device_buffer = NULL; + CreateZeroedDeviceBuffer(target_buffer_size, &device_buffer); + + iree_hal_command_buffer_t* command_buffer = NULL; + IREE_CHECK_OK(iree_hal_command_buffer_create( + device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, + IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, + /*binding_capacity=*/0, &command_buffer)); + IREE_CHECK_OK(iree_hal_command_buffer_begin(command_buffer)); + + // Issue the update_buffer command. + IREE_CHECK_OK(iree_hal_command_buffer_update_buffer( + command_buffer, + /*source_buffer=*/source_buffer.data(), /*source_offset=*/0, + iree_hal_make_buffer_ref(device_buffer, 0, target_buffer_size))); + IREE_CHECK_OK(iree_hal_command_buffer_end(command_buffer)); + IREE_CHECK_OK(SubmitCommandBufferAndWait(command_buffer)); + + // Check that the contents match what we expect. + std::vector actual_data(target_buffer_size); + IREE_CHECK_OK(iree_hal_device_transfer_d2h( + device_, device_buffer, /*source_offset=*/0, actual_data.data(), + actual_data.size(), IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, + iree_infinite_timeout())); + EXPECT_THAT(actual_data, ContainerEq(source_buffer)); + + iree_hal_command_buffer_release(command_buffer); + iree_hal_buffer_release(device_buffer); +} + +TEST_F(CommandBufferUpdateBufferTest, UpdateBufferWithOffsets) { + iree_device_size_t target_buffer_size = 16; + std::vector source_buffer{0x01, 0x02, 0x03, 0x04, // + 0x05, 0x06, 0x07, 0x08, // + 0xA1, 0xA2, 0xA3, 0xA4, // + 0xA5, 0xA6, 0xA7, 0xA8}; + + iree_hal_buffer_t* device_buffer = NULL; + CreateZeroedDeviceBuffer(target_buffer_size, &device_buffer); + + iree_hal_command_buffer_t* command_buffer = NULL; + IREE_CHECK_OK(iree_hal_command_buffer_create( + device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, + IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, + /*binding_capacity=*/0, &command_buffer)); + IREE_CHECK_OK(iree_hal_command_buffer_begin(command_buffer)); + + // Issue the update_buffer command. + IREE_CHECK_OK(iree_hal_command_buffer_update_buffer( + command_buffer, + /*source_buffer=*/source_buffer.data(), /*source_offset=*/4, + iree_hal_make_buffer_ref(device_buffer, + /*target_offset=*/4, /*length=*/8))); + IREE_CHECK_OK(iree_hal_command_buffer_end(command_buffer)); + IREE_CHECK_OK(SubmitCommandBufferAndWait(command_buffer)); + + // Check that the contents match what we expect. + std::vector actual_data(target_buffer_size); + IREE_CHECK_OK(iree_hal_device_transfer_d2h( + device_, device_buffer, /*source_offset=*/0, actual_data.data(), + actual_data.size(), IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, + iree_infinite_timeout())); + std::vector reference_buffer{0x00, 0x00, 0x00, 0x00, // + 0x05, 0x06, 0x07, 0x08, // + 0xA1, 0xA2, 0xA3, 0xA4, // + 0x00, 0x00, 0x00, 0x00}; + EXPECT_THAT(actual_data, ContainerEq(reference_buffer)); + + iree_hal_command_buffer_release(command_buffer); + iree_hal_buffer_release(device_buffer); +} + +TEST_F(CommandBufferUpdateBufferTest, UpdateBufferSubspan) { + iree_device_size_t target_buffer_size = 16; + std::vector source_buffer{0x01, 0x02, 0x03, 0x04, // + 0x05, 0x06, 0x07, 0x08, // + 0xA1, 0xA2, 0xA3, 0xA4, // + 0xA5, 0xA6, 0xA7, 0xA8}; + + iree_hal_buffer_t* device_buffer = NULL; + CreateZeroedDeviceBuffer(target_buffer_size, &device_buffer); + + // Create a subspan. + iree_device_size_t subspan_length = 8; + iree_hal_buffer_t* buffer_subspan; + IREE_ASSERT_OK(iree_hal_buffer_subspan(device_buffer, /*byte_offset=*/4, + subspan_length, &buffer_subspan)); + + iree_hal_command_buffer_t* command_buffer = NULL; + IREE_CHECK_OK(iree_hal_command_buffer_create( + device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, + IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, + /*binding_capacity=*/0, &command_buffer)); + IREE_CHECK_OK(iree_hal_command_buffer_begin(command_buffer)); + + // Issue the update_buffer command. + IREE_CHECK_OK(iree_hal_command_buffer_update_buffer( + command_buffer, + /*source_buffer=*/source_buffer.data(), /*source_offset=*/4, + iree_hal_make_buffer_ref(buffer_subspan, + /*target_offset=*/4, /*length=*/4))); + IREE_CHECK_OK(iree_hal_command_buffer_end(command_buffer)); + IREE_CHECK_OK(SubmitCommandBufferAndWait(command_buffer)); + + // Check that the contents match what we expect. + std::vector actual_data(target_buffer_size); + IREE_ASSERT_OK(iree_hal_device_transfer_d2h( + device_, device_buffer, /*source_offset=*/0, actual_data.data(), + actual_data.size(), IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, + iree_infinite_timeout())); + std::vector reference_buffer{0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00, // + 0x05, 0x06, 0x07, 0x08, // + 0x00, 0x00, 0x00, 0x00}; + EXPECT_THAT(actual_data, ContainerEq(reference_buffer)); + // Also check the subspan. + std::vector actual_data_subspan(subspan_length); + IREE_ASSERT_OK(iree_hal_device_transfer_d2h( + device_, buffer_subspan, /*source_offset=*/0, actual_data_subspan.data(), + actual_data_subspan.size(), IREE_HAL_TRANSFER_BUFFER_FLAG_DEFAULT, + iree_infinite_timeout())); + std::vector reference_buffer_subspan{0x00, 0x00, 0x00, 0x00, // + 0x05, 0x06, 0x07, 0x08}; + EXPECT_THAT(actual_data_subspan, ContainerEq(reference_buffer_subspan)); + + iree_hal_command_buffer_release(command_buffer); + iree_hal_buffer_release(buffer_subspan); + iree_hal_buffer_release(device_buffer); +} + +} // namespace iree::hal::cts + +#endif // IREE_HAL_CTS_COMMAND_BUFFER_UPDATE_BUFFER_TEST_H_ diff --git a/runtime/src/iree/hal/cts/cts_test_base.h b/runtime/src/iree/hal/cts/cts_test_base.h index bdef0285105e..919933e0d616 100644 --- a/runtime/src/iree/hal/cts/cts_test_base.h +++ b/runtime/src/iree/hal/cts/cts_test_base.h @@ -36,53 +36,103 @@ const char* get_test_executable_format(); // Leaf test binaries must implement this function. iree_const_byte_span_t get_test_executable_data(iree_string_view_t file_name); +enum class RecordingType { + kDirect = 0, + kIndirect, +}; + +struct GenerateTestName { + template + std::string operator()( + const ::testing::TestParamInfo& info) const { + switch (info.param) { + default: + return ""; + case RecordingType::kDirect: + return "direct"; + case RecordingType::kIndirect: + return "indirect"; + } + } +}; + +// Gets a HAL driver with the provided name, if available. +static iree_status_t TryGetDriver(const std::string& driver_name, + iree_hal_driver_t** out_driver) { + static std::set unavailable_driver_names; + + // If creation failed before, don't try again. + if (unavailable_driver_names.find(driver_name) != + unavailable_driver_names.end()) { + return iree_make_status(IREE_STATUS_UNAVAILABLE, "driver unavailable"); + } + + // No existing driver, attempt to create. + iree_hal_driver_t* driver = NULL; + iree_status_t status = iree_hal_driver_registry_try_create( + iree_hal_driver_registry_default(), + iree_make_string_view(driver_name.data(), driver_name.size()), + iree_allocator_system(), &driver); + if (iree_status_is_unavailable(status)) { + unavailable_driver_names.insert(driver_name); + } + if (iree_status_is_ok(status)) { + *out_driver = driver; + } + return status; +} + +// Statics available in CTSTestBase without template magic. +// Note that this header is intended to be included in a single .cc so we can +// define the static member storage here. +class CTSTestResources { + public: + static iree_hal_driver_t* driver_; + static iree_hal_device_t* device_; + static iree_hal_allocator_t* device_allocator_; +}; +/*static*/ iree_hal_driver_t* CTSTestResources::driver_ = NULL; +/*static*/ iree_hal_device_t* CTSTestResources::device_ = NULL; +/*static*/ iree_hal_allocator_t* CTSTestResources::device_allocator_ = NULL; + // Common setup for tests parameterized on driver names. template -class CTSTestBase : public BaseType { - protected: +class CTSTestBase : public BaseType, public CTSTestResources { + public: static void SetUpTestSuite() { iree_status_t status = register_test_driver(iree_hal_driver_registry_default()); if (iree_status_is_already_exists(status)) { - return; + status = iree_status_ignore(status); } IREE_CHECK_OK(status); - } - - virtual void SetUp() { - const std::string& driver_name = get_test_driver_name(); // Get driver with the given name and create its default device. // Skip drivers that are (gracefully) unavailable, fail if creation fails. iree_hal_driver_t* driver = NULL; - iree_status_t status = TryGetDriver(driver_name, &driver); + status = TryGetDriver(get_test_driver_name(), &driver); if (iree_status_is_unavailable(status)) { - iree_status_free(status); - GTEST_SKIP() << "Skipping test as '" << driver_name - << "' driver is unavailable"; + iree_status_ignore(status); return; } - IREE_ASSERT_OK(status); + IREE_CHECK_OK(status); driver_ = driver; iree_hal_device_t* device = NULL; status = iree_hal_driver_create_default_device( driver_, iree_allocator_system(), &device); if (iree_status_is_unavailable(status)) { - iree_status_free(status); - GTEST_SKIP() << "Skipping test as default device for '" << driver_name - << "' driver is unavailable"; + iree_status_ignore(status); return; } - IREE_ASSERT_OK(status); - iree_status_free(status); + IREE_CHECK_OK(status); device_ = device; device_allocator_ = iree_hal_device_allocator(device_); iree_hal_allocator_retain(device_allocator_); } - virtual void TearDown() { + static void TearDownTestSuite() { if (device_allocator_) { iree_hal_allocator_release(device_allocator_); device_allocator_ = NULL; @@ -97,18 +147,84 @@ class CTSTestBase : public BaseType { } } + virtual void SetUp() { + if (!driver_) { + GTEST_SKIP() << "Skipping test as '" << get_test_driver_name() + << "' driver is unavailable"; + return; + } + if (!device_) { + GTEST_SKIP() << "Skipping test as default device for '" + << get_test_driver_name() << "' driver is unavailable"; + return; + } + } + + virtual void TearDown() {} + + void CreateUninitializedDeviceBuffer(iree_device_size_t buffer_size, + iree_hal_buffer_t** out_buffer) { + iree_hal_buffer_params_t params = {0}; + params.type = + IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL | IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; + params.usage = + IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | IREE_HAL_BUFFER_USAGE_TRANSFER; + iree_hal_buffer_t* device_buffer = NULL; + IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( + iree_hal_device_allocator(device_), params, buffer_size, out_buffer)); + } + + void CreateZeroedDeviceBuffer(iree_device_size_t buffer_size, + iree_hal_buffer_t** out_buffer) { + iree_hal_buffer_params_t params = {0}; + params.type = + IREE_HAL_MEMORY_TYPE_DEVICE_LOCAL | IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; + params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | + IREE_HAL_BUFFER_USAGE_TRANSFER | + IREE_HAL_BUFFER_USAGE_MAPPING; + iree_hal_buffer_t* device_buffer = NULL; + IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( + iree_hal_device_allocator(device_), params, buffer_size, + &device_buffer)); + IREE_ASSERT_OK( + iree_hal_buffer_map_zero(device_buffer, 0, IREE_WHOLE_BUFFER)); + *out_buffer = device_buffer; + } + + template + void CreateFilledDeviceBuffer(iree_device_size_t buffer_size, + PatternType pattern, + iree_hal_buffer_t** out_buffer) { + iree_hal_buffer_params_t params = {0}; + params.type = IREE_HAL_MEMORY_TYPE_OPTIMAL_FOR_DEVICE | + IREE_HAL_MEMORY_TYPE_HOST_VISIBLE; + params.usage = IREE_HAL_BUFFER_USAGE_DISPATCH_STORAGE | + IREE_HAL_BUFFER_USAGE_TRANSFER | + IREE_HAL_BUFFER_USAGE_MAPPING; + iree_hal_buffer_t* device_buffer = NULL; + IREE_ASSERT_OK(iree_hal_allocator_allocate_buffer( + iree_hal_device_allocator(device_), params, buffer_size, + &device_buffer)); + IREE_ASSERT_OK(iree_hal_buffer_map_fill(device_buffer, 0, IREE_WHOLE_BUFFER, + &pattern, sizeof(pattern))); + *out_buffer = device_buffer; + } + // Submits |command_buffer| to the device and waits for it to complete before // returning. iree_status_t SubmitCommandBufferAndWait( - iree_hal_command_buffer_t* command_buffer) { - return SubmitCommandBuffersAndWait(1, &command_buffer); + iree_hal_command_buffer_t* command_buffer, + iree_hal_buffer_binding_table_t binding_table = + iree_hal_buffer_binding_table_empty()) { + return SubmitCommandBuffersAndWait(1, &command_buffer, &binding_table); } // Submits |command_buffers| to the device and waits for them to complete // before returning. iree_status_t SubmitCommandBuffersAndWait( iree_host_size_t command_buffer_count, - iree_hal_command_buffer_t** command_buffers) { + iree_hal_command_buffer_t** command_buffers, + const iree_hal_buffer_binding_table_t* binding_tables = nullptr) { // No wait semaphores. iree_hal_semaphore_list_t wait_semaphores = iree_hal_semaphore_list_empty(); @@ -126,7 +242,7 @@ class CTSTestBase : public BaseType { iree_status_t status = iree_hal_device_queue_execute( device_, IREE_HAL_QUEUE_AFFINITY_ANY, wait_semaphores, signal_semaphores, command_buffer_count, command_buffers, - /*binding_tables=*/NULL); + binding_tables); if (iree_status_is_ok(status)) { status = iree_hal_semaphore_wait(signal_semaphore, target_payload_value, iree_infinite_timeout()); @@ -136,12 +252,13 @@ class CTSTestBase : public BaseType { return status; } - iree_hal_command_buffer_t* CreateEmptyCommandBuffer() { + iree_hal_command_buffer_t* CreateEmptyCommandBuffer( + iree_host_size_t binding_capacity = 0) { iree_hal_command_buffer_t* command_buffer = NULL; IREE_EXPECT_OK(iree_hal_command_buffer_create( device_, IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT, IREE_HAL_COMMAND_CATEGORY_DISPATCH, IREE_HAL_QUEUE_AFFINITY_ANY, - /*binding_capacity=*/0, &command_buffer)); + binding_capacity, &command_buffer)); IREE_EXPECT_OK(iree_hal_command_buffer_begin(command_buffer)); IREE_EXPECT_OK(iree_hal_command_buffer_end(command_buffer)); return command_buffer; @@ -159,37 +276,6 @@ class CTSTestBase : public BaseType { IREE_EXPECT_OK(iree_hal_semaphore_query(semaphore, &value)); EXPECT_EQ(expected_value, value); } - - iree_hal_driver_t* driver_ = NULL; - iree_hal_device_t* device_ = NULL; - iree_hal_allocator_t* device_allocator_ = NULL; - - private: - // Gets a HAL driver with the provided name, if available. - static iree_status_t TryGetDriver(const std::string& driver_name, - iree_hal_driver_t** out_driver) { - static std::set unavailable_driver_names; - - // If creation failed before, don't try again. - if (unavailable_driver_names.find(driver_name) != - unavailable_driver_names.end()) { - return iree_make_status(IREE_STATUS_UNAVAILABLE, "driver unavailable"); - } - - // No existing driver, attempt to create. - iree_hal_driver_t* driver = NULL; - iree_status_t status = iree_hal_driver_registry_try_create( - iree_hal_driver_registry_default(), - iree_make_string_view(driver_name.data(), driver_name.size()), - iree_allocator_system(), &driver); - if (iree_status_is_unavailable(status)) { - unavailable_driver_names.insert(driver_name); - } - if (iree_status_is_ok(status)) { - *out_driver = driver; - } - return status; - } }; } // namespace cts diff --git a/runtime/src/iree/hal/cts/descriptor_set_layout_test.h b/runtime/src/iree/hal/cts/descriptor_set_layout_test.h index 401220a12f08..ffeb03c1861f 100644 --- a/runtime/src/iree/hal/cts/descriptor_set_layout_test.h +++ b/runtime/src/iree/hal/cts/descriptor_set_layout_test.h @@ -13,15 +13,13 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { -class descriptor_set_layout_test : public CTSTestBase<> {}; +class DescriptorSetLayoutTest : public CTSTestBase<> {}; // Note: bindingCount == 0 is valid in VkDescriptorSetLayoutCreateInfo: // https://www.khronos.org/registry/vulkan/specs/1.2-extensions/man/html/VkDescriptorSetLayoutCreateInfo.html -TEST_F(descriptor_set_layout_test, CreateWithNoBindings) { +TEST_F(DescriptorSetLayoutTest, CreateWithNoBindings) { iree_hal_descriptor_set_layout_t* descriptor_set_layout = NULL; IREE_ASSERT_OK(iree_hal_descriptor_set_layout_create( device_, IREE_HAL_DESCRIPTOR_SET_LAYOUT_FLAG_NONE, @@ -30,7 +28,7 @@ TEST_F(descriptor_set_layout_test, CreateWithNoBindings) { iree_hal_descriptor_set_layout_release(descriptor_set_layout); } -TEST_F(descriptor_set_layout_test, CreateWithOneBinding) { +TEST_F(DescriptorSetLayoutTest, CreateWithOneBinding) { iree_hal_descriptor_set_layout_t* descriptor_set_layout = NULL; iree_hal_descriptor_set_layout_binding_t descriptor_set_layout_bindings[] = { { @@ -46,7 +44,7 @@ TEST_F(descriptor_set_layout_test, CreateWithOneBinding) { iree_hal_descriptor_set_layout_release(descriptor_set_layout); } -TEST_F(descriptor_set_layout_test, CreateWithTwoBindings) { +TEST_F(DescriptorSetLayoutTest, CreateWithTwoBindings) { iree_hal_descriptor_set_layout_t* descriptor_set_layout = NULL; iree_hal_descriptor_set_layout_binding_t descriptor_set_layout_bindings[] = { { @@ -67,7 +65,7 @@ TEST_F(descriptor_set_layout_test, CreateWithTwoBindings) { iree_hal_descriptor_set_layout_release(descriptor_set_layout); } -TEST_F(descriptor_set_layout_test, CreateWithPushDescriptorType) { +TEST_F(DescriptorSetLayoutTest, CreateWithPushDescriptorType) { iree_hal_descriptor_set_layout_t* descriptor_set_layout = NULL; iree_hal_descriptor_set_layout_binding_t descriptor_set_layout_bindings[] = { { @@ -88,8 +86,6 @@ TEST_F(descriptor_set_layout_test, CreateWithPushDescriptorType) { iree_hal_descriptor_set_layout_release(descriptor_set_layout); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_DESCRIPTOR_SET_LAYOUT_TEST_H_ diff --git a/runtime/src/iree/hal/cts/driver_test.h b/runtime/src/iree/hal/cts/driver_test.h index 092dd75a7df8..515244fbb6c7 100644 --- a/runtime/src/iree/hal/cts/driver_test.h +++ b/runtime/src/iree/hal/cts/driver_test.h @@ -16,19 +16,44 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { -class driver_test : public CTSTestBase<> { +// NOTE: does not use CTSTestBase as we don't want the automatically created +// driver/device it provides. +class DriverTest : public ::testing::Test { protected: + using DriverPtr = + std::unique_ptr; + DriverPtr CreateDriver() { + iree_status_t status = + register_test_driver(iree_hal_driver_registry_default()); + if (iree_status_is_already_exists(status)) { + status = iree_status_ignore(status); + } + IREE_CHECK_OK(status); + + // Get driver with the given name and create its default device. + // Skip drivers that are (gracefully) unavailable, fail if creation fails. + iree_hal_driver_t* driver = NULL; + status = TryGetDriver(get_test_driver_name(), &driver); + if (iree_status_is_unavailable(status)) { + iree_status_ignore(status); + return DriverPtr{nullptr, iree_hal_driver_release}; + } + IREE_CHECK_OK(status); + + return DriverPtr{driver, iree_hal_driver_release}; + } + void CheckCreateDeviceViaPath(iree_string_view_t name, iree_string_view_t path) { + auto driver = CreateDriver(); + std::cout << " Creating device '" << std::string(name.data, name.size) << "' with path '" << std::string(path.data, path.size) << "'\n"; iree_hal_device_t* device = NULL; iree_status_t status = iree_hal_driver_create_device_by_path( - driver_, name, path, /*param_count=*/0, /*params=*/NULL, + driver.get(), name, path, /*param_count=*/0, /*params=*/NULL, iree_allocator_system(), &device); // Creation via path is HAL driver specific. Allow unimplemented cases. @@ -46,11 +71,14 @@ class driver_test : public CTSTestBase<> { } }; -TEST_F(driver_test, QueryAndCreateAvailableDevicesByID) { +TEST_F(DriverTest, QueryAndCreateAvailableDevicesByID) { + auto driver = CreateDriver(); + iree_host_size_t device_info_count = 0; iree_hal_device_info_t* device_infos = NULL; IREE_ASSERT_OK(iree_hal_driver_query_available_devices( - driver_, iree_allocator_system(), &device_info_count, &device_infos)); + driver.get(), iree_allocator_system(), &device_info_count, + &device_infos)); std::cout << "Driver has " << device_info_count << " device(s)\n"; for (iree_host_size_t i = 0; i < device_info_count; ++i) { @@ -60,8 +88,8 @@ TEST_F(driver_test, QueryAndCreateAvailableDevicesByID) { << "'\n"; iree_hal_device_t* device = NULL; IREE_ASSERT_OK(iree_hal_driver_create_device_by_id( - driver_, device_infos[i].device_id, /*param_count=*/0, /*params=*/NULL, - iree_allocator_system(), &device)); + driver.get(), device_infos[i].device_id, /*param_count=*/0, + /*params=*/NULL, iree_allocator_system(), &device)); iree_string_view_t device_id = iree_hal_device_id(device); std::cout << " Created device with id: '" << std::string(device_id.data, device_id.size) << "'\n"; @@ -71,11 +99,14 @@ TEST_F(driver_test, QueryAndCreateAvailableDevicesByID) { iree_allocator_free(iree_allocator_system(), device_infos); } -TEST_F(driver_test, QueryAndCreateAvailableDevicesByOrdinal) { +TEST_F(DriverTest, QueryAndCreateAvailableDevicesByOrdinal) { + auto driver = CreateDriver(); + iree_host_size_t device_info_count = 0; iree_hal_device_info_t* device_infos = NULL; IREE_ASSERT_OK(iree_hal_driver_query_available_devices( - driver_, iree_allocator_system(), &device_info_count, &device_infos)); + driver.get(), iree_allocator_system(), &device_info_count, + &device_infos)); std::cout << "Driver has " << device_info_count << " device(s)\n"; for (iree_host_size_t i = 0; i < device_info_count; ++i) { @@ -85,8 +116,8 @@ TEST_F(driver_test, QueryAndCreateAvailableDevicesByOrdinal) { << "'\n"; iree_hal_device_t* device = NULL; IREE_ASSERT_OK(iree_hal_driver_create_device_by_ordinal( - driver_, i, /*param_count=*/0, /*params=*/NULL, iree_allocator_system(), - &device)); + driver.get(), i, /*param_count=*/0, /*params=*/NULL, + iree_allocator_system(), &device)); iree_string_view_t device_id = iree_hal_device_id(device); std::cout << " Created device with id: '" << std::string(device_id.data, device_id.size) << "'\n"; @@ -96,11 +127,14 @@ TEST_F(driver_test, QueryAndCreateAvailableDevicesByOrdinal) { iree_allocator_free(iree_allocator_system(), device_infos); } -TEST_F(driver_test, QueryAndCreateAvailableDevicesByPath) { +TEST_F(DriverTest, QueryAndCreateAvailableDevicesByPath) { + auto driver = CreateDriver(); + iree_host_size_t device_info_count = 0; iree_hal_device_info_t* device_infos = NULL; IREE_ASSERT_OK(iree_hal_driver_query_available_devices( - driver_, iree_allocator_system(), &device_info_count, &device_infos)); + driver.get(), iree_allocator_system(), &device_info_count, + &device_infos)); std::cout << "Driver has " << device_info_count << " device(s)\n"; if (device_info_count == 0) GTEST_SKIP() << "No available devices"; @@ -119,8 +153,6 @@ TEST_F(driver_test, QueryAndCreateAvailableDevicesByPath) { iree_allocator_free(iree_allocator_system(), device_infos); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_DRIVER_TEST_H_ diff --git a/runtime/src/iree/hal/cts/event_test.h b/runtime/src/iree/hal/cts/event_test.h index 47808a094304..6ddb4db41bff 100644 --- a/runtime/src/iree/hal/cts/event_test.h +++ b/runtime/src/iree/hal/cts/event_test.h @@ -15,19 +15,17 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { -class event_test : public CTSTestBase<> {}; +class EventTest : public CTSTestBase<> {}; -TEST_F(event_test, Create) { +TEST_F(EventTest, Create) { iree_hal_event_t* event = NULL; IREE_ASSERT_OK(iree_hal_event_create(device_, &event)); iree_hal_event_release(event); } -TEST_F(event_test, SignalAndReset) { +TEST_F(EventTest, SignalAndReset) { iree_hal_event_t* event = NULL; IREE_ASSERT_OK(iree_hal_event_create(device_, &event)); @@ -50,7 +48,7 @@ TEST_F(event_test, SignalAndReset) { iree_hal_command_buffer_release(command_buffer); } -TEST_F(event_test, SubmitWithChainedCommandBuffers) { +TEST_F(EventTest, SubmitWithChainedCommandBuffers) { iree_hal_event_t* event = NULL; IREE_ASSERT_OK(iree_hal_event_create(device_, &event)); @@ -96,8 +94,6 @@ TEST_F(event_test, SubmitWithChainedCommandBuffers) { iree_hal_event_release(event); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_EVENT_TEST_H_ diff --git a/runtime/src/iree/hal/cts/executable_cache_test.h b/runtime/src/iree/hal/cts/executable_cache_test.h index cb130e4e2b8e..8792fd97560e 100644 --- a/runtime/src/iree/hal/cts/executable_cache_test.h +++ b/runtime/src/iree/hal/cts/executable_cache_test.h @@ -14,13 +14,11 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { -class executable_cache_test : public CTSTestBase<> {}; +class ExecutableCacheTest : public CTSTestBase<> {}; -TEST_F(executable_cache_test, Create) { +TEST_F(ExecutableCacheTest, Create) { iree_status_t loop_status = iree_ok_status(); iree_hal_executable_cache_t* executable_cache = NULL; IREE_ASSERT_OK(iree_hal_executable_cache_create( @@ -31,7 +29,7 @@ TEST_F(executable_cache_test, Create) { IREE_ASSERT_OK(loop_status); } -TEST_F(executable_cache_test, CantPrepareUnknownFormat) { +TEST_F(ExecutableCacheTest, CantPrepareUnknownFormat) { iree_status_t loop_status = iree_ok_status(); iree_hal_executable_cache_t* executable_cache = NULL; IREE_ASSERT_OK(iree_hal_executable_cache_create( @@ -45,7 +43,7 @@ TEST_F(executable_cache_test, CantPrepareUnknownFormat) { IREE_ASSERT_OK(loop_status); } -TEST_F(executable_cache_test, PrepareExecutable) { +TEST_F(ExecutableCacheTest, PrepareExecutable) { iree_status_t loop_status = iree_ok_status(); iree_hal_executable_cache_t* executable_cache = NULL; IREE_ASSERT_OK(iree_hal_executable_cache_create( @@ -97,8 +95,6 @@ TEST_F(executable_cache_test, PrepareExecutable) { IREE_ASSERT_OK(loop_status); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_EXECUTABLE_CACHE_TEST_H_ diff --git a/runtime/src/iree/hal/cts/file_test.h b/runtime/src/iree/hal/cts/file_test.h index b59e61a71abd..b7db5f4c96b7 100644 --- a/runtime/src/iree/hal/cts/file_test.h +++ b/runtime/src/iree/hal/cts/file_test.h @@ -16,9 +16,7 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { using ::testing::ContainerEq; @@ -26,7 +24,7 @@ namespace { constexpr iree_device_size_t kMinimumAlignment = 128; } // namespace -class file_test : public CTSTestBase<> { +class FileTest : public CTSTestBase<> { protected: void CreatePatternedDeviceBuffer(iree_device_size_t buffer_size, uint8_t pattern, @@ -86,7 +84,7 @@ class file_test : public CTSTestBase<> { }; // Reads the entire file into a buffer and check the contents match. -TEST_F(file_test, ReadEntireFile) { +TEST_F(FileTest, ReadEntireFile) { iree_device_size_t file_size = 128; iree_hal_file_t* file = NULL; CreatePatternedMemoryFile(IREE_HAL_MEMORY_ACCESS_READ, file_size, 0xDEu, @@ -133,8 +131,6 @@ TEST_F(file_test, ReadEntireFile) { iree_hal_file_release(file); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_FILE_TEST_H_ diff --git a/runtime/src/iree/hal/cts/pipeline_layout_test.h b/runtime/src/iree/hal/cts/pipeline_layout_test.h index 1cb02cf99a08..4342c6a0ee62 100644 --- a/runtime/src/iree/hal/cts/pipeline_layout_test.h +++ b/runtime/src/iree/hal/cts/pipeline_layout_test.h @@ -13,13 +13,11 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { -class pipeline_layout_test : public CTSTestBase<> {}; +class PipelineLayoutTest : public CTSTestBase<> {}; -TEST_F(pipeline_layout_test, CreateWithNoLayouts) { +TEST_F(PipelineLayoutTest, CreateWithNoLayouts) { iree_hal_pipeline_layout_t* pipeline_layout = NULL; IREE_ASSERT_OK(iree_hal_pipeline_layout_create(device_, /*push_constants=*/0, /*set_layout_count=*/0, NULL, @@ -28,7 +26,7 @@ TEST_F(pipeline_layout_test, CreateWithNoLayouts) { iree_hal_pipeline_layout_release(pipeline_layout); } -TEST_F(pipeline_layout_test, CreateWithPushConstants) { +TEST_F(PipelineLayoutTest, CreateWithPushConstants) { iree_hal_pipeline_layout_t* pipeline_layout = NULL; // Note: The Vulkan maxPushConstantsSize limit must be at least 128 bytes: // https://www.khronos.org/registry/vulkan/specs/1.2/html/vkspec.html#limits-minmax @@ -39,7 +37,7 @@ TEST_F(pipeline_layout_test, CreateWithPushConstants) { iree_hal_pipeline_layout_release(pipeline_layout); } -TEST_F(pipeline_layout_test, CreateWithOneLayout) { +TEST_F(PipelineLayoutTest, CreateWithOneLayout) { iree_hal_descriptor_set_layout_t* descriptor_set_layout = NULL; iree_hal_descriptor_set_layout_binding_t descriptor_set_layout_bindings[] = { { @@ -67,7 +65,7 @@ TEST_F(pipeline_layout_test, CreateWithOneLayout) { iree_hal_descriptor_set_layout_release(descriptor_set_layout); } -TEST_F(pipeline_layout_test, CreateWithTwoLayouts) { +TEST_F(PipelineLayoutTest, CreateWithTwoLayouts) { iree_hal_descriptor_set_layout_t* descriptor_set_layouts[2] = {NULL}; iree_hal_descriptor_set_layout_binding_t layout_bindings_0[] = { { @@ -118,8 +116,6 @@ TEST_F(pipeline_layout_test, CreateWithTwoLayouts) { iree_hal_descriptor_set_layout_release(descriptor_set_layouts[1]); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_PIPELINE_LAYOUT_TEST_H_ diff --git a/runtime/src/iree/hal/cts/semaphore_submission_test.h b/runtime/src/iree/hal/cts/semaphore_submission_test.h index 824471eb6b23..40d0a0f878e5 100644 --- a/runtime/src/iree/hal/cts/semaphore_submission_test.h +++ b/runtime/src/iree/hal/cts/semaphore_submission_test.h @@ -17,13 +17,11 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { -class semaphore_submission_test : public CTSTestBase<> {}; +class SemaphoreSubmissionTest : public CTSTestBase<> {}; -TEST_F(semaphore_submission_test, SubmitWithNoCommandBuffers) { +TEST_F(SemaphoreSubmissionTest, SubmitWithNoCommandBuffers) { // No waits, one signal which we immediately wait on after submit. iree_hal_semaphore_t* signal_semaphore = CreateSemaphore(); uint64_t signal_payload_values[] = {1}; @@ -43,7 +41,7 @@ TEST_F(semaphore_submission_test, SubmitWithNoCommandBuffers) { iree_hal_semaphore_release(signal_semaphore); } -TEST_F(semaphore_submission_test, SubmitAndSignal) { +TEST_F(SemaphoreSubmissionTest, SubmitAndSignal) { iree_hal_command_buffer_t* command_buffer = CreateEmptyCommandBuffer(); // No waits, one signal which we immediately wait on after submit. @@ -66,7 +64,7 @@ TEST_F(semaphore_submission_test, SubmitAndSignal) { iree_hal_semaphore_release(signal_semaphore); } -TEST_F(semaphore_submission_test, SubmitWithWait) { +TEST_F(SemaphoreSubmissionTest, SubmitWithWait) { // Empty command buffer. iree_hal_command_buffer_t* command_buffer = CreateEmptyCommandBuffer(); @@ -105,7 +103,7 @@ TEST_F(semaphore_submission_test, SubmitWithWait) { iree_hal_semaphore_release(signal_semaphore); } -TEST_F(semaphore_submission_test, SubmitWithMultipleSemaphores) { +TEST_F(SemaphoreSubmissionTest, SubmitWithMultipleSemaphores) { iree_hal_command_buffer_t* command_buffer = CreateEmptyCommandBuffer(); iree_hal_semaphore_t* wait_semaphore_1 = CreateSemaphore(); @@ -154,7 +152,7 @@ TEST_F(semaphore_submission_test, SubmitWithMultipleSemaphores) { } // Tests we can wait on both host and device semaphore to singal. -TEST_F(semaphore_submission_test, WaitAllHostAndDeviceSemaphores) { +TEST_F(SemaphoreSubmissionTest, WaitAllHostAndDeviceSemaphores) { iree_hal_command_buffer_t* command_buffer = CreateEmptyCommandBuffer(); // Create two semaphores, one for the host thread to wait on, and one for the @@ -217,7 +215,7 @@ TEST_F(semaphore_submission_test, WaitAllHostAndDeviceSemaphores) { // Tests that we can wait on any host and device semaphore to singal, // and device signals. -TEST_F(semaphore_submission_test, +TEST_F(SemaphoreSubmissionTest, WaitAnyHostAndDeviceSemaphoresAndDeviceSignals) { iree_hal_command_buffer_t* command_buffer = CreateEmptyCommandBuffer(); @@ -286,8 +284,7 @@ TEST_F(semaphore_submission_test, // Tests we can wait on any host and device semaphore to singal, // and host signals. -TEST_F(semaphore_submission_test, - WaitAnyHostAndDeviceSemaphoresAndHostSignals) { +TEST_F(SemaphoreSubmissionTest, WaitAnyHostAndDeviceSemaphoresAndHostSignals) { iree_hal_command_buffer_t* command_buffer = CreateEmptyCommandBuffer(); // Create two semaphores, one for the host thread to wait on, and one for the @@ -359,7 +356,7 @@ TEST_F(semaphore_submission_test, // Test device -> device synchronization: submit two batches with a // semaphore signal -> wait dependency. -TEST_F(semaphore_submission_test, IntermediateSemaphoreBetweenDeviceBatches) { +TEST_F(SemaphoreSubmissionTest, IntermediateSemaphoreBetweenDeviceBatches) { // The signaling relationship is // command_buffer1 -> semaphore1 -> command_buffer2 -> semaphore2 @@ -419,7 +416,7 @@ TEST_F(semaphore_submission_test, IntermediateSemaphoreBetweenDeviceBatches) { // Test device -> device synchronization: submit multiple batches with // multiple later batches waiting on the same signaling from a former batch. -TEST_F(semaphore_submission_test, TwoBatchesWaitingOn1FormerBatchAmongst2) { +TEST_F(SemaphoreSubmissionTest, TwoBatchesWaitingOn1FormerBatchAmongst2) { // The signaling-wait relation is: // command_buffer11 command_buffer12 // ↓ @@ -502,7 +499,7 @@ TEST_F(semaphore_submission_test, TwoBatchesWaitingOn1FormerBatchAmongst2) { // Test device -> device synchronization: submit multiple batches with // a former batch signaling a value greater than all other batches' (different) // wait values. -TEST_F(semaphore_submission_test, TwoBatchesWaitingOnDifferentSemaphoreValues) { +TEST_F(SemaphoreSubmissionTest, TwoBatchesWaitingOnDifferentSemaphoreValues) { // The signal-wait relation is // // command_buffer11 @@ -590,7 +587,7 @@ TEST_F(semaphore_submission_test, TwoBatchesWaitingOnDifferentSemaphoreValues) { // Test host + device -> device synchronization: submit two batches // with a later batch waiting on both a host and device signal to proceed. -TEST_F(semaphore_submission_test, BatchWaitingOnAnotherAndHostSignal) { +TEST_F(SemaphoreSubmissionTest, BatchWaitingOnAnotherAndHostSignal) { // Signal/wait relation: // // command_buffer1 @@ -665,7 +662,7 @@ TEST_F(semaphore_submission_test, BatchWaitingOnAnotherAndHostSignal) { // Test device -> host + device synchronization: submit two batches // with a former batch signaling to enable both host and device to proceed. -TEST_F(semaphore_submission_test, DeviceBatchSignalAnotherAndHost) { +TEST_F(SemaphoreSubmissionTest, DeviceBatchSignalAnotherAndHost) { // Signal-wait relation: // // command_buffer1 @@ -753,7 +750,7 @@ TEST_F(semaphore_submission_test, DeviceBatchSignalAnotherAndHost) { // Test signaling a larger value before enqueuing waiting a smaller // value to the device. -TEST_F(semaphore_submission_test, BatchWaitingOnSmallerValueAfterSignaled) { +TEST_F(SemaphoreSubmissionTest, BatchWaitingOnSmallerValueAfterSignaled) { // signal-wait relation: // // signal value 2 @@ -797,7 +794,7 @@ TEST_F(semaphore_submission_test, BatchWaitingOnSmallerValueAfterSignaled) { // Test signaling a larger value after enqueuing waiting a smaller // value to the device. -TEST_F(semaphore_submission_test, BatchWaitingOnSmallerValueBeforeSignaled) { +TEST_F(SemaphoreSubmissionTest, BatchWaitingOnSmallerValueBeforeSignaled) { // signal-wait relation: // // signal value 2 @@ -844,8 +841,6 @@ TEST_F(semaphore_submission_test, BatchWaitingOnSmallerValueBeforeSignaled) { iree_hal_command_buffer_release(command_buffer); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_SEMAPHORE_SUBMISSION_TEST_H_ diff --git a/runtime/src/iree/hal/cts/semaphore_test.h b/runtime/src/iree/hal/cts/semaphore_test.h index da2f297d069c..2f84b6f9f4d3 100644 --- a/runtime/src/iree/hal/cts/semaphore_test.h +++ b/runtime/src/iree/hal/cts/semaphore_test.h @@ -17,15 +17,14 @@ #include "iree/testing/gtest.h" #include "iree/testing/status_matchers.h" -namespace iree { -namespace hal { -namespace cts { +namespace iree::hal::cts { + using namespace std::chrono_literals; -class semaphore_test : public CTSTestBase<> {}; +class SemaphoreTest : public CTSTestBase<> {}; // Tests that a semaphore that is unused properly cleans itself up. -TEST_F(semaphore_test, NoOp) { +TEST_F(SemaphoreTest, NoOp) { iree_hal_semaphore_t* semaphore = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 123ull, &semaphore)); @@ -37,7 +36,7 @@ TEST_F(semaphore_test, NoOp) { } // Tests that a semaphore will accept new values as it is signaled. -TEST_F(semaphore_test, NormalSignaling) { +TEST_F(SemaphoreTest, NormalSignaling) { iree_hal_semaphore_t* semaphore = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 2ull, &semaphore)); @@ -59,7 +58,7 @@ TEST_F(semaphore_test, NormalSignaling) { // while others may accept the new, decreasing, values. // Tests semaphore failure handling. -TEST_F(semaphore_test, Failure) { +TEST_F(SemaphoreTest, Failure) { iree_hal_semaphore_t* semaphore = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 2ull, &semaphore)); @@ -80,7 +79,7 @@ TEST_F(semaphore_test, Failure) { } // Tests waiting on no semaphores. -TEST_F(semaphore_test, EmptyWait) { +TEST_F(SemaphoreTest, EmptyWait) { IREE_ASSERT_OK(iree_hal_device_wait_semaphores( device_, IREE_HAL_WAIT_MODE_ANY, iree_hal_semaphore_list_empty(), iree_make_deadline(IREE_TIME_INFINITE_FUTURE))); @@ -97,7 +96,7 @@ TEST_F(semaphore_test, EmptyWait) { } // Tests waiting on a semaphore that has already been signaled. -TEST_F(semaphore_test, WaitAlreadySignaled) { +TEST_F(SemaphoreTest, WaitAlreadySignaled) { iree_hal_semaphore_t* semaphore = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 2ull, &semaphore)); @@ -116,7 +115,7 @@ TEST_F(semaphore_test, WaitAlreadySignaled) { } // Tests waiting on a semaphore that has not been signaled. -TEST_F(semaphore_test, WaitUnsignaled) { +TEST_F(SemaphoreTest, WaitUnsignaled) { iree_hal_semaphore_t* semaphore = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 2ull, &semaphore)); @@ -130,7 +129,7 @@ TEST_F(semaphore_test, WaitUnsignaled) { } // Tests waiting on a semaphore that has signals past the desired value. -TEST_F(semaphore_test, WaitLaterSignaledBeyond) { +TEST_F(SemaphoreTest, WaitLaterSignaledBeyond) { iree_hal_semaphore_t* semaphore = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 2ull, &semaphore)); @@ -152,7 +151,7 @@ TEST_F(semaphore_test, WaitLaterSignaledBeyond) { // return UnknownError while others may succeed. // Tests IREE_HAL_WAIT_MODE_ALL when not all are signaled. -TEST_F(semaphore_test, WaitAllButNotAllSignaled) { +TEST_F(SemaphoreTest, WaitAllButNotAllSignaled) { iree_hal_semaphore_t* semaphore_a = NULL; iree_hal_semaphore_t* semaphore_b = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 0ull, &semaphore_a)); @@ -177,7 +176,7 @@ TEST_F(semaphore_test, WaitAllButNotAllSignaled) { } // Tests IREE_HAL_WAIT_MODE_ALL when all are signaled. -TEST_F(semaphore_test, WaitAllAndAllSignaled) { +TEST_F(SemaphoreTest, WaitAllAndAllSignaled) { iree_hal_semaphore_t* semaphore_a = NULL; iree_hal_semaphore_t* semaphore_b = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 1ull, &semaphore_a)); @@ -202,7 +201,7 @@ TEST_F(semaphore_test, WaitAllAndAllSignaled) { } // Tests IREE_HAL_WAIT_MODE_ANY. -TEST_F(semaphore_test, WaitAnyAlreadySignaled) { +TEST_F(SemaphoreTest, WaitAnyAlreadySignaled) { iree_hal_semaphore_t* semaphore_a = NULL; iree_hal_semaphore_t* semaphore_b = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 0ull, &semaphore_a)); @@ -223,7 +222,7 @@ TEST_F(semaphore_test, WaitAnyAlreadySignaled) { iree_hal_semaphore_release(semaphore_b); } -TEST_F(semaphore_test, WaitAnyLaterSignaled) { +TEST_F(SemaphoreTest, WaitAnyLaterSignaled) { iree_hal_semaphore_t* semaphore_a = NULL; iree_hal_semaphore_t* semaphore_b = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 0ull, &semaphore_a)); @@ -253,7 +252,7 @@ TEST_F(semaphore_test, WaitAnyLaterSignaled) { // Tests threading behavior by ping-ponging between the test main thread and // a little thread. -TEST_F(semaphore_test, PingPong) { +TEST_F(SemaphoreTest, PingPong) { iree_hal_semaphore_t* a2b = NULL; iree_hal_semaphore_t* b2a = NULL; IREE_ASSERT_OK(iree_hal_semaphore_create(device_, 0ull, &a2b)); @@ -278,7 +277,7 @@ TEST_F(semaphore_test, PingPong) { } // Waiting the same value multiple times. -TEST_F(semaphore_test, WaitOnTheSameValueMultipleTimes) { +TEST_F(SemaphoreTest, WaitOnTheSameValueMultipleTimes) { iree_hal_semaphore_t* semaphore = CreateSemaphore(); std::thread thread( [&]() { IREE_ASSERT_OK(iree_hal_semaphore_signal(semaphore, 1)); }); @@ -297,7 +296,7 @@ TEST_F(semaphore_test, WaitOnTheSameValueMultipleTimes) { } // Waiting for a finite amount of time. -TEST_F(semaphore_test, WaitForFiniteTime) { +TEST_F(SemaphoreTest, WaitForFiniteTime) { auto generic_test_fn = [this](auto wait_fn) { iree_hal_semaphore_t* semaphore = this->CreateSemaphore(); @@ -341,7 +340,7 @@ TEST_F(semaphore_test, WaitForFiniteTime) { } // Wait on all semaphores on multiple places simultaneously. -TEST_F(semaphore_test, SimultaneousMultiWaitAll) { +TEST_F(SemaphoreTest, SimultaneousMultiWaitAll) { iree_hal_semaphore_t* semaphore1 = this->CreateSemaphore(); iree_hal_semaphore_t* semaphore2 = this->CreateSemaphore(); @@ -378,8 +377,6 @@ TEST_F(semaphore_test, SimultaneousMultiWaitAll) { iree_hal_semaphore_release(semaphore2); } -} // namespace cts -} // namespace hal -} // namespace iree +} // namespace iree::hal::cts #endif // IREE_HAL_CTS_SEMAPHORE_TEST_H_ diff --git a/runtime/src/iree/hal/cts/testdata/command_buffer_dispatch_test.mlir b/runtime/src/iree/hal/cts/testdata/command_buffer_dispatch_test.mlir index 366a9f34bb0e..0067230abaf7 100644 --- a/runtime/src/iree/hal/cts/testdata/command_buffer_dispatch_test.mlir +++ b/runtime/src/iree/hal/cts/testdata/command_buffer_dispatch_test.mlir @@ -1,8 +1,8 @@ // Bootstrapped from this source IR: // -// func.func @abs(%input : tensor) -> (tensor) { -// %result = math.absf %input : tensor -// return %result : tensor +// func.func @abs(%input : tensor<2xf32>) -> (tensor<2xf32>) { +// %result = math.absf %input : tensor<2xf32> +// return %result : tensor<2xf32> // } #pipeline_layout = #hal.pipeline.layout - %1 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(32) offset(%c0) : !flow.dispatch.tensor + %0 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(4) offset(%c0) : !flow.dispatch.tensor> + %1 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(4) offset(%c0) : !flow.dispatch.tensor> - %2 = flow.dispatch.tensor.load %0, offsets = [], sizes = [], strides = [] : !flow.dispatch.tensor -> tensor - %3 = tensor.empty() : tensor - %4 = linalg.generic {indexing_maps = [affine_map<() -> ()>, affine_map<() -> ()>], iterator_types = []} ins(%2 : tensor) outs(%3 : tensor) { + %2 = flow.dispatch.tensor.load %0, offsets = [0], sizes = [2], strides = [1] : !flow.dispatch.tensor> -> tensor<2xf32> + %3 = tensor.empty() : tensor<2xf32> + %4 = linalg.generic {indexing_maps = [affine_map<(d0) -> (d0)>, affine_map<(d0) -> (d0)>], iterator_types = ["parallel"]} ins(%2 : tensor<2xf32>) outs(%3 : tensor<2xf32>) { ^bb0(%arg0: f32, %arg1: f32): %5 = math.absf %arg0 : f32 linalg.yield %5 : f32 - } -> tensor - flow.dispatch.tensor.store %4, %1, offsets = [], sizes = [], strides = [] : tensor -> !flow.dispatch.tensor + } -> tensor<2xf32> + flow.dispatch.tensor.store %4, %1, offsets = [0], sizes = [2], strides = [1] : tensor<2xf32> -> !flow.dispatch.tensor> return } diff --git a/runtime/src/iree/hal/drivers/cuda/pending_queue_actions.c b/runtime/src/iree/hal/drivers/cuda/pending_queue_actions.c index 717932d94457..86d5762379f2 100644 --- a/runtime/src/iree/hal/drivers/cuda/pending_queue_actions.c +++ b/runtime/src/iree/hal/drivers/cuda/pending_queue_actions.c @@ -738,9 +738,14 @@ static iree_status_t iree_hal_cuda_pending_queue_actions_issue_execution( } else { iree_hal_command_buffer_t* stream_command_buffer = NULL; iree_hal_command_buffer_mode_t mode = + iree_hal_command_buffer_mode(command_buffer) | IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT | IREE_HAL_COMMAND_BUFFER_MODE_ALLOW_INLINE_EXECUTION | - IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED; + // NOTE: we need to validate if a binding table is provided as the + // bindings were not known when it was originally recorded. + (iree_hal_buffer_binding_table_is_empty(binding_table) + ? IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED + : 0); IREE_RETURN_AND_END_ZONE_IF_ERROR( z0, iree_hal_cuda_device_create_stream_command_buffer( action->device, mode, IREE_HAL_COMMAND_CATEGORY_ANY, diff --git a/runtime/src/iree/hal/drivers/hip/pending_queue_actions.c b/runtime/src/iree/hal/drivers/hip/pending_queue_actions.c index 7e1dcff66c9e..7fe40b5601c0 100644 --- a/runtime/src/iree/hal/drivers/hip/pending_queue_actions.c +++ b/runtime/src/iree/hal/drivers/hip/pending_queue_actions.c @@ -743,9 +743,14 @@ static iree_status_t iree_hal_hip_pending_queue_actions_issue_execution( } else { iree_hal_command_buffer_t* stream_command_buffer = NULL; iree_hal_command_buffer_mode_t mode = + iree_hal_command_buffer_mode(command_buffer) | IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT | IREE_HAL_COMMAND_BUFFER_MODE_ALLOW_INLINE_EXECUTION | - IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED; + // NOTE: we need to validate if a binding table is provided as the + // bindings were not known when it was originally recorded. + (iree_hal_buffer_binding_table_is_empty(binding_table) + ? IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED + : 0); IREE_RETURN_AND_END_ZONE_IF_ERROR( z0, iree_hal_hip_device_create_stream_command_buffer( action->device, mode, IREE_HAL_COMMAND_CATEGORY_ANY, diff --git a/runtime/src/iree/hal/drivers/local_sync/sync_device.c b/runtime/src/iree/hal/drivers/local_sync/sync_device.c index e3a384c65693..b29174378328 100644 --- a/runtime/src/iree/hal/drivers/local_sync/sync_device.c +++ b/runtime/src/iree/hal/drivers/local_sync/sync_device.c @@ -398,11 +398,27 @@ static iree_status_t iree_hal_sync_device_apply_deferred_command_buffers( // Stack allocate storage for an inline command buffer we'll use to replay // the deferred command buffers. We want to reset it between each apply so // that we don't get state carrying across. - iree_host_size_t storage_size = iree_hal_inline_command_buffer_size( - IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED, - /*binding_capacity=*/0); + iree_host_size_t max_storage_size = 0; + for (iree_host_size_t i = 0; i < command_buffer_count; ++i) { + iree_hal_command_buffer_t* command_buffer = command_buffers[i]; + iree_hal_buffer_binding_table_t binding_table = + binding_tables ? binding_tables[i] + : iree_hal_buffer_binding_table_empty(); + max_storage_size = iree_max( + max_storage_size, + iree_hal_inline_command_buffer_size( + iree_hal_command_buffer_mode(command_buffer) | + IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT | + IREE_HAL_COMMAND_BUFFER_MODE_ALLOW_INLINE_EXECUTION | + // NOTE: we need to validate if a binding table is provided as + // the bindings were not known when it was originally recorded. + (iree_hal_buffer_binding_table_is_empty(binding_table) + ? IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED + : 0), + /*binding_capacity=*/0)); + } iree_byte_span_t storage = - iree_make_byte_span(iree_alloca(storage_size), storage_size); + iree_make_byte_span(iree_alloca(max_storage_size), max_storage_size); // NOTE: we ignore any inline command buffers that may be passed in as they've // already executed during recording. The caller is probably in for a bad time @@ -419,9 +435,15 @@ static iree_status_t iree_hal_sync_device_apply_deferred_command_buffers( IREE_RETURN_IF_ERROR(iree_hal_inline_command_buffer_initialize( device->device_allocator, iree_hal_command_buffer_mode(command_buffer) | + IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT | IREE_HAL_COMMAND_BUFFER_MODE_ALLOW_INLINE_EXECUTION | - IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED, - IREE_HAL_COMMAND_CATEGORY_ANY, IREE_HAL_QUEUE_AFFINITY_ANY, + // NOTE: we need to validate if a binding table is provided as the + // bindings were not known when it was originally recorded. + (iree_hal_buffer_binding_table_is_empty(binding_table) + ? IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED + : 0), + iree_hal_command_buffer_allowed_categories(command_buffer), + IREE_HAL_QUEUE_AFFINITY_ANY, /*binding_capacity=*/0, device->host_allocator, storage, &inline_command_buffer)); iree_status_t status = iree_hal_deferred_command_buffer_apply( diff --git a/runtime/src/iree/hal/drivers/local_task/task_queue.c b/runtime/src/iree/hal/drivers/local_task/task_queue.c index ebf093d9eb31..dde76b5ae9ab 100644 --- a/runtime/src/iree/hal/drivers/local_task/task_queue.c +++ b/runtime/src/iree/hal/drivers/local_task/task_queue.c @@ -189,6 +189,8 @@ typedef struct iree_hal_task_queue_issue_cmd_t { iree_hal_task_queue_t* queue; // A resource set containing all binding table buffers. + // Owned by the retire command and any resources added will be retained until + // the submission has completed (or failed). iree_hal_resource_set_t* resource_set; // Command buffers to be issued in the order they appeared in the submission. @@ -209,14 +211,21 @@ static iree_status_t iree_hal_task_queue_issue_cmd_deferred( // they may not run immediately. iree_hal_command_buffer_t* task_command_buffer = NULL; IREE_RETURN_AND_END_ZONE_IF_ERROR( - z0, iree_hal_task_command_buffer_create( - cmd->queue->device_allocator, &cmd->queue->scope, - iree_hal_command_buffer_mode(command_buffer), - iree_hal_command_buffer_allowed_categories(command_buffer), - cmd->queue->affinity, /*binding_capacity=*/0, - cmd->queue->large_block_pool, - iree_hal_allocator_host_allocator(cmd->queue->device_allocator), - &task_command_buffer)); + z0, + iree_hal_task_command_buffer_create( + cmd->queue->device_allocator, &cmd->queue->scope, + iree_hal_command_buffer_mode(command_buffer) | + IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT | + // NOTE: we need to validate if a binding table is provided as the + // bindings were not known when it was originally recorded. + (iree_hal_buffer_binding_table_is_empty(binding_table) + ? IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED + : 0), + iree_hal_command_buffer_allowed_categories(command_buffer), + cmd->queue->affinity, /*binding_capacity=*/0, + cmd->queue->large_block_pool, + iree_hal_allocator_host_allocator(cmd->queue->device_allocator), + &task_command_buffer)); // Keep the command buffer live until the queue operation completes. iree_status_t status = @@ -242,6 +251,9 @@ static iree_status_t iree_hal_task_queue_issue_cmd_deferred( cmd->task.header.completion_task, cmd->arena, pending_submission)); + // Still retained in the resource set until retirement. + iree_hal_command_buffer_release(task_command_buffer); + IREE_TRACE_ZONE_END(z0); return iree_ok_status(); } @@ -287,24 +299,11 @@ static iree_status_t iree_hal_task_queue_issue_cmd( return status; } -// Cleanup for iree_hal_task_queue_issue_cmd_t that releases the retained -// semaphores. -static void iree_hal_task_queue_issue_cmd_cleanup( - iree_task_t* task, iree_status_code_t status_code) { - iree_hal_task_queue_issue_cmd_t* cmd = (iree_hal_task_queue_issue_cmd_t*)task; - if (cmd->resource_set) { - IREE_TRACE_ZONE_BEGIN(z0); - iree_hal_resource_set_free(cmd->resource_set); - IREE_TRACE_ZONE_END(z0); - } -} - // Allocates and initializes a iree_hal_task_queue_issue_cmd_t task. static iree_status_t iree_hal_task_queue_issue_cmd_allocate( void* user_data, iree_task_scope_t* scope, iree_hal_task_queue_t* queue, - iree_host_size_t resource_count, iree_hal_resource_t* const* resources, iree_task_t* retire_task, iree_arena_allocator_t* arena, - iree_task_t** out_issue_task) { + iree_hal_resource_set_t* resource_set, iree_task_t** out_issue_task) { iree_hal_task_submission_batch_t* batch = (iree_hal_task_submission_batch_t*)user_data; @@ -329,30 +328,15 @@ static iree_status_t iree_hal_task_queue_issue_cmd_allocate( iree_task_call_initialize( scope, iree_task_make_call_closure(iree_hal_task_queue_issue_cmd, 0), &cmd->task); - iree_task_set_cleanup_fn(&cmd->task.header, - iree_hal_task_queue_issue_cmd_cleanup); iree_task_set_completion_task(&cmd->task.header, retire_task); cmd->arena = arena; cmd->queue = queue; + cmd->resource_set = resource_set; cmd->command_buffer_count = batch->command_buffer_count; cmd->command_buffers = (iree_hal_command_buffer_t**)((uint8_t*)cmd + sizeof(*cmd)); - bool has_any_deferred = false; - for (iree_host_size_t i = 0; i < batch->command_buffer_count; ++i) { - iree_hal_command_buffer_t* command_buffer = batch->command_buffers[i]; - cmd->command_buffers[i] = batch->command_buffers[i]; - has_any_deferred |= iree_hal_deferred_command_buffer_isa(command_buffer); - } - - // Only create a resource set if we know we need it. - // NOTE: if this fails we'll unwind and release the cmd arena in the caller. - if (has_any_deferred || binding_table_elements_size > 0) { - IREE_RETURN_IF_ERROR( - iree_hal_resource_set_allocate(arena->block_pool, &cmd->resource_set)); - } else { - cmd->resource_set = NULL; - } + memcpy(cmd->command_buffers, batch->command_buffers, command_buffers_size); // Binding tables are optional and we only need this extra work if there were // any non-empty binding tables provided during submission. @@ -363,7 +347,8 @@ static iree_status_t iree_hal_task_queue_issue_cmd_allocate( (iree_hal_buffer_binding_table_t*)((uint8_t*)cmd->command_buffers + command_buffers_size); iree_hal_buffer_binding_t* binding_element_ptr = - (iree_hal_buffer_binding_t*)(cmd->binding_tables + binding_tables_size); + (iree_hal_buffer_binding_t*)((uint8_t*)cmd->binding_tables + + binding_tables_size); for (iree_host_size_t i = 0; i < batch->command_buffer_count; ++i) { iree_host_size_t element_count = batch->binding_tables[i].count; cmd->binding_tables[i].count = element_count; @@ -388,9 +373,6 @@ static iree_status_t iree_hal_task_queue_issue_cmd_allocate( if (iree_status_is_ok(status)) { *out_issue_task = &cmd->task.header; - } else { - iree_hal_resource_set_free(cmd->resource_set); - cmd->resource_set = NULL; } return status; } @@ -418,8 +400,10 @@ typedef struct iree_hal_task_queue_retire_cmd_t { // Resources retained until all have retired. // We could release them earlier but that would require tracking individual // resource-level completion. - iree_host_size_t resource_count; - iree_hal_resource_t* resources[]; + // + // This resource set is allocated from the small block pool and is expected to + // only have a small number of resources (command buffers, etc). + iree_hal_resource_set_t* resource_set; } iree_hal_task_queue_retire_cmd_t; // Retires a submission by signaling semaphores to their desired value and @@ -431,13 +415,11 @@ static iree_status_t iree_hal_task_queue_retire_cmd( (iree_hal_task_queue_retire_cmd_t*)task; IREE_TRACE_ZONE_BEGIN(z0); - // Release command buffers now that all are known to have retired. + // Release retained resources (command buffers, etc). // We do this before signaling so that waiting threads can immediately reuse // resources that are released. - for (iree_host_size_t i = 0; i < cmd->resource_count; ++i) { - iree_hal_resource_release(cmd->resources[i]); - cmd->resources[i] = NULL; - } + iree_hal_resource_set_free(cmd->resource_set); + cmd->resource_set = NULL; // Signal all semaphores to their new values. // Note that if any signal fails then the whole command will fail and all @@ -466,11 +448,9 @@ static void iree_hal_task_queue_retire_cmd_cleanup( // Release resources now that all are known to have retired. // In success cases we try to do this eagerly to allow for more potential // reuse but during full/partial failures they may still be live here. - for (iree_host_size_t i = 0; i < cmd->resource_count; ++i) { - if (cmd->resources[i]) { - iree_hal_resource_release(cmd->resources[i]); - cmd->resources[i] = NULL; - } + if (!cmd->resource_set) { + iree_hal_resource_set_free(cmd->resource_set); + cmd->resource_set = NULL; } // If the command failed then fail all semaphores to ensure future @@ -497,8 +477,7 @@ static void iree_hal_task_queue_retire_cmd_cleanup( // The command will own an arena that can be used for other submission-related // allocations. static iree_status_t iree_hal_task_queue_retire_cmd_allocate( - iree_task_scope_t* scope, iree_host_size_t resource_count, - iree_hal_resource_t* const* resources, + iree_task_scope_t* scope, const iree_hal_semaphore_list_t* signal_semaphores, iree_arena_block_pool_t* block_pool, iree_hal_task_queue_retire_cmd_t** out_cmd) { @@ -508,18 +487,21 @@ static iree_status_t iree_hal_task_queue_retire_cmd_allocate( // Allocate the command from the arena. iree_hal_task_queue_retire_cmd_t* cmd = NULL; - iree_host_size_t total_cmd_size = - sizeof(*cmd) + resource_count * sizeof(*cmd->resources); iree_status_t status = - iree_arena_allocate(&arena, total_cmd_size, (void**)&cmd); - if (iree_status_is_ok(status)) { - iree_task_call_initialize( - scope, iree_task_make_call_closure(iree_hal_task_queue_retire_cmd, 0), - &cmd->task); - iree_task_set_cleanup_fn(&cmd->task.header, - iree_hal_task_queue_retire_cmd_cleanup); + iree_arena_allocate(&arena, sizeof(*cmd), (void**)&cmd); + if (!iree_status_is_ok(status)) { + iree_arena_deinitialize(&arena); + return status; } + iree_task_call_initialize( + scope, iree_task_make_call_closure(iree_hal_task_queue_retire_cmd, 0), + &cmd->task); + iree_task_set_cleanup_fn(&cmd->task.header, + iree_hal_task_queue_retire_cmd_cleanup); + cmd->signal_semaphores = iree_hal_semaphore_list_empty(); + cmd->resource_set = NULL; + // Clone the signal semaphores from the batch - we retain them and their // payloads. if (iree_status_is_ok(status)) { @@ -527,20 +509,22 @@ static iree_status_t iree_hal_task_queue_retire_cmd_allocate( &cmd->signal_semaphores); } + // Create a lightweight resource set to retain any resources used by the + // command. Note that this is coming from the small block pool and is intended + // only for a small number of resources. + if (iree_status_is_ok(status)) { + status = iree_hal_resource_set_allocate(block_pool, &cmd->resource_set); + } + if (iree_status_is_ok(status)) { // Transfer ownership of the arena to command. memcpy(&cmd->arena, &arena, sizeof(cmd->arena)); - - // Retain command buffers. - cmd->resource_count = resource_count; - for (iree_host_size_t i = 0; i < resource_count; ++i) { - iree_hal_resource_t* resource = resources[i]; - iree_hal_resource_retain(resource); - cmd->resources[i] = resource; - } - *out_cmd = cmd; } else { + if (cmd) { + iree_hal_resource_set_free(cmd->resource_set); + iree_hal_semaphore_list_release(&cmd->signal_semaphores); + } iree_arena_deinitialize(&arena); } return status; @@ -599,9 +583,8 @@ void iree_hal_task_queue_trim(iree_hal_task_queue_t* queue) { typedef iree_status_t(IREE_API_PTR* iree_hal_task_queue_issue_t)( void* user_data, iree_task_scope_t* scope, iree_hal_task_queue_t* queue, - iree_host_size_t resource_count, iree_hal_resource_t* const* resources, iree_task_t* retire_task, iree_arena_allocator_t* arena, - iree_task_t** out_issue_task); + iree_hal_resource_set_t* resource_set, iree_task_t** out_issue_task); static iree_status_t iree_hal_task_queue_submit( iree_hal_task_queue_t* queue, iree_hal_semaphore_list_t wait_semaphores, @@ -613,18 +596,27 @@ static iree_status_t iree_hal_task_queue_submit( // arena which we will use to allocate all other commands. iree_hal_task_queue_retire_cmd_t* retire_cmd = NULL; IREE_RETURN_IF_ERROR(iree_hal_task_queue_retire_cmd_allocate( - &queue->scope, resource_count, resources, &signal_semaphores, - queue->small_block_pool, &retire_cmd)); + &queue->scope, &signal_semaphores, queue->small_block_pool, &retire_cmd)); + // If the caller provided any resources they wanted to retain we add them to + // the resource set for them. This is just a helper to avoid needing to pass + // too much state back to issue callbacks. + // // NOTE: if we fail from here on we must drop the retire_cmd arena. iree_status_t status = iree_ok_status(); + if (resource_count > 0) { + status = iree_hal_resource_set_insert(retire_cmd->resource_set, + resource_count, resources); + } // A fence we'll use to detect when the entire submission has completed. // TODO(benvanik): fold into the retire command. iree_task_fence_t* fence = NULL; - status = - iree_task_executor_acquire_fence(queue->executor, &queue->scope, &fence); - iree_task_set_completion_task(&retire_cmd->task.header, &fence->header); + if (iree_status_is_ok(status)) { + status = iree_task_executor_acquire_fence(queue->executor, &queue->scope, + &fence); + iree_task_set_completion_task(&retire_cmd->task.header, &fence->header); + } // Task to fork and wait for unsatisfied semaphore dependencies. // This is optional and only required if we have previous submissions still @@ -640,8 +632,8 @@ static iree_status_t iree_hal_task_queue_submit( // completed and the issued commands may complete in any order. iree_task_t* issue_cmd = NULL; if (iree_status_is_ok(status) && issue != NULL) { - status = issue(user_data, &queue->scope, queue, resource_count, resources, - &retire_cmd->task.header, &retire_cmd->arena, &issue_cmd); + status = issue(user_data, &queue->scope, queue, &retire_cmd->task.header, + &retire_cmd->arena, retire_cmd->resource_set, &issue_cmd); } // Last chance for failure - from here on we are submitting. @@ -715,9 +707,8 @@ iree_status_t iree_hal_task_queue_submit_commands( static iree_status_t iree_hal_task_queue_callback_cmd_allocate( void* user_data, iree_task_scope_t* scope, iree_hal_task_queue_t* queue, - iree_host_size_t resource_count, iree_hal_resource_t* const* resources, iree_task_t* retire_task, iree_arena_allocator_t* arena, - iree_task_t** out_issue_task) { + iree_hal_resource_set_t* resource_set, iree_task_t** out_issue_task) { iree_task_call_closure_t callback = *(iree_task_call_closure_t*)user_data; iree_task_call_t* cmd = NULL; diff --git a/runtime/src/iree/hal/drivers/vulkan/vulkan_device.cc b/runtime/src/iree/hal/drivers/vulkan/vulkan_device.cc index bc7d64ee58aa..a156d1ee36fe 100644 --- a/runtime/src/iree/hal/drivers/vulkan/vulkan_device.cc +++ b/runtime/src/iree/hal/drivers/vulkan/vulkan_device.cc @@ -1747,13 +1747,23 @@ static iree_status_t iree_hal_vulkan_device_queue_execute( iree_hal_command_buffer_t* command_buffer = command_buffers[i]; if (iree_hal_deferred_command_buffer_isa(command_buffers[i])) { iree_hal_command_buffer_t* translated_command_buffer = NULL; + iree_hal_buffer_binding_table_t binding_table = + binding_tables ? binding_tables[i] + : iree_hal_buffer_binding_table_empty(); status = iree_hal_vulkan_device_create_command_buffer( - base_device, iree_hal_command_buffer_mode(command_buffer), + base_device, + iree_hal_command_buffer_mode(command_buffer) | + IREE_HAL_COMMAND_BUFFER_MODE_ONE_SHOT | + // NOTE: we need to validate if a binding table is provided as the + // bindings were not known when it was originally recorded. + (iree_hal_buffer_binding_table_is_empty(binding_table) + ? IREE_HAL_COMMAND_BUFFER_MODE_UNVALIDATED + : 0), iree_hal_command_buffer_allowed_categories(command_buffer), queue_affinity, /*binding_capacity=*/0, &translated_command_buffer); if (iree_status_is_ok(status)) { status = iree_hal_deferred_command_buffer_apply( - command_buffer, translated_command_buffer, binding_tables[i]); + command_buffer, translated_command_buffer, binding_table); } translated_command_buffers[i] = translated_command_buffer; } else { @@ -1772,16 +1782,18 @@ static iree_status_t iree_hal_vulkan_device_queue_execute( status = queue->Submit(1, &batch); } - for (iree_host_size_t i = 0; i < command_buffer_count; ++i) { - iree_hal_command_buffer_release(translated_command_buffers[i]); - } - // HACK: we don't track async resource lifetimes so we have to block. if (iree_status_is_ok(status)) { status = iree_hal_semaphore_list_wait(signal_semaphore_list, iree_infinite_timeout()); } + // TODO(indirect-cmd): when async these need to be retained until the + // submission completes. + for (iree_host_size_t i = 0; i < command_buffer_count; ++i) { + iree_hal_command_buffer_release(translated_command_buffers[i]); + } + return status; }