Automate include grouping using clang-format

.clang-format

@@ -71,8 +71,20 @@ ForEachMacros:
   - foreach
   - Q_FOREACH
   - BOOST_FOREACH
-IncludeBlocks: Preserve
-IncludeIsMainRegex: '([-_](test|unittest))?$'
+IncludeBlocks: Regroup
+IncludeCategories:
+  - Regex:           '^"' # quoted includes
+    Priority:        1
+  - Regex:           '^<rmm/' # RMM includes
+    Priority:        2
+  - Regex:           '^<(thrust|cub|cuda)/' # CCCL includes
+    Priority:        3
+  - Regex:           '^<(cooperative_groups|cuco|cuda.h|cuda_runtime|device_types|math_constants|nvtx3)' # CUDA includes
+    Priority:        3
+  - Regex:           '^<.*\..*' # other system includes (e.g. with a '.')
+    Priority:        4
+  - Regex:           '^<[^.]+' # STL includes (no '.')
+    Priority:        5
 IndentCaseLabels: true
 IndentPPDirectives: None
 IndentWidth:     2

benchmarks/device_uvector/device_uvector_bench.cu

@@ -26,13 +26,12 @@
 #include <rmm/mr/device/per_device_resource.hpp>
 #include <rmm/mr/device/pool_memory_resource.hpp>
 
-#include <benchmark/benchmark.h>
-
 #include <cuda_runtime_api.h>
-
 #include <thrust/device_vector.h>
 #include <thrust/memory.h>
 
+#include <benchmark/benchmark.h>
+
 #include <cstdio>
 #include <type_traits>
 

benchmarks/multi_stream_allocations/multi_stream_allocations_bench.cu

@@ -14,8 +14,6 @@
  * limitations under the License.
  */
 
-#include <benchmarks/utilities/cxxopts.hpp>
-
 #include <rmm/cuda_device.hpp>
 #include <rmm/cuda_stream.hpp>
 #include <rmm/cuda_stream_pool.hpp>
@@ -31,6 +29,7 @@
 #include <cuda_runtime_api.h>
 
 #include <benchmark/benchmark.h>
+#include <benchmarks/utilities/cxxopts.hpp>
 
 #include <cstddef>
 

benchmarks/random_allocations/random_allocations.cpp

@@ -14,8 +14,6 @@
  * limitations under the License.
  */
 
-#include <benchmarks/utilities/cxxopts.hpp>
-
 #include <rmm/cuda_device.hpp>
 #include <rmm/mr/device/arena_memory_resource.hpp>
 #include <rmm/mr/device/binning_memory_resource.hpp>
@@ -26,6 +24,7 @@
 #include <rmm/mr/device/pool_memory_resource.hpp>
 
 #include <benchmark/benchmark.h>
+#include <benchmarks/utilities/cxxopts.hpp>
 
 #include <array>
 #include <cstddef>

benchmarks/replay/replay.cpp

@@ -14,10 +14,6 @@
  * limitations under the License.
  */
 
-#include <benchmarks/utilities/cxxopts.hpp>
-#include <benchmarks/utilities/log_parser.hpp>
-#include <benchmarks/utilities/simulated_memory_resource.hpp>
-
 #include <rmm/cuda_stream_view.hpp>
 #include <rmm/detail/error.hpp>
 #include <rmm/mr/device/arena_memory_resource.hpp>
@@ -33,7 +29,9 @@
 #include <thrust/reduce.h>
 
 #include <benchmark/benchmark.h>
-
+#include <benchmarks/utilities/cxxopts.hpp>
+#include <benchmarks/utilities/log_parser.hpp>
+#include <benchmarks/utilities/simulated_memory_resource.hpp>
 #include <spdlog/common.h>
 
 #include <atomic>

benchmarks/synchronization/synchronization.hpp

@@ -61,9 +61,10 @@
 #include <rmm/cuda_stream_view.hpp>
 
 // Google Benchmark library
-#include <benchmark/benchmark.h>
 #include <cuda_runtime_api.h>
 
+#include <benchmark/benchmark.h>
+
 class cuda_event_timer {
  public:
   /**

benchmarks/utilities/log_parser.hpp

@@ -16,12 +16,12 @@
 
 #pragma once
 
-#include <chrono>
+#include "rapidcsv.h"
+
 #include <rmm/detail/error.hpp>
 #include <rmm/mr/device/device_memory_resource.hpp>
 
-#include "rapidcsv.h"
-
+#include <chrono>
 #include <cstdint>
 #include <iomanip>
 #include <limits>

include/rmm/cuda_stream.hpp

@@ -16,13 +16,13 @@
 
 #pragma once
 
-#include <functional>
 #include <rmm/cuda_stream_view.hpp>
 #include <rmm/detail/error.hpp>
 #include <rmm/detail/logging_assert.hpp>
 
 #include <cuda_runtime_api.h>
 
+#include <functional>
 #include <memory>
 
 namespace rmm {

include/rmm/cuda_stream_view.hpp

@@ -18,9 +18,8 @@
 
 #include <rmm/detail/error.hpp>
 
-#include <cuda_runtime_api.h>
-
 #include <cuda/stream_ref>
+#include <cuda_runtime_api.h>
 
 #include <atomic>
 #include <cstddef>

include/rmm/detail/logging_assert.hpp

@@ -20,9 +20,10 @@
 // logger can be extremely expensive to compile, so we want to avoid including
 // it.
 #if !defined(NDEBUG)
-#include <cassert>
 #include <rmm/detail/error.hpp>
 #include <rmm/logger.hpp>
+
+#include <cassert>
 #endif
 
 /**

include/rmm/device_buffer.hpp

@@ -21,15 +21,14 @@
 #include <rmm/mr/device/per_device_resource.hpp>
 #include <rmm/resource_ref.hpp>
 
+#include <cuda/memory_resource>
 #include <cuda_runtime_api.h>
 
 #include <cassert>
 #include <cstddef>
 #include <stdexcept>
 #include <utility>
 
-#include <cuda/memory_resource>
-
 namespace rmm {
 /**
  * @addtogroup data_containers

include/rmm/device_uvector.hpp

@@ -23,11 +23,11 @@
 #include <rmm/mr/device/per_device_resource.hpp>
 #include <rmm/resource_ref.hpp>
 
+#include <cuda/memory_resource>
+
 #include <cstddef>
 #include <vector>
 
-#include <cuda/memory_resource>
-
 namespace rmm {
 /**
  * @addtogroup data_containers

include/rmm/device_vector.hpp

@@ -16,9 +16,9 @@
 
 #pragma once
 
+#include <rmm/detail/thrust_namespace.h>
 #include <rmm/mr/device/thrust_allocator_adaptor.hpp>
 
-#include <rmm/detail/thrust_namespace.h>
 #include <thrust/device_vector.h>
 
 namespace rmm {

include/rmm/exec_policy.hpp

@@ -22,10 +22,10 @@
 #pragma once
 
 #include <rmm/cuda_stream_view.hpp>
+#include <rmm/detail/thrust_namespace.h>
 #include <rmm/mr/device/thrust_allocator_adaptor.hpp>
 #include <rmm/resource_ref.hpp>
 
-#include <rmm/detail/thrust_namespace.h>
 #include <thrust/system/cuda/execution_policy.h>
 #include <thrust/version.h>
 

include/rmm/mr/device/cuda_async_memory_resource.hpp

@@ -19,13 +19,12 @@
 #include <rmm/cuda_stream_view.hpp>
 #include <rmm/detail/dynamic_load_runtime.hpp>
 #include <rmm/detail/error.hpp>
+#include <rmm/detail/thrust_namespace.h>
 #include <rmm/mr/device/cuda_async_view_memory_resource.hpp>
 #include <rmm/mr/device/device_memory_resource.hpp>
 
-#include <rmm/detail/thrust_namespace.h>
-#include <thrust/optional.h>
-
 #include <cuda_runtime_api.h>
+#include <thrust/optional.h>
 
 #include <cstddef>
 #include <limits>

include/rmm/mr/device/cuda_async_view_memory_resource.hpp

@@ -19,12 +19,11 @@
 #include <rmm/cuda_stream_view.hpp>
 #include <rmm/detail/dynamic_load_runtime.hpp>
 #include <rmm/detail/error.hpp>
-#include <rmm/mr/device/device_memory_resource.hpp>
-
 #include <rmm/detail/thrust_namespace.h>
-#include <thrust/optional.h>
+#include <rmm/mr/device/device_memory_resource.hpp>
 
 #include <cuda_runtime_api.h>
+#include <thrust/optional.h>
 
 #include <cstddef>
 #include <limits>

include/rmm/mr/device/cuda_memory_resource.hpp

@@ -15,10 +15,9 @@
  */
 #pragma once
 
-#include <rmm/mr/device/device_memory_resource.hpp>
-
 #include <rmm/cuda_stream_view.hpp>
 #include <rmm/detail/error.hpp>
+#include <rmm/mr/device/device_memory_resource.hpp>
 
 #include <cstddef>
 

include/rmm/mr/device/fixed_size_memory_resource.hpp

@@ -19,15 +19,14 @@
 #include <rmm/cuda_stream_view.hpp>
 #include <rmm/detail/error.hpp>
 #include <rmm/detail/logging_assert.hpp>
+#include <rmm/detail/thrust_namespace.h>
 #include <rmm/mr/device/detail/fixed_size_free_list.hpp>
 #include <rmm/mr/device/detail/stream_ordered_memory_resource.hpp>
 
-#include <rmm/detail/thrust_namespace.h>
+#include <cuda_runtime_api.h>
 #include <thrust/iterator/counting_iterator.h>
 #include <thrust/iterator/transform_iterator.h>
 
-#include <cuda_runtime_api.h>
-
 #include <algorithm>
 #include <cstddef>
 #include <list>

include/rmm/mr/device/managed_memory_resource.hpp

@@ -15,10 +15,9 @@
  */
 #pragma once
 
-#include <rmm/mr/device/device_memory_resource.hpp>
-
 #include <rmm/cuda_stream_view.hpp>
 #include <rmm/detail/error.hpp>
+#include <rmm/mr/device/device_memory_resource.hpp>
 
 #include <cstddef>
 

include/rmm/mr/device/pool_memory_resource.hpp

@@ -19,20 +19,19 @@
 #include <rmm/cuda_stream_view.hpp>
 #include <rmm/detail/error.hpp>
 #include <rmm/detail/logging_assert.hpp>
+#include <rmm/detail/thrust_namespace.h>
 #include <rmm/logger.hpp>
 #include <rmm/mr/device/detail/coalescing_free_list.hpp>
 #include <rmm/mr/device/detail/stream_ordered_memory_resource.hpp>
 #include <rmm/mr/device/device_memory_resource.hpp>
 
-#include <rmm/detail/thrust_namespace.h>
+#include <cuda_runtime_api.h>
 #include <thrust/iterator/counting_iterator.h>
 #include <thrust/iterator/transform_iterator.h>
 #include <thrust/optional.h>
 
 #include <fmt/core.h>
 
-#include <cuda_runtime_api.h>
-
 #include <algorithm>
 #include <cstddef>
 #include <iostream>

include/rmm/mr/device/thrust_allocator_adaptor.hpp

@@ -16,16 +16,15 @@
 
 #pragma once
 
+#include <rmm/detail/thrust_namespace.h>
 #include <rmm/mr/device/per_device_resource.hpp>
 #include <rmm/resource_ref.hpp>
 
-#include <rmm/detail/thrust_namespace.h>
+#include <cuda/memory_resource>
 #include <thrust/device_malloc_allocator.h>
 #include <thrust/device_ptr.h>
 #include <thrust/memory.h>
 
-#include <cuda/memory_resource>
-
 namespace rmm::mr {
 /**
  * @addtogroup device_resource_adaptors

include/rmm/mr/host/new_delete_resource.hpp

@@ -15,10 +15,9 @@
  */
 #pragma once
 
-#include <rmm/mr/host/host_memory_resource.hpp>
-
 #include <rmm/aligned.hpp>
 #include <rmm/detail/aligned.hpp>
+#include <rmm/mr/host/host_memory_resource.hpp>
 
 #include <cstddef>
 #include <utility>

include/rmm/mr/pinned_host_memory_resource.hpp

@@ -21,7 +21,6 @@
 
 #include <cuda/memory_resource>
 #include <cuda/stream_ref>
-
 #include <cuda_runtime_api.h>
 
 #include <cstddef>

include/rmm/thrust_rmm_allocator.h

@@ -17,10 +17,10 @@
 #pragma once
 
 #include <rmm/cuda_stream_view.hpp>
+#include <rmm/detail/thrust_namespace.h>
 #include <rmm/device_vector.hpp>
 #include <rmm/mr/device/thrust_allocator_adaptor.hpp>
 
-#include <rmm/detail/thrust_namespace.h>
 #include <thrust/execution_policy.h>
 
 namespace rmm {

python/rmm/_lib/_torch_allocator.cpp

@@ -14,12 +14,12 @@
  * limitations under the License.
  */
 
-#include <cuda_runtime_api.h>
-
 #include <rmm/cuda_device.hpp>
 #include <rmm/cuda_stream_view.hpp>
 #include <rmm/mr/device/per_device_resource.hpp>
 
+#include <cuda_runtime_api.h>
+
 // These signatures must match those required by CUDAPluggableAllocator in
 // github.com/pytorch/pytorch/blob/main/torch/csrc/cuda/CUDAPluggableAllocator.h
 // Since the loading is done at runtime via dlopen, no error checking

tests/cuda_stream_pool_tests.cpp

@@ -18,10 +18,10 @@
 #include <rmm/detail/error.hpp>
 #include <rmm/device_uvector.hpp>
 
-#include <gtest/gtest.h>
-
 #include <cuda_runtime_api.h>
 
+#include <gtest/gtest.h>
+
 struct CudaStreamPoolTest : public ::testing::Test {
   rmm::cuda_stream_pool pool{};
 };