Refactor CSV reader benchmarks with nvbench (#11678)

Closes #10941 This PR refactors the CSV reader benchmarks with nvbench and reduces the number of test cases by isolating data type, IO type, column selection, and row selection. Example output of the new benchmarks: <details> <summary>Benchmark results</summary> ## csv_read_data_type ### [0] Quadro RTX 8000 | data_type | Samples | CPU Time | Noise | GPU Time | Noise | bytes_per_second | peak_memory_usage | encoded_file_size | |-----------|---------|------------|-------|------------|-------|------------------|-------------------|-------------------| | INTEGRAL | 5x | 1.140 s | 0.09% | 1.140 s | 0.09% | 235553841 | 1.202 GiB | 668.564 MiB | | FLOAT | 5x | 1.262 s | 0.04% | 1.262 s | 0.04% | 212718321 | 1.041 GiB | 713.885 MiB | | DECIMAL | 5x | 272.787 ms | 0.03% | 272.784 ms | 0.03% | 984060406 | 396.279 MiB | 167.951 MiB | | TIMESTAMP | 7x | 1.681 s | 0.47% | 1.681 s | 0.47% | 159723724 | 2.281 GiB | 814.268 MiB | | DURATION | 7x | 2.121 s | 0.50% | 2.121 s | 0.50% | 126587514 | 2.588 GiB | 971.320 MiB | | STRING | 19x | 496.713 ms | 0.50% | 496.710 ms | 0.50% | 540426462 | 859.526 MiB | 277.082 MiB | ## csv_read_io ### [0] Quadro RTX 8000 | io | Samples | CPU Time | Noise | GPU Time | Noise | bytes_per_second | peak_memory_usage | encoded_file_size | |-------------|---------|----------|-------|----------|-------|------------------|-------------------|-------------------| | FILEPATH | 9x | 1.185 s | 0.49% | 1.185 s | 0.49% | 226466264 | 1.445 GiB | 618.876 MiB | | HOST_BUFFER | 5x | 1.170 s | 0.14% | 1.170 s | 0.14% | 229459856 | 1.445 GiB | 618.876 MiB | ## csv_read_column_selection ### [0] Quadro RTX 8000 | column_selection | row_selection | Samples | CPU Time | Noise | GPU Time | Noise | bytes_per_second | peak_memory_usage | encoded_file_size | |------------------|---------------|---------|----------|-------|----------|-------|------------------|-------------------|-------------------| | ALL | ALL | 5x | 1.246 s | 0.18% | 1.246 s | 0.18% | 215514992 | 1.582 GiB | 653.520 MiB | | ALTERNATE | ALL | 5x | 1.128 s | 0.08% | 1.128 s | 0.08% | 119009844 | 1.116 GiB | 648.908 MiB | | FIRST_HALF | ALL | 5x | 1.143 s | 0.07% | 1.143 s | 0.07% | 117443933 | 1.121 GiB | 653.520 MiB | | SECOND_HALF | ALL | 5x | 1.152 s | 0.16% | 1.152 s | 0.16% | 116478469 | 1.121 GiB | 653.520 MiB | ## csv_read_row_selection ### [0] Quadro RTX 8000 | column_selection | row_selection | num_chunks | Samples | CPU Time | Noise | GPU Time | Noise | bytes_per_second | peak_memory_usage | encoded_file_size | |------------------|---------------|------------|---------|----------|-------|----------|-------|------------------|-------------------|-------------------| | ALL | BYTE_RANGE | 1 | 5x | 1.244 s | 0.16% | 1.244 s | 0.16% | 215763257 | 1.582 GiB | 653.520 MiB | | ALL | BYTE_RANGE | 8 | 5x | 1.170 s | 0.04% | 1.170 s | 0.04% | 229339594 | 202.596 MiB | 653.520 MiB | | ALL | NROWS | 1 | 5x | 1.244 s | 0.12% | 1.244 s | 0.12% | 215808401 | 1.582 GiB | 653.520 MiB | | ALL | NROWS | 8 | 4x | 4.560 s | inf% | 4.560 s | inf% | 58870122 | 320.771 MiB | 653.520 MiB | | ALL | SKIPFOOTER | 1 | 5x | 1.245 s | 0.10% | 1.245 s | 0.10% | 215660012 | 1.582 GiB | 653.520 MiB | | ALL | SKIPFOOTER | 8 | 3x | 7.443 s | inf% | 7.443 s | inf% | 36065528 | 1.269 GiB | 653.520 MiB | </details> Authors: - Yunsong Wang (https://github.com/PointKernel) Approvers: - Nghia Truong (https://github.com/ttnghia) - Vukasin Milovanovic (https://github.com/vuule) URL: #11678
rapidsai · Sep 22, 2022 · d4f46fc · d4f46fc
1 parent a91853d
commit d4f46fc
Show file tree

Hide file tree

Showing 5 changed files with 251 additions and 175 deletions.
diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt
@@ -238,7 +238,7 @@ ConfigureNVBench(ORC_READER_NVBENCH io/orc/orc_reader_input.cpp io/orc/orc_reade
 
 # ##################################################################################################
 # * csv reader benchmark --------------------------------------------------------------------------
-ConfigureBench(CSV_READER_BENCH io/csv/csv_reader.cpp)
+ConfigureNVBench(CSV_READER_NVBENCH io/csv/csv_reader_input.cpp io/csv/csv_reader_options.cpp)
 
 # ##################################################################################################
 # * parquet writer benchmark ----------------------------------------------------------------------

diff --git a/cpp/benchmarks/io/csv/csv_reader.cpp b/cpp/benchmarks/io/csv/csv_reader.cpp
diff --git a/cpp/benchmarks/io/csv/csv_reader_input.cpp b/cpp/benchmarks/io/csv/csv_reader_input.cpp
@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2022, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <benchmarks/common/generate_input.hpp>
+#include <benchmarks/fixture/benchmark_fixture.hpp>
+#include <benchmarks/fixture/rmm_pool_raii.hpp>
+#include <benchmarks/io/cuio_common.hpp>
+#include <benchmarks/io/nvbench_helpers.hpp>
+
+#include <cudf/io/csv.hpp>
+#include <cudf/utilities/default_stream.hpp>
+
+#include <nvbench/nvbench.cuh>
+
+constexpr size_t data_size         = 256 << 20;
+constexpr cudf::size_type num_cols = 64;
+
+template <typename DataType>
+void csv_read_common(DataType const& data_types,
+                     cudf::io::io_type const& source_type,
+                     nvbench::state& state)
+{
+  auto const tbl =
+    create_random_table(cycle_dtypes(data_types, num_cols), table_size_bytes{data_size});
+  auto const view = tbl->view();
+
+  cuio_source_sink_pair source_sink(source_type);
+  cudf::io::csv_writer_options options =
+    cudf::io::csv_writer_options::builder(source_sink.make_sink_info(), view).include_header(true);
+
+  cudf::io::write_csv(options);
+
+  cudf::io::csv_reader_options const read_options =
+    cudf::io::csv_reader_options::builder(source_sink.make_source_info());
+
+  auto const mem_stats_logger = cudf::memory_stats_logger();  // init stats logger
+  state.set_cuda_stream(nvbench::make_cuda_stream_view(cudf::default_stream_value.value()));
+  state.exec(nvbench::exec_tag::sync | nvbench::exec_tag::timer,
+             [&](nvbench::launch& launch, auto& timer) {
+               try_drop_l3_cache();  // Drop L3 cache for accurate measurement
+
+               timer.start();
+               cudf::io::read_csv(read_options);
+               timer.stop();
+             });
+
+  auto const time = state.get_summary("nv/cold/time/gpu/mean").get_float64("value");
+  state.add_element_count(static_cast<double>(data_size) / time, "bytes_per_second");
+  state.add_buffer_size(
+    mem_stats_logger.peak_memory_usage(), "peak_memory_usage", "peak_memory_usage");
+  state.add_buffer_size(source_sink.size(), "encoded_file_size", "encoded_file_size");
+}
+
+template <data_type DataType>
+void BM_csv_read_input(nvbench::state& state, nvbench::type_list<nvbench::enum_type<DataType>>)
+{
+  cudf::rmm_pool_raii rmm_pool;
+
+  auto const d_type      = get_type_or_group(static_cast<int32_t>(DataType));
+  auto const source_type = io_type::FILEPATH;
+
+  csv_read_common(d_type, source_type, state);
+}
+
+template <cudf::io::io_type IO>
+void BM_csv_read_io(nvbench::state& state, nvbench::type_list<nvbench::enum_type<IO>>)
+{
+  cudf::rmm_pool_raii rmm_pool;
+
+  auto const d_type      = get_type_or_group({static_cast<int32_t>(data_type::INTEGRAL),
+                                         static_cast<int32_t>(data_type::FLOAT),
+                                         static_cast<int32_t>(data_type::DECIMAL),
+                                         static_cast<int32_t>(data_type::TIMESTAMP),
+                                         static_cast<int32_t>(data_type::DURATION),
+                                         static_cast<int32_t>(data_type::STRING)});
+  auto const source_type = IO;
+
+  csv_read_common(d_type, source_type, state);
+}
+
+using d_type_list = nvbench::enum_type_list<data_type::INTEGRAL,
+                                            data_type::FLOAT,
+                                            data_type::DECIMAL,
+                                            data_type::TIMESTAMP,
+                                            data_type::DURATION,
+                                            data_type::STRING>;
+
+using io_list =
+  nvbench::enum_type_list<cudf::io::io_type::FILEPATH, cudf::io::io_type::HOST_BUFFER>;
+
+NVBENCH_BENCH_TYPES(BM_csv_read_input, NVBENCH_TYPE_AXES(d_type_list))
+  .set_name("csv_read_data_type")
+  .set_type_axes_names({"data_type"})
+  .set_min_samples(4);
+
+NVBENCH_BENCH_TYPES(BM_csv_read_io, NVBENCH_TYPE_AXES(io_list))
+  .set_name("csv_read_io")
+  .set_type_axes_names({"io"})
+  .set_min_samples(4);