Skip to content

Commit

Permalink
ARROW-7749: [C++] Link more tests together
Browse files Browse the repository at this point in the history
This should make unity builds slightly faster, by allowing CMake to coalesce
more source files and save on header file parsing.

Closes #6493 from pitrou/ARROW-7749-link-tests-together and squashes the following commits:

33cbae2 <Antoine Pitrou> ARROW-7749:  Link more tests together

Authored-by: Antoine Pitrou <antoine@python.org>
Signed-off-by: Wes McKinney <wesm+git@apache.org>
  • Loading branch information
pitrou authored and wesm committed Mar 2, 2020
1 parent e3acbab commit 467129e
Show file tree
Hide file tree
Showing 13 changed files with 140 additions and 129 deletions.
18 changes: 12 additions & 6 deletions cpp/src/arrow/CMakeLists.txt
Expand Up @@ -305,6 +305,8 @@ if(ARROW_CSV)
csv/options.cc
csv/parser.cc
csv/reader.cc)

list(APPEND ARROW_TESTING_SRCS csv/test_common.cc)
endif()

if(ARROW_COMPUTE)
Expand Down Expand Up @@ -515,8 +517,13 @@ if(ARROW_IPC)
add_arrow_test(extension_type_test)
endif()

add_arrow_test(memory_pool_test)
add_arrow_test(pretty_print_test)
add_arrow_test(misc_test
SOURCES
memory_pool_test.cc
result_test.cc
pretty_print_test.cc
status_test.cc)

add_arrow_test(public_api_test)

set_source_files_properties(public_api_test.cc
Expand All @@ -526,12 +533,11 @@ set_source_files_properties(public_api_test.cc
SKIP_UNITY_BUILD_INCLUSION
ON)

add_arrow_test(result_test)
add_arrow_test(scalar_test)
add_arrow_test(status_test)
add_arrow_test(type_test)
add_arrow_test(table_test)
add_arrow_test(table_builder_test)

add_arrow_test(table_test SOURCES table_test.cc table_builder_test.cc)

add_arrow_test(tensor_test)
add_arrow_test(sparse_tensor_test)

Expand Down
10 changes: 6 additions & 4 deletions cpp/src/arrow/csv/CMakeLists.txt
Expand Up @@ -15,10 +15,12 @@
# specific language governing permissions and limitations
# under the License.

add_arrow_test(chunker_test PREFIX "arrow-csv")
add_arrow_test(column_builder_test PREFIX "arrow-csv")
add_arrow_test(converter_test PREFIX "arrow-csv")
add_arrow_test(parser_test PREFIX "arrow-csv")
add_arrow_test(csv-test
SOURCES
chunker_test.cc
column_builder_test.cc
converter_test.cc
parser_test.cc)

add_arrow_benchmark(converter_benchmark PREFIX "arrow-csv")
add_arrow_benchmark(parser_benchmark PREFIX "arrow-csv")
Expand Down
1 change: 1 addition & 0 deletions cpp/src/arrow/csv/column_builder_test.cc
Expand Up @@ -26,6 +26,7 @@
#include "arrow/csv/test_common.h"
#include "arrow/memory_pool.h"
#include "arrow/table.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/testing/util.h"
#include "arrow/type.h"
#include "arrow/util/checked_cast.h"
Expand Down
65 changes: 65 additions & 0 deletions cpp/src/arrow/csv/test_common.cc
@@ -0,0 +1,65 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include "arrow/csv/test_common.h"
#include "arrow/testing/gtest_util.h"

namespace arrow {
namespace csv {

std::string MakeCSVData(std::vector<std::string> lines) {
std::string s;
for (const auto& line : lines) {
s += line;
}
return s;
}

void MakeCSVParser(std::vector<std::string> lines, ParseOptions options, int32_t num_cols,
std::shared_ptr<BlockParser>* out) {
auto csv = MakeCSVData(lines);
auto parser = std::make_shared<BlockParser>(options, num_cols);
uint32_t out_size;
ASSERT_OK(parser->Parse(util::string_view(csv), &out_size));
ASSERT_EQ(out_size, csv.size()) << "trailing CSV data not parsed";
*out = parser;
}

void MakeCSVParser(std::vector<std::string> lines, ParseOptions options,
std::shared_ptr<BlockParser>* out) {
return MakeCSVParser(lines, options, -1, out);
}

void MakeCSVParser(std::vector<std::string> lines, std::shared_ptr<BlockParser>* out) {
MakeCSVParser(lines, ParseOptions::Defaults(), out);
}

void MakeColumnParser(std::vector<std::string> items, std::shared_ptr<BlockParser>* out) {
auto options = ParseOptions::Defaults();
// Need this to test for null (empty) values
options.ignore_empty_lines = false;
std::vector<std::string> lines;
for (const auto& item : items) {
lines.push_back(item + '\n');
}
MakeCSVParser(lines, options, 1, out);
ASSERT_EQ((*out)->num_cols(), 1) << "Should have seen only 1 CSV column";
ASSERT_EQ((*out)->num_rows(), items.size());
}

} // namespace csv
} // namespace arrow
50 changes: 12 additions & 38 deletions cpp/src/arrow/csv/test_common.h
Expand Up @@ -15,62 +15,36 @@
// specific language governing permissions and limitations
// under the License.

#ifndef ARROW_CSV_TEST_COMMON_H
#define ARROW_CSV_TEST_COMMON_H
#pragma once

#include <memory>
#include <string>
#include <vector>

#include "arrow/csv/parser.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/util/visibility.h"

namespace arrow {
namespace csv {

std::string MakeCSVData(std::vector<std::string> lines) {
std::string s;
for (const auto& line : lines) {
s += line;
}
return s;
}
ARROW_EXPORT
std::string MakeCSVData(std::vector<std::string> lines);

// Make a BlockParser from a vector of lines representing a CSV file
ARROW_EXPORT
void MakeCSVParser(std::vector<std::string> lines, ParseOptions options, int32_t num_cols,
std::shared_ptr<BlockParser>* out) {
auto csv = MakeCSVData(lines);
auto parser = std::make_shared<BlockParser>(options, num_cols);
uint32_t out_size;
ASSERT_OK(parser->Parse(util::string_view(csv), &out_size));
ASSERT_EQ(out_size, csv.size()) << "trailing CSV data not parsed";
*out = parser;
}
std::shared_ptr<BlockParser>* out);

ARROW_EXPORT
void MakeCSVParser(std::vector<std::string> lines, ParseOptions options,
std::shared_ptr<BlockParser>* out) {
return MakeCSVParser(lines, options, -1, out);
}
std::shared_ptr<BlockParser>* out);

void MakeCSVParser(std::vector<std::string> lines, std::shared_ptr<BlockParser>* out) {
MakeCSVParser(lines, ParseOptions::Defaults(), out);
}
ARROW_EXPORT
void MakeCSVParser(std::vector<std::string> lines, std::shared_ptr<BlockParser>* out);

// Make a BlockParser from a vector of strings representing a single CSV column
void MakeColumnParser(std::vector<std::string> items, std::shared_ptr<BlockParser>* out) {
auto options = ParseOptions::Defaults();
// Need this to test for null (empty) values
options.ignore_empty_lines = false;
std::vector<std::string> lines;
for (const auto& item : items) {
lines.push_back(item + '\n');
}
MakeCSVParser(lines, options, 1, out);
ASSERT_EQ((*out)->num_cols(), 1) << "Should have seen only 1 CSV column";
ASSERT_EQ((*out)->num_rows(), items.size());
}
ARROW_EXPORT
void MakeColumnParser(std::vector<std::string> items, std::shared_ptr<BlockParser>* out);

} // namespace csv
} // namespace arrow

#endif // ARROW_CSV_TEST_COMMON_H
8 changes: 5 additions & 3 deletions cpp/src/arrow/filesystem/CMakeLists.txt
Expand Up @@ -21,9 +21,11 @@ arrow_install_all_headers("arrow/filesystem")
# pkg-config support
arrow_add_pkg_config("arrow-filesystem")

add_arrow_test(filesystem_test)
add_arrow_test(localfs_test)
add_arrow_test(path_forest_test)
add_arrow_test(filesystem-test
SOURCES
filesystem_test.cc
localfs_test.cc
path_forest_test.cc)

if(ARROW_S3)
add_arrow_test(s3fs_test)
Expand Down
17 changes: 9 additions & 8 deletions cpp/src/arrow/memory_pool_test.cc
Expand Up @@ -107,26 +107,27 @@ TEST(DefaultMemoryPoolDeathTest, MaxMemory) {
#endif // ARROW_VALGRIND

TEST(LoggingMemoryPool, Logging) {
MemoryPool* pool = default_memory_pool();
auto pool = MemoryPool::CreateDefault();

LoggingMemoryPool lp(pool);
LoggingMemoryPool lp(pool.get());

uint8_t* data;
ASSERT_OK(pool->Allocate(100, &data));
ASSERT_OK(lp.Allocate(100, &data));

uint8_t* data2;
ASSERT_OK(pool->Allocate(100, &data2));
ASSERT_OK(lp.Allocate(100, &data2));

pool->Free(data, 100);
pool->Free(data2, 100);
lp.Free(data, 100);
lp.Free(data2, 100);

ASSERT_EQ(200, lp.max_memory());
ASSERT_EQ(200, pool->max_memory());
}

TEST(ProxyMemoryPool, Logging) {
MemoryPool* pool = default_memory_pool();
auto pool = MemoryPool::CreateDefault();

ProxyMemoryPool pp(pool);
ProxyMemoryPool pp(pool.get());

uint8_t* data;
ASSERT_OK(pool->Allocate(100, &data));
Expand Down
7 changes: 7 additions & 0 deletions cpp/src/arrow/testing/gtest_util.cc
Expand Up @@ -25,6 +25,7 @@
#endif

#include <algorithm>
#include <chrono>
#include <cstdint>
#include <cstdlib>
#include <iostream>
Expand All @@ -34,6 +35,7 @@
#include <sstream>
#include <stdexcept>
#include <string>
#include <thread>
#include <vector>

#include "arrow/array.h"
Expand Down Expand Up @@ -371,6 +373,11 @@ void TestInitialized(const Array& array) {
}
}

void SleepFor(double seconds) {
std::this_thread::sleep_for(
std::chrono::nanoseconds(static_cast<int64_t>(seconds * 1e9)));
}

///////////////////////////////////////////////////////////////////////////
// Extension types

Expand Down
3 changes: 3 additions & 0 deletions cpp/src/arrow/testing/gtest_util.h
Expand Up @@ -405,6 +405,9 @@ void AssertSortedEquals(std::vector<T> u, std::vector<T> v) {
ASSERT_EQ(u, v);
}

ARROW_EXPORT
void SleepFor(double seconds);

// A RAII-style object that switches to a new locale, and switches back
// to the old locale when going out of scope. Doesn't do anything if the
// new locale doesn't exist on the local machine.
Expand Down
15 changes: 7 additions & 8 deletions cpp/src/arrow/util/CMakeLists.txt
Expand Up @@ -29,29 +29,28 @@ arrow_install_all_headers("arrow/util")
add_arrow_test(utility-test
SOURCES
align_util_test.cc
bit_util_test.cc
checked_cast_test.cc
compression_test.cc
decimal_test.cc
formatting_util_test.cc
key_value_metadata_test.cc
hashing_test.cc
int_util_test.cc
io_util_test.cc
iterator_test.cc
logging_test.cc
parsing_util_test.cc
range_test.cc
rle_encoding_test.cc
stl_util_test.cc
string_test.cc
time_test.cc
trie_test.cc
uri_test.cc
utf8_util_test.cc)

add_arrow_test(bit_util_test)
add_arrow_test(compression_test)
add_arrow_test(decimal_test)
add_arrow_test(logging_test)
add_arrow_test(rle_encoding_test)
add_arrow_test(task_group_test)
add_arrow_test(thread_pool_test)
add_arrow_test(uri_test)
add_arrow_test(threading-utility-test SOURCES task_group_test thread_pool_test)

add_arrow_benchmark(bit_util_benchmark)
add_arrow_benchmark(compression_benchmark)
Expand Down
39 changes: 0 additions & 39 deletions cpp/src/arrow/util/logging_test.cc
Expand Up @@ -68,45 +68,6 @@ TEST(PrintLogTest, LogTestWithInit) {
ArrowLog::ShutDownArrowLog();
}

#ifdef ARROW_WITH_TIMING_TESTS

// This test will output large amount of logs to stderr, should be disabled in travis.
TEST(LogPerfTest, PerfTest) {
ArrowLog::StartArrowLog("/fake/path/to/appdire/LogPerfTest", ArrowLogLevel::ARROW_ERROR,
"/tmp/");
int rounds = 10000;

int64_t start_time = current_time_ms();
for (int i = 0; i < rounds; ++i) {
ARROW_LOG(DEBUG) << "This is the "
<< "ARROW_DEBUG message";
}
int64_t elapsed = current_time_ms() - start_time;
std::cout << "Testing DEBUG log for " << rounds << " rounds takes " << elapsed << " ms."
<< std::endl;

start_time = current_time_ms();
for (int i = 0; i < rounds; ++i) {
ARROW_LOG(ERROR) << "This is the "
<< "RARROW_ERROR message";
}
elapsed = current_time_ms() - start_time;
std::cout << "Testing ARROW_ERROR log for " << rounds << " rounds takes " << elapsed
<< " ms." << std::endl;

start_time = current_time_ms();
for (int i = 0; i < rounds; ++i) {
ARROW_CHECK(i >= 0) << "This is a ARROW_CHECK "
<< "message but it won't show up";
}
elapsed = current_time_ms() - start_time;
std::cout << "Testing ARROW_CHECK(true) for " << rounds << " rounds takes " << elapsed
<< " ms." << std::endl;
ArrowLog::ShutDownArrowLog();
}

#endif

} // namespace util

TEST(DcheckMacros, DoNotEvaluateReleaseMode) {
Expand Down

0 comments on commit 467129e

Please sign in to comment.