Skip to content

Commit

Permalink
Integrate filesystem unit tests into a single test. (#4635)
Browse files Browse the repository at this point in the history 
Begin work on integrating _all_ `unit-<filesystem>.cc` tests into a
_single_ `unit-vfs.cc` test. Eventually, the individual tests will be
removed entirely and the single test will exist solely as a _unit_ test
in `tiledb/sm/filesystem/test`. This work begins that transition.

---
TYPE: NO_HISTORY
DESC: Integrate filesystem unit tests into a single test.
  • Loading branch information
@bekadavis9
bekadavis9 committed Jan 25, 2024
1 parent 0df8863 commit ca11ed2
Show file tree
Hide file tree
Showing 5 changed files with 298 additions and 616 deletions.
137 changes: 1 addition & 136 deletions test/src/unit-azure.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
*
* The MIT License
*
* @copyright Copyright (c) 2017-2021 TileDB, Inc.
* @copyright Copyright (c) 2017-2023 TileDB, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
Expand Down Expand Up @@ -131,146 +131,11 @@ std::string AzureFx::random_container_name(const std::string& prefix) {
return ss.str();
}

TEST_CASE_METHOD(AzureFx, "Test Azure filesystem, file management", "[azure]") {
Config config;
REQUIRE(config.set("vfs.azure.use_block_list_upload", "true").ok());

auto settings =
GENERATE(from_range(test_settings.begin(), test_settings.end()));
init_azure(std::move(config), settings);

/* Create the following file hierarchy:
*
* TEST_DIR/dir/subdir/file1
* TEST_DIR/dir/subdir/file2
* TEST_DIR/dir/file3
* TEST_DIR/file4
* TEST_DIR/file5
*/
auto dir = TEST_DIR + "dir/";
auto dir2 = TEST_DIR + "dir2/";
auto subdir = dir + "subdir/";
auto file1 = subdir + "file1";
auto file2 = subdir + "file2";
auto file3 = dir + "file3";
auto file4 = TEST_DIR + "file4";
auto file5 = TEST_DIR + "file5";
auto file6 = TEST_DIR + "file6";

// Check that container is empty
bool is_empty;
REQUIRE(azure_.is_empty_container(AZURE_CONTAINER, &is_empty).ok());
REQUIRE(is_empty);

// Continue building the hierarchy
bool is_blob = false;
REQUIRE(azure_.touch(URI(file1)).ok());
REQUIRE(azure_.is_blob(URI(file1), &is_blob).ok());
REQUIRE(is_blob);
REQUIRE(azure_.touch(URI(file2)).ok());
REQUIRE(azure_.is_blob(URI(file2), &is_blob).ok());
REQUIRE(is_blob);
REQUIRE(azure_.touch(URI(file3)).ok());
REQUIRE(azure_.is_blob(URI(file3), &is_blob).ok());
REQUIRE(is_blob);
REQUIRE(azure_.touch(URI(file4)).ok());
REQUIRE(azure_.is_blob(URI(file4), &is_blob).ok());
REQUIRE(is_blob);
REQUIRE(azure_.touch(URI(file5)).ok());
REQUIRE(azure_.is_blob(URI(file5), &is_blob).ok());
REQUIRE(is_blob);

// Check that container is not empty
REQUIRE(azure_.is_empty_container(AZURE_CONTAINER, &is_empty).ok());
REQUIRE(!is_empty);

// Check invalid file
REQUIRE(azure_.is_blob(URI(TEST_DIR + "foo"), &is_blob).ok());
REQUIRE(!is_blob);

// List with prefix
std::vector<std::string> paths;
REQUIRE(azure_.ls(URI(TEST_DIR), &paths).ok());
REQUIRE(paths.size() == 3);
paths.clear();
REQUIRE(azure_.ls(URI(dir), &paths).ok());
REQUIRE(paths.size() == 2);
paths.clear();
REQUIRE(azure_.ls(URI(subdir), &paths).ok());
REQUIRE(paths.size() == 2);
paths.clear();
REQUIRE(azure_.ls(AZURE_CONTAINER, &paths, "").ok()); // No delimiter
REQUIRE(paths.size() == 5);

// Check if a directory exists
bool is_dir = false;
REQUIRE(azure_.is_dir(URI(file1), &is_dir).ok());
REQUIRE(!is_dir); // Not a dir
REQUIRE(azure_.is_dir(URI(file4), &is_dir).ok());
REQUIRE(!is_dir); // Not a dir
REQUIRE(azure_.is_dir(URI(dir), &is_dir).ok());
REQUIRE(is_dir); // This is viewed as a dir
REQUIRE(azure_.is_dir(URI(TEST_DIR + "dir"), &is_dir).ok());
REQUIRE(is_dir); // This is viewed as a dir

// ls_with_sizes
std::string s = "abcdef";
CHECK(azure_.write(URI(file3), s.data(), s.size()).ok());
REQUIRE(azure_.flush_blob(URI(file3)).ok());

auto&& [status, rv] = azure_.ls_with_sizes(URI(dir));
auto children = *rv;
REQUIRE(status.ok());

REQUIRE(children.size() == 2);
CHECK(children[0].path().native() == file3);
CHECK(children[1].path().native() == subdir.substr(0, subdir.size() - 1));

CHECK(children[0].file_size() == s.size());
// Directories don't get a size
CHECK(children[1].file_size() == 0);

// Move file
REQUIRE(azure_.move_object(URI(file5), URI(file6)).ok());
REQUIRE(azure_.is_blob(URI(file5), &is_blob).ok());
REQUIRE(!is_blob);
REQUIRE(azure_.is_blob(URI(file6), &is_blob).ok());
REQUIRE(is_blob);
paths.clear();
REQUIRE(azure_.ls(AZURE_CONTAINER, &paths, "").ok()); // No delimiter
REQUIRE(paths.size() == 5);

// Move directory
REQUIRE(azure_.move_dir(URI(dir), URI(dir2)).ok());
REQUIRE(azure_.is_dir(URI(dir), &is_dir).ok());
REQUIRE(!is_dir);
REQUIRE(azure_.is_dir(URI(dir2), &is_dir).ok());
REQUIRE(is_dir);
paths.clear();
REQUIRE(azure_.ls(AZURE_CONTAINER, &paths, "").ok()); // No delimiter
REQUIRE(paths.size() == 5);

// Remove files
REQUIRE(azure_.remove_blob(URI(file4)).ok());
REQUIRE(azure_.is_blob(URI(file4), &is_blob).ok());
REQUIRE(!is_blob);

// Remove directories
REQUIRE(azure_.remove_dir(URI(dir2)).ok());
REQUIRE(azure_.is_blob(URI(file1), &is_blob).ok());
REQUIRE(!is_blob);
REQUIRE(azure_.is_blob(URI(file2), &is_blob).ok());
REQUIRE(!is_blob);
REQUIRE(azure_.is_blob(URI(file3), &is_blob).ok());
REQUIRE(!is_blob);
}

TEST_CASE_METHOD(
AzureFx, "Test Azure filesystem, file I/O", "[azure][multipart]") {
Config config;
const uint64_t max_parallel_ops = 2;
const uint64_t block_list_block_size = 4 * 1024 * 1024;
REQUIRE(config.set("vfs.azure.use_block_list_upload", "true").ok());
REQUIRE(
config.set("vfs.azure.max_parallel_ops", std::to_string(max_parallel_ops))
.ok());
Expand Down
133 changes: 1 addition & 132 deletions test/src/unit-gcs.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
*
* The MIT License
*
* @copyright Copyright (c) 2017-2021 TileDB, Inc.
* @copyright Copyright (c) 2017-2023 TileDB, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
Expand Down Expand Up @@ -120,137 +120,6 @@ TEST_CASE_METHOD(GCSFx, "Test GCS init", "[gcs]") {
}
}

TEST_CASE_METHOD(GCSFx, "Test GCS filesystem, file management", "[gcs]") {
Config config;
REQUIRE(config.set("vfs.gcs.use_multi_part_upload", "true").ok());
init_gcs(std::move(config));

/* Create the following file hierarchy:
*
* TEST_DIR/dir/subdir/file1
* TEST_DIR/dir/subdir/file2
* TEST_DIR/dir/file3
* TEST_DIR/file4
* TEST_DIR/file5
*/
auto dir = TEST_DIR + "dir/";
auto dir2 = TEST_DIR + "dir2/";
auto subdir = dir + "subdir/";
auto file1 = subdir + "file1";
auto file2 = subdir + "file2";
auto file3 = dir + "file3";
auto file4 = TEST_DIR + "file4";
auto file5 = TEST_DIR + "file5";
auto file6 = TEST_DIR + "file6";

// Check that bucket is empty
bool is_empty;
REQUIRE(gcs_.is_empty_bucket(GCS_BUCKET, &is_empty).ok());
REQUIRE(is_empty);

// Continue building the hierarchy
bool is_object = false;
REQUIRE(gcs_.touch(URI(file1)).ok());
REQUIRE(gcs_.is_object(URI(file1), &is_object).ok());
REQUIRE(is_object);
REQUIRE(gcs_.touch(URI(file2)).ok());
REQUIRE(gcs_.is_object(URI(file2), &is_object).ok());
REQUIRE(is_object);
REQUIRE(gcs_.touch(URI(file3)).ok());
REQUIRE(gcs_.is_object(URI(file3), &is_object).ok());
REQUIRE(is_object);
REQUIRE(gcs_.touch(URI(file4)).ok());
REQUIRE(gcs_.is_object(URI(file4), &is_object).ok());
REQUIRE(is_object);
REQUIRE(gcs_.touch(URI(file5)).ok());
REQUIRE(gcs_.is_object(URI(file5), &is_object).ok());
REQUIRE(is_object);

// Check that bucket is not empty
REQUIRE(gcs_.is_empty_bucket(GCS_BUCKET, &is_empty).ok());
REQUIRE(!is_empty);

// Check invalid file
REQUIRE(gcs_.is_object(URI(TEST_DIR + "foo"), &is_object).ok());
REQUIRE(!is_object);

// List with prefix
std::vector<std::string> paths;
REQUIRE(gcs_.ls(URI(TEST_DIR), &paths).ok());
REQUIRE(paths.size() == 3);
paths.clear();
REQUIRE(gcs_.ls(URI(dir), &paths).ok());
REQUIRE(paths.size() == 2);
paths.clear();
REQUIRE(gcs_.ls(URI(subdir), &paths).ok());
REQUIRE(paths.size() == 2);
paths.clear();
REQUIRE(gcs_.ls(GCS_BUCKET, &paths, "").ok()); // No delimiter
REQUIRE(paths.size() == 5);

// Check if a directory exists
bool is_dir = false;
REQUIRE(gcs_.is_dir(URI(file1), &is_dir).ok());
REQUIRE(!is_dir); // Not a dir
REQUIRE(gcs_.is_dir(URI(file4), &is_dir).ok());
REQUIRE(!is_dir); // Not a dir
REQUIRE(gcs_.is_dir(URI(dir), &is_dir).ok());
REQUIRE(is_dir); // This is viewed as a dir
REQUIRE(gcs_.is_dir(URI(TEST_DIR + "dir"), &is_dir).ok());
REQUIRE(is_dir); // This is viewed as a dir

// ls_with_sizes
std::string s = "abcdef";
CHECK(gcs_.write(URI(file3), s.data(), s.size()).ok());
REQUIRE(gcs_.flush_object(URI(file3)).ok());

auto&& [status, rv] = gcs_.ls_with_sizes(URI(dir));
auto children = *rv;
REQUIRE(status.ok());

REQUIRE(children.size() == 2);
CHECK(children[0].path().native() == file3);
CHECK(children[1].path().native() == subdir.substr(0, subdir.size() - 1));

CHECK(children[0].file_size() == s.size());
// Directories don't get a size
CHECK(children[1].file_size() == 0);

// Move file
REQUIRE(gcs_.move_object(URI(file5), URI(file6)).ok());
REQUIRE(gcs_.is_object(URI(file5), &is_object).ok());
REQUIRE(!is_object);
REQUIRE(gcs_.is_object(URI(file6), &is_object).ok());
REQUIRE(is_object);
paths.clear();
REQUIRE(gcs_.ls(GCS_BUCKET, &paths, "").ok()); // No delimiter
REQUIRE(paths.size() == 5);

// Move directory
REQUIRE(gcs_.move_dir(URI(dir), URI(dir2)).ok());
REQUIRE(gcs_.is_dir(URI(dir), &is_dir).ok());
REQUIRE(!is_dir);
REQUIRE(gcs_.is_dir(URI(dir2), &is_dir).ok());
REQUIRE(is_dir);
paths.clear();
REQUIRE(gcs_.ls(GCS_BUCKET, &paths, "").ok()); // No delimiter
REQUIRE(paths.size() == 5);

// Remove files
REQUIRE(gcs_.remove_object(URI(file4)).ok());
REQUIRE(gcs_.is_object(URI(file4), &is_object).ok());
REQUIRE(!is_object);

// Remove directories
REQUIRE(gcs_.remove_dir(URI(dir2)).ok());
REQUIRE(gcs_.is_object(URI(file1), &is_object).ok());
REQUIRE(!is_object);
REQUIRE(gcs_.is_object(URI(file2), &is_object).ok());
REQUIRE(!is_object);
REQUIRE(gcs_.is_object(URI(file3), &is_object).ok());
REQUIRE(!is_object);
}

TEST_CASE_METHOD(
GCSFx,
"Test GCS filesystem I/O, multipart, serial",
Expand Down
38 changes: 1 addition & 37 deletions test/src/unit-hdfs-filesystem.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
*
* The MIT License
*
* @copyright Copyright (c) 2017-2021 TileDB, Inc.
* @copyright Copyright (c) 2017-2023 TileDB, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
Expand Down Expand Up @@ -118,42 +118,6 @@ TEST_CASE("Test HDFS filesystem", "[hdfs]") {
}
CHECK(allok);

std::vector<std::string> paths;
st = hdfs.ls(URI("hdfs:///"), &paths);
CHECK(st.ok());
CHECK(paths.size() > 0);

// ls_with_sizes
// Dir structure:
// ...../subdir
// ...../subdir/file
// ...../subdir/subsubdir

std::string subdir = "hdfs://localhost:9000/tiledb_test/subdir";
std::string file = subdir + "/file";
std::string subsubdir = subdir + "/subsubdir";

CHECK(hdfs.create_dir(URI(subdir)).ok());
CHECK(hdfs.create_dir(URI(subsubdir)).ok());
CHECK(hdfs.touch(URI(file)).ok());

std::string s = "abcdef";
CHECK(hdfs.write(URI(file), s.data(), s.size()).ok());

auto&& [status, rv] = hdfs.ls_with_sizes(URI(subdir));
auto children = *rv;
REQUIRE(status.ok());

REQUIRE(children.size() == 2);
CHECK(children[0].path().native() == file);
CHECK(children[1].path().native() == subsubdir.substr(0, subsubdir.size()));

CHECK(children[0].file_size() == s.size());
// Directories don't get a size
CHECK(children[1].file_size() == 0);
// Cleanup
CHECK(hdfs.remove_dir(URI(subdir)).ok());

uint64_t nbytes = 0;
st = hdfs.file_size(URI("hdfs:///tiledb_test/tiledb_test_file"), &nbytes);
CHECK(st.ok());
Expand Down

0 comments on commit ca11ed2

Please sign in to comment.