diff --git a/test/src/unit-azure.cc b/test/src/unit-azure.cc index cb0efff4dcd..f0ad9fbdc36 100644 --- a/test/src/unit-azure.cc +++ b/test/src/unit-azure.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -131,146 +131,11 @@ std::string AzureFx::random_container_name(const std::string& prefix) { return ss.str(); } -TEST_CASE_METHOD(AzureFx, "Test Azure filesystem, file management", "[azure]") { - Config config; - REQUIRE(config.set("vfs.azure.use_block_list_upload", "true").ok()); - - auto settings = - GENERATE(from_range(test_settings.begin(), test_settings.end())); - init_azure(std::move(config), settings); - - /* Create the following file hierarchy: - * - * TEST_DIR/dir/subdir/file1 - * TEST_DIR/dir/subdir/file2 - * TEST_DIR/dir/file3 - * TEST_DIR/file4 - * TEST_DIR/file5 - */ - auto dir = TEST_DIR + "dir/"; - auto dir2 = TEST_DIR + "dir2/"; - auto subdir = dir + "subdir/"; - auto file1 = subdir + "file1"; - auto file2 = subdir + "file2"; - auto file3 = dir + "file3"; - auto file4 = TEST_DIR + "file4"; - auto file5 = TEST_DIR + "file5"; - auto file6 = TEST_DIR + "file6"; - - // Check that container is empty - bool is_empty; - REQUIRE(azure_.is_empty_container(AZURE_CONTAINER, &is_empty).ok()); - REQUIRE(is_empty); - - // Continue building the hierarchy - bool is_blob = false; - REQUIRE(azure_.touch(URI(file1)).ok()); - REQUIRE(azure_.is_blob(URI(file1), &is_blob).ok()); - REQUIRE(is_blob); - REQUIRE(azure_.touch(URI(file2)).ok()); - REQUIRE(azure_.is_blob(URI(file2), &is_blob).ok()); - REQUIRE(is_blob); - REQUIRE(azure_.touch(URI(file3)).ok()); - REQUIRE(azure_.is_blob(URI(file3), &is_blob).ok()); - REQUIRE(is_blob); - REQUIRE(azure_.touch(URI(file4)).ok()); - REQUIRE(azure_.is_blob(URI(file4), &is_blob).ok()); - REQUIRE(is_blob); - REQUIRE(azure_.touch(URI(file5)).ok()); - REQUIRE(azure_.is_blob(URI(file5), &is_blob).ok()); - REQUIRE(is_blob); - - // Check that container is not empty - REQUIRE(azure_.is_empty_container(AZURE_CONTAINER, &is_empty).ok()); - REQUIRE(!is_empty); - - // Check invalid file - REQUIRE(azure_.is_blob(URI(TEST_DIR + "foo"), &is_blob).ok()); - REQUIRE(!is_blob); - - // List with prefix - std::vector paths; - REQUIRE(azure_.ls(URI(TEST_DIR), &paths).ok()); - REQUIRE(paths.size() == 3); - paths.clear(); - REQUIRE(azure_.ls(URI(dir), &paths).ok()); - REQUIRE(paths.size() == 2); - paths.clear(); - REQUIRE(azure_.ls(URI(subdir), &paths).ok()); - REQUIRE(paths.size() == 2); - paths.clear(); - REQUIRE(azure_.ls(AZURE_CONTAINER, &paths, "").ok()); // No delimiter - REQUIRE(paths.size() == 5); - - // Check if a directory exists - bool is_dir = false; - REQUIRE(azure_.is_dir(URI(file1), &is_dir).ok()); - REQUIRE(!is_dir); // Not a dir - REQUIRE(azure_.is_dir(URI(file4), &is_dir).ok()); - REQUIRE(!is_dir); // Not a dir - REQUIRE(azure_.is_dir(URI(dir), &is_dir).ok()); - REQUIRE(is_dir); // This is viewed as a dir - REQUIRE(azure_.is_dir(URI(TEST_DIR + "dir"), &is_dir).ok()); - REQUIRE(is_dir); // This is viewed as a dir - - // ls_with_sizes - std::string s = "abcdef"; - CHECK(azure_.write(URI(file3), s.data(), s.size()).ok()); - REQUIRE(azure_.flush_blob(URI(file3)).ok()); - - auto&& [status, rv] = azure_.ls_with_sizes(URI(dir)); - auto children = *rv; - REQUIRE(status.ok()); - - REQUIRE(children.size() == 2); - CHECK(children[0].path().native() == file3); - CHECK(children[1].path().native() == subdir.substr(0, subdir.size() - 1)); - - CHECK(children[0].file_size() == s.size()); - // Directories don't get a size - CHECK(children[1].file_size() == 0); - - // Move file - REQUIRE(azure_.move_object(URI(file5), URI(file6)).ok()); - REQUIRE(azure_.is_blob(URI(file5), &is_blob).ok()); - REQUIRE(!is_blob); - REQUIRE(azure_.is_blob(URI(file6), &is_blob).ok()); - REQUIRE(is_blob); - paths.clear(); - REQUIRE(azure_.ls(AZURE_CONTAINER, &paths, "").ok()); // No delimiter - REQUIRE(paths.size() == 5); - - // Move directory - REQUIRE(azure_.move_dir(URI(dir), URI(dir2)).ok()); - REQUIRE(azure_.is_dir(URI(dir), &is_dir).ok()); - REQUIRE(!is_dir); - REQUIRE(azure_.is_dir(URI(dir2), &is_dir).ok()); - REQUIRE(is_dir); - paths.clear(); - REQUIRE(azure_.ls(AZURE_CONTAINER, &paths, "").ok()); // No delimiter - REQUIRE(paths.size() == 5); - - // Remove files - REQUIRE(azure_.remove_blob(URI(file4)).ok()); - REQUIRE(azure_.is_blob(URI(file4), &is_blob).ok()); - REQUIRE(!is_blob); - - // Remove directories - REQUIRE(azure_.remove_dir(URI(dir2)).ok()); - REQUIRE(azure_.is_blob(URI(file1), &is_blob).ok()); - REQUIRE(!is_blob); - REQUIRE(azure_.is_blob(URI(file2), &is_blob).ok()); - REQUIRE(!is_blob); - REQUIRE(azure_.is_blob(URI(file3), &is_blob).ok()); - REQUIRE(!is_blob); -} - TEST_CASE_METHOD( AzureFx, "Test Azure filesystem, file I/O", "[azure][multipart]") { Config config; const uint64_t max_parallel_ops = 2; const uint64_t block_list_block_size = 4 * 1024 * 1024; - REQUIRE(config.set("vfs.azure.use_block_list_upload", "true").ok()); REQUIRE( config.set("vfs.azure.max_parallel_ops", std::to_string(max_parallel_ops)) .ok()); diff --git a/test/src/unit-gcs.cc b/test/src/unit-gcs.cc index a897f965d34..e8065a9e665 100644 --- a/test/src/unit-gcs.cc +++ b/test/src/unit-gcs.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -120,137 +120,6 @@ TEST_CASE_METHOD(GCSFx, "Test GCS init", "[gcs]") { } } -TEST_CASE_METHOD(GCSFx, "Test GCS filesystem, file management", "[gcs]") { - Config config; - REQUIRE(config.set("vfs.gcs.use_multi_part_upload", "true").ok()); - init_gcs(std::move(config)); - - /* Create the following file hierarchy: - * - * TEST_DIR/dir/subdir/file1 - * TEST_DIR/dir/subdir/file2 - * TEST_DIR/dir/file3 - * TEST_DIR/file4 - * TEST_DIR/file5 - */ - auto dir = TEST_DIR + "dir/"; - auto dir2 = TEST_DIR + "dir2/"; - auto subdir = dir + "subdir/"; - auto file1 = subdir + "file1"; - auto file2 = subdir + "file2"; - auto file3 = dir + "file3"; - auto file4 = TEST_DIR + "file4"; - auto file5 = TEST_DIR + "file5"; - auto file6 = TEST_DIR + "file6"; - - // Check that bucket is empty - bool is_empty; - REQUIRE(gcs_.is_empty_bucket(GCS_BUCKET, &is_empty).ok()); - REQUIRE(is_empty); - - // Continue building the hierarchy - bool is_object = false; - REQUIRE(gcs_.touch(URI(file1)).ok()); - REQUIRE(gcs_.is_object(URI(file1), &is_object).ok()); - REQUIRE(is_object); - REQUIRE(gcs_.touch(URI(file2)).ok()); - REQUIRE(gcs_.is_object(URI(file2), &is_object).ok()); - REQUIRE(is_object); - REQUIRE(gcs_.touch(URI(file3)).ok()); - REQUIRE(gcs_.is_object(URI(file3), &is_object).ok()); - REQUIRE(is_object); - REQUIRE(gcs_.touch(URI(file4)).ok()); - REQUIRE(gcs_.is_object(URI(file4), &is_object).ok()); - REQUIRE(is_object); - REQUIRE(gcs_.touch(URI(file5)).ok()); - REQUIRE(gcs_.is_object(URI(file5), &is_object).ok()); - REQUIRE(is_object); - - // Check that bucket is not empty - REQUIRE(gcs_.is_empty_bucket(GCS_BUCKET, &is_empty).ok()); - REQUIRE(!is_empty); - - // Check invalid file - REQUIRE(gcs_.is_object(URI(TEST_DIR + "foo"), &is_object).ok()); - REQUIRE(!is_object); - - // List with prefix - std::vector paths; - REQUIRE(gcs_.ls(URI(TEST_DIR), &paths).ok()); - REQUIRE(paths.size() == 3); - paths.clear(); - REQUIRE(gcs_.ls(URI(dir), &paths).ok()); - REQUIRE(paths.size() == 2); - paths.clear(); - REQUIRE(gcs_.ls(URI(subdir), &paths).ok()); - REQUIRE(paths.size() == 2); - paths.clear(); - REQUIRE(gcs_.ls(GCS_BUCKET, &paths, "").ok()); // No delimiter - REQUIRE(paths.size() == 5); - - // Check if a directory exists - bool is_dir = false; - REQUIRE(gcs_.is_dir(URI(file1), &is_dir).ok()); - REQUIRE(!is_dir); // Not a dir - REQUIRE(gcs_.is_dir(URI(file4), &is_dir).ok()); - REQUIRE(!is_dir); // Not a dir - REQUIRE(gcs_.is_dir(URI(dir), &is_dir).ok()); - REQUIRE(is_dir); // This is viewed as a dir - REQUIRE(gcs_.is_dir(URI(TEST_DIR + "dir"), &is_dir).ok()); - REQUIRE(is_dir); // This is viewed as a dir - - // ls_with_sizes - std::string s = "abcdef"; - CHECK(gcs_.write(URI(file3), s.data(), s.size()).ok()); - REQUIRE(gcs_.flush_object(URI(file3)).ok()); - - auto&& [status, rv] = gcs_.ls_with_sizes(URI(dir)); - auto children = *rv; - REQUIRE(status.ok()); - - REQUIRE(children.size() == 2); - CHECK(children[0].path().native() == file3); - CHECK(children[1].path().native() == subdir.substr(0, subdir.size() - 1)); - - CHECK(children[0].file_size() == s.size()); - // Directories don't get a size - CHECK(children[1].file_size() == 0); - - // Move file - REQUIRE(gcs_.move_object(URI(file5), URI(file6)).ok()); - REQUIRE(gcs_.is_object(URI(file5), &is_object).ok()); - REQUIRE(!is_object); - REQUIRE(gcs_.is_object(URI(file6), &is_object).ok()); - REQUIRE(is_object); - paths.clear(); - REQUIRE(gcs_.ls(GCS_BUCKET, &paths, "").ok()); // No delimiter - REQUIRE(paths.size() == 5); - - // Move directory - REQUIRE(gcs_.move_dir(URI(dir), URI(dir2)).ok()); - REQUIRE(gcs_.is_dir(URI(dir), &is_dir).ok()); - REQUIRE(!is_dir); - REQUIRE(gcs_.is_dir(URI(dir2), &is_dir).ok()); - REQUIRE(is_dir); - paths.clear(); - REQUIRE(gcs_.ls(GCS_BUCKET, &paths, "").ok()); // No delimiter - REQUIRE(paths.size() == 5); - - // Remove files - REQUIRE(gcs_.remove_object(URI(file4)).ok()); - REQUIRE(gcs_.is_object(URI(file4), &is_object).ok()); - REQUIRE(!is_object); - - // Remove directories - REQUIRE(gcs_.remove_dir(URI(dir2)).ok()); - REQUIRE(gcs_.is_object(URI(file1), &is_object).ok()); - REQUIRE(!is_object); - REQUIRE(gcs_.is_object(URI(file2), &is_object).ok()); - REQUIRE(!is_object); - REQUIRE(gcs_.is_object(URI(file3), &is_object).ok()); - REQUIRE(!is_object); -} - TEST_CASE_METHOD( GCSFx, "Test GCS filesystem I/O, multipart, serial", diff --git a/test/src/unit-hdfs-filesystem.cc b/test/src/unit-hdfs-filesystem.cc index 24ff6c0ae4f..4715ccb098f 100644 --- a/test/src/unit-hdfs-filesystem.cc +++ b/test/src/unit-hdfs-filesystem.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -118,42 +118,6 @@ TEST_CASE("Test HDFS filesystem", "[hdfs]") { } CHECK(allok); - std::vector paths; - st = hdfs.ls(URI("hdfs:///"), &paths); - CHECK(st.ok()); - CHECK(paths.size() > 0); - - // ls_with_sizes - // Dir structure: - // ...../subdir - // ...../subdir/file - // ...../subdir/subsubdir - - std::string subdir = "hdfs://localhost:9000/tiledb_test/subdir"; - std::string file = subdir + "/file"; - std::string subsubdir = subdir + "/subsubdir"; - - CHECK(hdfs.create_dir(URI(subdir)).ok()); - CHECK(hdfs.create_dir(URI(subsubdir)).ok()); - CHECK(hdfs.touch(URI(file)).ok()); - - std::string s = "abcdef"; - CHECK(hdfs.write(URI(file), s.data(), s.size()).ok()); - - auto&& [status, rv] = hdfs.ls_with_sizes(URI(subdir)); - auto children = *rv; - REQUIRE(status.ok()); - - REQUIRE(children.size() == 2); - CHECK(children[0].path().native() == file); - CHECK(children[1].path().native() == subsubdir.substr(0, subsubdir.size())); - - CHECK(children[0].file_size() == s.size()); - // Directories don't get a size - CHECK(children[1].file_size() == 0); - // Cleanup - CHECK(hdfs.remove_dir(URI(subdir)).ok()); - uint64_t nbytes = 0; st = hdfs.file_size(URI("hdfs:///tiledb_test/tiledb_test_file"), &nbytes); CHECK(st.ok()); diff --git a/test/src/unit-s3.cc b/test/src/unit-s3.cc index bc24ffc3741..a6e4d05eff2 100644 --- a/test/src/unit-s3.cc +++ b/test/src/unit-s3.cc @@ -105,133 +105,6 @@ Config S3Fx::set_config_params() { return config; } -TEST_CASE_METHOD(S3Fx, "Test S3 filesystem, file management", "[s3]") { - /* Create the following file hierarchy: - * - * TEST_DIR/dir/subdir/file1 - * TEST_DIR/dir/subdir/file2 - * TEST_DIR/dir/file3 - * TEST_DIR/file4 - * TEST_DIR/file5 - */ - auto dir = TEST_DIR + "dir/"; - auto dir2 = TEST_DIR + "dir2/"; - auto subdir = dir + "subdir/"; - auto file1 = subdir + "file1"; - auto file2 = subdir + "file2"; - auto file3 = dir + "file3"; - auto file4 = TEST_DIR + "file4"; - auto file5 = TEST_DIR + "file5"; - auto file6 = TEST_DIR + "file6"; - - // Check that bucket is empty - bool is_empty; - CHECK(s3_.is_empty_bucket(S3_BUCKET, &is_empty).ok()); - CHECK(is_empty); - - // Continue building the hierarchy - bool exists = false; - CHECK(s3_.touch(URI(file1)).ok()); - CHECK(s3_.is_object(URI(file1), &exists).ok()); - CHECK(exists); - CHECK(s3_.touch(URI(file2)).ok()); - CHECK(s3_.is_object(URI(file2), &exists).ok()); - CHECK(exists); - CHECK(s3_.touch(URI(file3)).ok()); - CHECK(s3_.is_object(URI(file3), &exists).ok()); - CHECK(exists); - CHECK(s3_.touch(URI(file4)).ok()); - CHECK(s3_.is_object(URI(file4), &exists).ok()); - CHECK(exists); - CHECK(s3_.touch(URI(file5)).ok()); - CHECK(s3_.is_object(URI(file5), &exists).ok()); - CHECK(exists); - - // Check that the bucket is not empty - CHECK(s3_.is_empty_bucket(S3_BUCKET, &is_empty).ok()); - CHECK(!is_empty); - - // Check invalid file - CHECK(s3_.is_object(URI(TEST_DIR + "foo"), &exists).ok()); - CHECK(!exists); - - // List with prefix - std::vector paths; - CHECK(s3_.ls(URI(TEST_DIR), &paths).ok()); - CHECK(paths.size() == 3); - paths.clear(); - CHECK(s3_.ls(URI(dir), &paths).ok()); - CHECK(paths.size() == 2); - paths.clear(); - CHECK(s3_.ls(URI(subdir), &paths).ok()); - CHECK(paths.size() == 2); - paths.clear(); - CHECK(s3_.ls(S3_BUCKET, &paths, "").ok()); // No delimiter - CHECK(paths.size() == 5); - - // Check if a directory exists - bool is_dir = false; - CHECK(s3_.is_dir(URI(file1), &is_dir).ok()); - CHECK(!is_dir); // Not a dir - CHECK(s3_.is_dir(URI(file4), &is_dir).ok()); - CHECK(!is_dir); // Not a dir - CHECK(s3_.is_dir(URI(dir), &is_dir).ok()); - CHECK(is_dir); // This is viewed as a dir - CHECK(s3_.is_dir(URI(TEST_DIR + "dir"), &is_dir).ok()); - CHECK(is_dir); // This is viewed as a dir - - // ls_with_sizes - std::string s = "abcdef"; - CHECK(s3_.write(URI(file3), s.data(), s.size()).ok()); - CHECK(s3_.flush_object(URI(file3)).ok()); - - auto&& [status, rv] = s3_.ls_with_sizes(URI(dir)); - auto children = *rv; - REQUIRE(status.ok()); - - REQUIRE(children.size() == 2); - CHECK(children[0].path().native() == file3); - CHECK(children[1].path().native() == subdir.substr(0, subdir.size() - 1)); - - CHECK(children[0].file_size() == s.size()); - // Directories don't get a size - CHECK(children[1].file_size() == 0); - - // Move file - CHECK(s3_.move_object(URI(file5), URI(file6)).ok()); - CHECK(s3_.is_object(URI(file5), &exists).ok()); - CHECK(!exists); - CHECK(s3_.is_object(URI(file6), &exists).ok()); - CHECK(exists); - paths.clear(); - CHECK(s3_.ls(S3_BUCKET, &paths, "").ok()); // No delimiter - CHECK(paths.size() == 5); - - // Move directory - CHECK(s3_.move_dir(URI(dir), URI(dir2)).ok()); - CHECK(s3_.is_dir(URI(dir), &is_dir).ok()); - CHECK(!is_dir); - CHECK(s3_.is_dir(URI(dir2), &is_dir).ok()); - CHECK(is_dir); - paths.clear(); - CHECK(s3_.ls(S3_BUCKET, &paths, "").ok()); // No delimiter - CHECK(paths.size() == 5); - - // Remove files - CHECK(s3_.remove_object(URI(file4)).ok()); - CHECK(s3_.is_object(URI(file4), &exists).ok()); - CHECK(!exists); - - // Remove directories - CHECK(s3_.remove_dir(URI(dir2)).ok()); - CHECK(s3_.is_object(URI(file1), &exists).ok()); - CHECK(!exists); - CHECK(s3_.is_object(URI(file2), &exists).ok()); - CHECK(!exists); - CHECK(s3_.is_object(URI(file3), &exists).ok()); - CHECK(!exists); -} - TEST_CASE_METHOD(S3Fx, "Test S3 filesystem, file I/O", "[s3]") { // Prepare buffers uint64_t buffer_size = 5 * 1024 * 1024; diff --git a/test/src/unit-vfs.cc b/test/src/unit-vfs.cc index 189bbe384ce..fe70a04dd7a 100644 --- a/test/src/unit-vfs.cc +++ b/test/src/unit-vfs.cc @@ -31,215 +31,223 @@ */ #include -#include #include "test/support/src/helpers.h" +#include "test/support/src/temporary_local_directory.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/filesystem/vfs.h" #ifdef _WIN32 #include "tiledb/sm/filesystem/path_win.h" #endif -#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/tile/tile.h" +#include + using namespace tiledb::common; using namespace tiledb::sm; using namespace tiledb::test; -#ifdef _WIN32 +// The unique local directory object +tiledb::sm::TemporaryLocalDirectory unit_vfs_dir_{"tiledb_test"}; + +void require_tiledb_ok(Status st) { + REQUIRE(st.ok()); +} + +void require_tiledb_err(Status st) { + REQUIRE(!st.ok()); +} + +Config set_config_params() { + Config config; + + if constexpr (tiledb::sm::filesystem::gcs_enabled) { + require_tiledb_ok(config.set("vfs.gcs.project_id", "TODO")); + } -TEST_CASE("VFS: Test long paths (Win32)", "[vfs][windows]") { + if constexpr (tiledb::sm::filesystem::s3_enabled) { + require_tiledb_ok(config.set("vfs.s3.endpoint_override", "localhost:9999")); + require_tiledb_ok(config.set("vfs.s3.scheme", "https")); + require_tiledb_ok(config.set("vfs.s3.use_virtual_addressing", "false")); + require_tiledb_ok(config.set("vfs.s3.verify_ssl", "false")); + } + + if constexpr (tiledb::sm::filesystem::azure_enabled) { + require_tiledb_ok( + config.set("vfs.azure.storage_account_name", "devstoreaccount1")); + require_tiledb_ok(config.set( + "vfs.azure.storage_account_key", + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/" + "K1SZFPTOtr/KBHBeksoGMGw==")); + require_tiledb_ok(config.set( + "vfs.azure.blob_endpoint", "http://127.0.0.1:10000/devstoreaccount1")); + // Currently disabled because it does not work with the Azurite emulator + // The SAS path was manually tested against the Azure Blob Service. + // require_tiledb_ok(config.set( + // "vfs.azure.storage_account_name", "devstoreaccount2")); + // require_tiledb_ok(config.set("vfs.azure.storage_sas_token", "")); + // require_tiledb_ok(config.set( + // "vfs.azure.blob_endpoint", "http://127.0.0.1:10000/devstoreaccount2")); + } + + return config; +} + +TEST_CASE("VFS: Test long local paths", "[vfs]") { ThreadPool compute_tp(4); ThreadPool io_tp(4); - VFS vfs_long_path_win{&g_helper_stats, &compute_tp, &io_tp, Config{}}; - std::string tmpdir_base = tiledb::sm::Win::current_dir() + "\\tiledb_test\\"; - REQUIRE(vfs_long_path_win.create_dir(URI(tmpdir_base)).ok()); + VFS vfs{&g_helper_stats, &compute_tp, &io_tp, Config{}}; SECTION("- Deep hierarchy") { - // On some Windows platforms, the path length of a directory must be <= 248 - // chars. On others (that have opted in to a configuration that allows - // long paths) the limit is ~32,767. Here we check for either case. - std::string tmpdir = tmpdir_base; + // Create a nested path with a long total length + std::string local_prefix = ""; + if constexpr (!tiledb::sm::filesystem::windows_enabled) { + local_prefix = "file://"; + } + std::string tmpdir = local_prefix + unit_vfs_dir_.path(); bool success = true; while (tmpdir.size() < 512) { - tmpdir += "subdir\\"; - success &= vfs_long_path_win.create_dir(URI(tmpdir)).ok(); + tmpdir += "subdir/"; + success &= vfs.create_dir(URI(tmpdir)).ok(); + if constexpr (tiledb::sm::filesystem::posix_enabled) { + REQUIRE(success); + } } + // On some Windows platforms, the path length of a directory must be <= 248 + // chars. On others (that have opted in to a configuration that allows + // long paths) the limit is ~32,767. Here we check for either case. if (success) { // Check we can create files within the deep hierarchy URI testfile(tmpdir + "file.txt"); REQUIRE(!testfile.is_invalid()); bool exists = false; - REQUIRE(vfs_long_path_win.is_file(testfile, &exists).ok()); - if (exists) - REQUIRE(vfs_long_path_win.remove_file(testfile).ok()); - REQUIRE(vfs_long_path_win.touch(testfile).ok()); - REQUIRE(vfs_long_path_win.remove_file(testfile).ok()); + require_tiledb_ok(vfs.is_file(testfile, &exists)); + if (exists) { + require_tiledb_ok(vfs.remove_file(testfile)); + } + require_tiledb_ok(vfs.touch(testfile)); + require_tiledb_ok(vfs.remove_file(testfile)); } else { // Don't check anything; directory creation failed. } } SECTION("- Too long name") { + // This may not be long enough on some filesystems to pass the fail check. std::string name; - for (unsigned i = 0; i < 256; i++) + for (unsigned i = 0; i < 256; i++) { name += "x"; + } + std::string local_prefix = ""; + if constexpr (!tiledb::sm::filesystem::windows_enabled) { + local_prefix = "file://"; + } + std::string tmpdir = local_prefix + unit_vfs_dir_.path(); + URI testfile(tmpdir + name); + + // Creating the URI and checking its existence is fine on posix + if constexpr (tiledb::sm::filesystem::posix_enabled) { + REQUIRE(!testfile.is_invalid()); + bool exists = false; + require_tiledb_ok(vfs.is_file(testfile, &exists)); + + // Creating the file is not + require_tiledb_err(vfs.touch(testfile)); + } // Creating the URI is invalid on Win32 (failure to canonicalize path) - URI testfile(tmpdir_base + name); - REQUIRE(testfile.is_invalid()); + if constexpr (tiledb::sm::filesystem::windows_enabled) { + REQUIRE(testfile.is_invalid()); + } } - - REQUIRE(vfs_long_path_win.remove_dir(URI(tmpdir_base)).ok()); } -#else - -TEST_CASE("VFS: Test long posix paths", "[vfs]") { +TEST_CASE("VFS: URI semantics and file management", "[vfs][uri]") { ThreadPool compute_tp(4); ThreadPool io_tp(4); - VFS vfs_long_path_pos{&g_helper_stats, &compute_tp, &io_tp, Config{}}; - - std::string tmpdir_base = Posix::current_dir() + "/tiledb_test/"; - REQUIRE(vfs_long_path_pos.create_dir(URI(tmpdir_base)).ok()); - - SECTION("- Deep hierarchy") { - // Create a nested path with a long total length - std::string tmpdir = tmpdir_base; - while (tmpdir.size() < 512) { - tmpdir += "subdir/"; - REQUIRE(vfs_long_path_pos.create_dir(URI(tmpdir)).ok()); + Config config = set_config_params(); + VFS vfs{&g_helper_stats, &compute_tp, &io_tp, config}; + + // Sections to test each enabled filesystem + URI path; + std::string local_prefix = ""; + SECTION("Filesystem: Local") { + if constexpr (!tiledb::sm::filesystem::windows_enabled) { + local_prefix = "file://"; } - - // Check we can create files within the deep hierarchy - URI testfile("file://" + tmpdir + "file.txt"); - REQUIRE(!testfile.is_invalid()); - bool exists = false; - REQUIRE(vfs_long_path_pos.is_file(testfile, &exists).ok()); - if (exists) - REQUIRE(vfs_long_path_pos.remove_file(testfile).ok()); - REQUIRE(vfs_long_path_pos.touch(testfile).ok()); - REQUIRE(vfs_long_path_pos.remove_file(testfile).ok()); + path = URI(local_prefix + unit_vfs_dir_.path()); } - SECTION("- Too long name") { - // This may not be long enough on some filesystems to pass the fail check. - std::string name; - for (unsigned i = 0; i < 256; i++) - name += "x"; - - // Creating the URI and checking its existence is fine - URI testfile("file://" + tmpdir_base + name); - REQUIRE(!testfile.is_invalid()); - bool exists = false; - REQUIRE(vfs_long_path_pos.is_file(testfile, &exists).ok()); - - // Creating the file is not - REQUIRE(!vfs_long_path_pos.touch(testfile).ok()); + if constexpr (tiledb::sm::filesystem::gcs_enabled) { + SECTION("Filesystem: GCS") { + path = URI("gcs://vfs-" + random_label() + "/"); + } } - REQUIRE(vfs_long_path_pos.remove_dir(URI(tmpdir_base)).ok()); -} - -#endif - -TEST_CASE("VFS: URI semantics", "[vfs][uri]") { - ThreadPool compute_tp(4); - ThreadPool io_tp(4); - std::vector> root_pairs; - if constexpr (tiledb::sm::filesystem::s3_enabled) { - Config config; - REQUIRE(config.set("vfs.s3.endpoint_override", "localhost:9999").ok()); - REQUIRE(config.set("vfs.s3.scheme", "https").ok()); - REQUIRE(config.set("vfs.s3.use_virtual_addressing", "false").ok()); - REQUIRE(config.set("vfs.s3.verify_ssl", "false").ok()); - - root_pairs.emplace_back( - URI("s3://vfs-" + random_label() + "/"), std::move(config)); + SECTION("Filesystem: S3") { + path = URI("s3://vfs-" + random_label() + "/"); + } } + if constexpr (tiledb::sm::filesystem::hdfs_enabled) { - Config config; - root_pairs.emplace_back( - URI("hdfs:///vfs-" + random_label() + "/"), std::move(config)); + SECTION("Filesystem: HDFS") { + path = URI("hdfs:///vfs-" + random_label() + "/"); + } } + if constexpr (tiledb::sm::filesystem::azure_enabled) { - Config config; - REQUIRE( - config.set("vfs.azure.storage_account_name", "devstoreaccount1").ok()); - REQUIRE(config - .set( - "vfs.azure.storage_account_key", - "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4" - "I6tq/" - "K1SZFPTOtr/KBHBeksoGMGw==") - .ok()); - REQUIRE(config - .set( - "vfs.azure.blob_endpoint", - "http://127.0.0.1:10000/devstoreaccount1") - .ok()); - - root_pairs.emplace_back( - URI("azure://vfs-" + random_label() + "/"), std::move(config)); + SECTION("Filesystem: Azure") { + path = URI("azure://vfs-" + random_label() + "/"); + } } - Config config; -#ifdef _WIN32 - root_pairs.emplace_back( - URI(tiledb::sm::Win::current_dir() + "\\vfs-" + random_label() + "\\"), - std::move(config)); -#else - root_pairs.emplace_back( - URI(Posix::current_dir() + "/vfs-" + random_label() + "/"), - std::move(config)); -#endif - - for (const auto& root_pair : root_pairs) { - const URI& root = root_pair.first; - const Config& config = root_pair.second; - - VFS vfs_uri{&g_helper_stats, &compute_tp, &io_tp, config}; - - bool exists = false; - if (root.is_s3() || root.is_azure()) { - REQUIRE(vfs_uri.is_bucket(root, &exists).ok()); - if (exists) { - REQUIRE(vfs_uri.remove_bucket(root).ok()); - } - REQUIRE(vfs_uri.create_bucket(root).ok()); - } else { - REQUIRE(vfs_uri.is_dir(root, &exists).ok()); - if (exists) { - REQUIRE(vfs_uri.remove_dir(root).ok()); - } - REQUIRE(vfs_uri.create_dir(root).ok()); + // Set up + bool exists = false; + if (path.is_gcs() || path.is_s3() || path.is_azure()) { + require_tiledb_ok(vfs.is_bucket(path, &exists)); + if (exists) { + require_tiledb_ok(vfs.remove_bucket(path)); } + require_tiledb_ok(vfs.create_bucket(path)); + } else { + require_tiledb_ok(vfs.is_dir(path, &exists)); + if (exists) { + require_tiledb_ok(vfs.remove_dir(path)); + } + require_tiledb_ok(vfs.create_dir(path)); + } - std::string dir1 = root.to_string() + "dir1"; - REQUIRE(vfs_uri.create_dir(URI(dir1)).ok()); - - std::string dir2 = root.to_string() + "dir1/dir2/"; - REQUIRE(vfs_uri.create_dir(URI(dir2)).ok()); - - URI file1(root.to_string() + "file1"); - REQUIRE(vfs_uri.touch(file1).ok()); - - URI file2(root.to_string() + "file2"); - REQUIRE(vfs_uri.touch(file2).ok()); - - URI file3(root.to_string() + "dir1/file3"); - REQUIRE(vfs_uri.touch(file3).ok()); - - URI file4(root.to_string() + "dir1/dir2/file4"); - REQUIRE(vfs_uri.touch(file4).ok()); - - URI file5(root.to_string() + "file5/"); - REQUIRE(!vfs_uri.touch(file5).ok()); - + /* Create the following file hierarchy: + * + * path/dir1/subdir/file1 + * path/dir1/subdir/file2 + * path/dir1/file3 + * path/file4 + * path/file5 + */ + auto dir1 = URI(path.to_string() + "dir1/"); + auto subdir = URI(dir1.to_string() + "subdir/"); + auto file1 = URI(subdir.to_string() + "file1"); + auto file2 = URI(subdir.to_string() + "file2"); + auto file3 = URI(dir1.to_string() + "file3"); + auto file4 = URI(path.to_string() + "file4"); + auto file5 = URI(path.to_string() + "file5"); + require_tiledb_ok(vfs.create_dir(URI(dir1))); + require_tiledb_ok(vfs.create_dir(URI(subdir))); + require_tiledb_ok(vfs.touch(file1)); + require_tiledb_ok(vfs.touch(file2)); + require_tiledb_ok(vfs.touch(file3)); + require_tiledb_ok(vfs.touch(file4)); + require_tiledb_ok(vfs.touch(file5)); + + /** + * URI Semantics + */ + { + std::vector expected_uri_names = {"file4", "file5", "dir1"}; std::vector uris; - REQUIRE(vfs_uri.ls(root, &uris).ok()); - - std::vector expected_uri_names = {"file1", "file2", "dir1"}; + require_tiledb_ok(vfs.ls(path, &uris)); for (const auto& uri : uris) { // Ensure that the URIs do not contain a trailing backslash. @@ -268,12 +276,126 @@ TEST_CASE("VFS: URI semantics", "[vfs][uri]") { // Verify we found all expected file/dir names. REQUIRE(expected_uri_names.empty()); - - if (root.is_s3() || root.is_azure()) { - REQUIRE(vfs_uri.remove_bucket(root).ok()); - } else { - REQUIRE(vfs_uri.remove_dir(root).ok()); + } // URI Semantics + + /** + * File Management + */ + { + // Check invalid file + require_tiledb_ok(vfs.is_file(URI(path.to_string() + "foo"), &exists)); + CHECK(!exists); + + // List with prefix + std::vector paths; + require_tiledb_ok(vfs.ls(path, &paths)); + CHECK(paths.size() == 3); + paths.clear(); + require_tiledb_ok(vfs.ls(dir1, &paths)); + CHECK(paths.size() == 2); + paths.clear(); + require_tiledb_ok(vfs.ls(subdir, &paths)); + CHECK(paths.size() == 2); + paths.clear(); + + // Check if a directory exists + require_tiledb_ok(vfs.is_dir(file1, &exists)); + CHECK(!exists); // Not a dir + require_tiledb_ok(vfs.is_dir(file4, &exists)); + CHECK(!exists); // Not a dir + require_tiledb_ok(vfs.is_dir(dir1, &exists)); + CHECK(exists); // This is viewed as a dir + require_tiledb_ok(vfs.is_dir(URI(path.to_string() + "dir1"), &exists)); + CHECK(exists); // This is viewed as a dir + + // Check ls_with_sizes + URI ls_dir = dir1; + URI ls_subdir = subdir; + URI ls_file = file3; + if (path.is_hdfs()) { + // HDFS requires localhost-resolved paths for ls_with_sizes + auto localdir1 = + URI("hdfs://localhost:9000/vfs-" + random_label() + "/dir1/"); + require_tiledb_ok(vfs.create_dir(localdir1)); + auto localsubdir = URI(localdir1.to_string() + "subdir/"); + require_tiledb_ok(vfs.create_dir(localsubdir)); + auto localfile3 = URI(localdir1.to_string() + "file3"); + require_tiledb_ok(vfs.touch(localfile3)); + ls_dir = localdir1; + ls_subdir = localsubdir; + ls_file = localfile3; + } + std::string s = "abcdef"; + require_tiledb_ok(vfs.write(ls_file, s.data(), s.size())); + require_tiledb_ok(vfs.close_file(ls_file)); + auto&& [status, opt_children] = vfs.ls_with_sizes(ls_dir); + require_tiledb_ok(status); + auto children = opt_children.value(); +#ifdef _WIN32 + // Normalization only for Windows + ls_file = URI(tiledb::sm::path_win::uri_from_path(ls_file.to_string())); +#endif + REQUIRE(children.size() == 2); + CHECK(URI(children[0].path().native()) == ls_file); + CHECK( + URI(children[1].path().native()) == ls_subdir.remove_trailing_slash()); + CHECK(children[0].file_size() == s.size()); + CHECK(children[1].file_size() == 0); // Directories don't get a size + + if (path.is_hdfs()) { + // Clean up + require_tiledb_ok(vfs.remove_dir(ls_dir)); + require_tiledb_ok(vfs.is_dir(ls_dir, &exists)); + CHECK(!exists); } + + // Move file + auto file6 = URI(path.to_string() + "file6"); + require_tiledb_ok(vfs.move_file(file5, file6)); + require_tiledb_ok(vfs.is_file(file5, &exists)); + CHECK(!exists); + require_tiledb_ok(vfs.is_file(file6, &exists)); + CHECK(exists); + paths.clear(); + + // Move directory + auto dir2 = URI(path.to_string() + "dir2/"); + require_tiledb_ok(vfs.move_dir(dir1, URI(dir2))); + require_tiledb_ok(vfs.is_dir(dir1, &exists)); + CHECK(!exists); + require_tiledb_ok(vfs.is_dir(dir2, &exists)); + CHECK(exists); + paths.clear(); + + // Remove files + require_tiledb_ok(vfs.remove_file(file4)); + require_tiledb_ok(vfs.is_file(file4, &exists)); + CHECK(!exists); + require_tiledb_ok(vfs.remove_file(file6)); + require_tiledb_ok(vfs.is_file(file6, &exists)); + CHECK(!exists); + + // Remove directories + require_tiledb_ok(vfs.remove_dir(dir2)); + require_tiledb_ok(vfs.is_file(file1, &exists)); + CHECK(!exists); + require_tiledb_ok(vfs.is_file(file2, &exists)); + CHECK(!exists); + require_tiledb_ok(vfs.is_file(file3, &exists)); + CHECK(!exists); + require_tiledb_ok(vfs.is_dir(dir2, &exists)); + CHECK(!exists); + } // File Management + + // Clean up + if (path.is_gcs() || path.is_s3() || path.is_azure()) { + require_tiledb_ok(vfs.remove_bucket(path)); + require_tiledb_ok(vfs.is_bucket(path, &exists)); + REQUIRE(!exists); + } else { + require_tiledb_ok(vfs.remove_dir(path)); + require_tiledb_ok(vfs.is_dir(path, &exists)); + REQUIRE(!exists); } } @@ -282,44 +404,35 @@ TEST_CASE("VFS: test ls_with_sizes", "[vfs][ls-with-sizes]") { ThreadPool io_tp(4); VFS vfs_ls{&g_helper_stats, &compute_tp, &io_tp, Config{}}; -#ifdef _WIN32 - std::string path = tiledb::sm::Win::current_dir() + "\\vfs_test\\"; -#else - std::string path = - std::string("file://") + tiledb::sm::Posix::current_dir() + "/vfs_test/"; -#endif - - // Clean up - bool is_dir = false; - REQUIRE(vfs_ls.is_dir(URI(path), &is_dir).ok()); - if (is_dir) - REQUIRE(vfs_ls.remove_dir(URI(path)).ok()); - + std::string local_prefix = ""; + if constexpr (!tiledb::sm::filesystem::windows_enabled) { + local_prefix = "file://"; + } + std::string path = local_prefix + unit_vfs_dir_.path(); std::string dir = path + "ls_dir"; std::string file = dir + "/file"; std::string subdir = dir + "/subdir"; std::string subdir_file = subdir + "/file"; // Create directories and files - REQUIRE(vfs_ls.create_dir(URI(path)).ok()); - REQUIRE(vfs_ls.create_dir(URI(dir)).ok()); - REQUIRE(vfs_ls.create_dir(URI(subdir)).ok()); - REQUIRE(vfs_ls.touch(URI(file)).ok()); - REQUIRE(vfs_ls.touch(URI(subdir_file)).ok()); + require_tiledb_ok(vfs_ls.create_dir(URI(path))); + require_tiledb_ok(vfs_ls.create_dir(URI(dir))); + require_tiledb_ok(vfs_ls.create_dir(URI(subdir))); + require_tiledb_ok(vfs_ls.touch(URI(file))); + require_tiledb_ok(vfs_ls.touch(URI(subdir_file))); // Write to file std::string s1 = "abcdef"; - REQUIRE(vfs_ls.write(URI(file), s1.data(), s1.size()).ok()); + require_tiledb_ok(vfs_ls.write(URI(file), s1.data(), s1.size())); // Write to subdir file std::string s2 = "abcdef"; - REQUIRE(vfs_ls.write(URI(subdir_file), s2.data(), s2.size()).ok()); + require_tiledb_ok(vfs_ls.write(URI(subdir_file), s2.data(), s2.size())); // List auto&& [status, rv] = vfs_ls.ls_with_sizes(URI(dir)); auto children = *rv; - - REQUIRE(status.ok()); + require_tiledb_ok(status); #ifdef _WIN32 // Normalization only for Windows @@ -329,17 +442,15 @@ TEST_CASE("VFS: test ls_with_sizes", "[vfs][ls-with-sizes]") { // Check results REQUIRE(children.size() == 2); - REQUIRE(children[0].path().native() == URI(file).to_path()); REQUIRE(children[1].path().native() == URI(subdir).to_path()); - REQUIRE(children[0].file_size() == 6); // Directories don't get a size REQUIRE(children[1].file_size() == 0); // Clean up - REQUIRE(vfs_ls.remove_dir(URI(path)).ok()); + require_tiledb_ok(vfs_ls.remove_dir(URI(path))); } // Currently only S3 is supported for VFS::ls_recursive.