diff --git a/include/paimon/fs/file_system.h b/include/paimon/fs/file_system.h index bee3ecc8..de1b7b57 100644 --- a/include/paimon/fs/file_system.h +++ b/include/paimon/fs/file_system.h @@ -193,6 +193,15 @@ class PAIMON_EXPORT FileSystem { /// failure (e.g., file not found, permission denied). virtual Result> Open(const std::string& path) const = 0; + /// Open an existing file for reading with a known file size. + /// + /// File systems that can use the size to avoid metadata requests may override this method. + /// The default implementation ignores the size and opens the file normally. + virtual Result> Open(const std::string& path, + int64_t /*file_size*/) const { + return Open(path); + } + /// Create a new file for writing. /// @param path The file path to create. /// @param overwrite If true, overwrite existing file; if false, fail if file exists. diff --git a/src/paimon/common/fs/object_store_file_system.cpp b/src/paimon/common/fs/object_store_file_system.cpp index 2eb2b9be..fd929efd 100644 --- a/src/paimon/common/fs/object_store_file_system.cpp +++ b/src/paimon/common/fs/object_store_file_system.cpp @@ -388,6 +388,17 @@ Result> ObjectStoreFileSystem::Open(const std::stri ToUri(object_path), metadata.value().size); } +Result> ObjectStoreFileSystem::Open(const std::string& path, + int64_t file_size) const { + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(file_size, "file size")); + PAIMON_ASSIGN_OR_RAISE(ObjectStorePath object_path, ParsePath(path)); + if (object_path.key.empty()) { + return Status::Invalid(fmt::format("{} is a directory", path)); + } + return std::make_unique(client_, read_ahead_limiter_, object_path, + ToUri(object_path), file_size); +} + Result> ObjectStoreFileSystem::GetFileStatus( const std::string& path) const { PAIMON_ASSIGN_OR_RAISE(ObjectStorePath object_path, ParsePath(path)); diff --git a/src/paimon/common/fs/object_store_file_system.h b/src/paimon/common/fs/object_store_file_system.h index 9ef30528..872ecf12 100644 --- a/src/paimon/common/fs/object_store_file_system.h +++ b/src/paimon/common/fs/object_store_file_system.h @@ -85,6 +85,8 @@ class PAIMON_EXPORT ObjectStoreFileSystem : public FileSystem { ~ObjectStoreFileSystem() override = default; Result> Open(const std::string& path) const override; + Result> Open(const std::string& path, + int64_t file_size) const override; Result> GetFileStatus(const std::string& path) const override; Status ListDir(const std::string& directory, std::vector>* file_status_list) const override; diff --git a/src/paimon/common/fs/object_store_file_system_test.cpp b/src/paimon/common/fs/object_store_file_system_test.cpp index 1b34917a..8a018204 100644 --- a/src/paimon/common/fs/object_store_file_system_test.cpp +++ b/src/paimon/common/fs/object_store_file_system_test.cpp @@ -159,6 +159,15 @@ TEST(ObjectStoreFileSystemTest, TestOpenBucketRootIsDirectory) { ASSERT_EQ(client->list_calls_, 0); } +TEST(ObjectStoreFileSystemTest, TestOpenWithKnownLengthSkipsHead) { + auto client = std::make_shared(); + client->objects_["file"] = "data"; + ObjectStoreFileSystem fs("s3", client); + ASSERT_OK_AND_ASSIGN(auto stream, fs.Open("s3://bucket/file", 4)); + ASSERT_EQ(stream->Length().value(), 4); + ASSERT_EQ(client->head_calls_, 0); +} + TEST(ObjectStoreFileSystemTest, TestPathWithLeadingSlashes) { auto client = std::make_shared(); client->objects_["file"] = "data"; diff --git a/src/paimon/common/fs/resolving_file_system.cpp b/src/paimon/common/fs/resolving_file_system.cpp index 69fc2822..2746d40b 100644 --- a/src/paimon/common/fs/resolving_file_system.cpp +++ b/src/paimon/common/fs/resolving_file_system.cpp @@ -79,6 +79,12 @@ Result> ResolvingFileSystem::Open(const std::string return fs->Open(path); } +Result> ResolvingFileSystem::Open(const std::string& path, + int64_t file_size) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fs, GetRealFileSystem(path)); + return fs->Open(path, file_size); +} + Result> ResolvingFileSystem::Create(const std::string& path, bool overwrite) const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fs, GetRealFileSystem(path)); diff --git a/src/paimon/common/fs/resolving_file_system.h b/src/paimon/common/fs/resolving_file_system.h index 2c5c6434..151840b8 100644 --- a/src/paimon/common/fs/resolving_file_system.h +++ b/src/paimon/common/fs/resolving_file_system.h @@ -41,6 +41,8 @@ class ResolvingFileSystem : public FileSystem { ~ResolvingFileSystem() override = default; Result> Open(const std::string& path) const override; + Result> Open(const std::string& path, + int64_t file_size) const override; Result> Create(const std::string& path, bool overwrite) const override; diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp index defdd660..415dbd9b 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp @@ -62,6 +62,19 @@ Result> PrefetchFileBatchReaderImpl const std::shared_ptr& executor, bool initialize_read_ranges, PrefetchCacheMode prefetch_cache_mode, const CacheConfig& cache_config, const std::shared_ptr& pool) { + return Create(data_file_path, /*data_file_size=*/-1, reader_builder, fs, + prefetch_max_parallel_num, batch_size, prefetch_batch_count, + enable_adaptive_prefetch_strategy, executor, initialize_read_ranges, + prefetch_cache_mode, cache_config, pool); +} + +Result> PrefetchFileBatchReaderImpl::Create( + const std::string& data_file_path, int64_t data_file_size, const ReaderBuilder* reader_builder, + const std::shared_ptr& fs, uint32_t prefetch_max_parallel_num, int32_t batch_size, + uint32_t prefetch_batch_count, bool enable_adaptive_prefetch_strategy, + const std::shared_ptr& executor, bool initialize_read_ranges, + PrefetchCacheMode prefetch_cache_mode, const CacheConfig& cache_config, + const std::shared_ptr& pool) { if (prefetch_max_parallel_num == 0) { return Status::Invalid("prefetch max parallel num should be greater than 0."); } @@ -83,16 +96,17 @@ Result> PrefetchFileBatchReaderImpl std::shared_ptr cache; if (prefetch_cache_mode != PrefetchCacheMode::NEVER) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream, fs->Open(data_file_path)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream, + fs->Open(data_file_path, data_file_size)); cache = std::make_shared(input_stream, cache_config, pool); } std::vector>>> futures; for (uint32_t i = 0; i < prefetch_max_parallel_num; i++) { futures.push_back(Via(executor.get(), - [&fs, &data_file_path, &reader_builder, + [&fs, &data_file_path, data_file_size, &reader_builder, &cache]() -> Result> { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr input_stream, - fs->Open(data_file_path)); + fs->Open(data_file_path, data_file_size)); auto cache_input_stream = std::make_shared( std::move(input_stream), cache); return reader_builder->Build(cache_input_stream); diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.h b/src/paimon/common/reader/prefetch_file_batch_reader_impl.h index f0c302e2..1dca6964 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.h +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.h @@ -55,6 +55,14 @@ class Metrics; class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader { public: + static Result> Create( + const std::string& data_file_path, int64_t data_file_size, + const ReaderBuilder* reader_builder, const std::shared_ptr& fs, + uint32_t prefetch_max_parallel_num, int32_t batch_size, uint32_t prefetch_batch_count, + bool enable_adaptive_prefetch_strategy, const std::shared_ptr& executor, + bool initialize_read_ranges, PrefetchCacheMode prefetch_cache_mode, + const CacheConfig& cache_config, const std::shared_ptr& pool); + static Result> Create( const std::string& data_file_path, const ReaderBuilder* reader_builder, const std::shared_ptr& fs, uint32_t prefetch_max_parallel_num, diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp index 0850d943..98268bc8 100644 --- a/src/paimon/core/operation/abstract_split_read.cpp +++ b/src/paimon/core/operation/abstract_split_read.cpp @@ -144,13 +144,13 @@ Result> AbstractSplitRead::PrepareReaderBuilder( Result> AbstractSplitRead::CreateFileBatchReader( const std::string& file_format_identifier, const std::string& data_file_path, - const ReaderBuilder* reader_builder) const { + int64_t data_file_size, const ReaderBuilder* reader_builder) const { if (context_->EnablePrefetch() && file_format_identifier != "blob" && file_format_identifier != "avro") { PAIMON_ASSIGN_OR_RAISE( std::unique_ptr prefetch_reader, PrefetchFileBatchReaderImpl::Create( - data_file_path, reader_builder, options_.GetFileSystem(), + data_file_path, data_file_size, reader_builder, options_.GetFileSystem(), context_->GetPrefetchMaxParallelNum(), options_.GetReadBatchSize(), context_->GetPrefetchBatchCount(), options_.EnableAdaptivePrefetchStrategy(), executor_, @@ -159,7 +159,7 @@ Result> AbstractSplitRead::CreateFileBatchReade return std::make_unique(std::move(prefetch_reader)); } else { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream, - options_.GetFileSystem()->Open(data_file_path)); + options_.GetFileSystem()->Open(data_file_path, data_file_size)); return reader_builder->Build(input_stream); } } @@ -204,9 +204,9 @@ Result> AbstractSplitRead::CreateFieldMappingRe field_mapping->non_partition_info.non_partition_data_schema); PAIMON_ASSIGN_OR_RAISE(std::string file_format_identifier, file_meta->FileFormat()); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr file_reader, - CreateFileBatchReader(file_format_identifier, data_file_path, reader_builder)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_reader, + CreateFileBatchReader(file_format_identifier, data_file_path, + file_meta->file_size, reader_builder)); std::set skip_map_selected_keys_filter_field_ids; if (file_format_identifier != "blob") { std::pair, std::set> shared_shredding_result; diff --git a/src/paimon/core/operation/abstract_split_read.h b/src/paimon/core/operation/abstract_split_read.h index ea3f9070..27349fec 100644 --- a/src/paimon/core/operation/abstract_split_read.h +++ b/src/paimon/core/operation/abstract_split_read.h @@ -107,7 +107,7 @@ class AbstractSplitRead : public SplitRead { Result> CreateFileBatchReader( const std::string& file_format_identifier, const std::string& data_file_path, - const ReaderBuilder* reader_builder) const; + int64_t data_file_size, const ReaderBuilder* reader_builder) const; // return nullptr if data file is skipped by index or dv Result> CreateFieldMappingReader(