Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into improve_concurrent_parts_removal
- Loading branch information
Showing
131 changed files
with
4,917 additions
and
1,589 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,15 +1,30 @@ | ||
| set (SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest") | ||
| set (SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/googletest") | ||
|
|
||
| add_library(_gtest "${SRC_DIR}/src/gtest-all.cc") | ||
| add_library(_gtest "${SRC_DIR}/googletest/src/gtest-all.cc") | ||
| set_target_properties(_gtest PROPERTIES VERSION "1.0.0") | ||
| target_compile_definitions (_gtest PUBLIC GTEST_HAS_POSIX_RE=0) | ||
| target_include_directories(_gtest SYSTEM PUBLIC "${SRC_DIR}/include") | ||
| target_include_directories(_gtest PRIVATE "${SRC_DIR}") | ||
| target_include_directories(_gtest SYSTEM PUBLIC "${SRC_DIR}/googletest/include") | ||
| target_include_directories(_gtest PRIVATE "${SRC_DIR}/googletest") | ||
|
|
||
| add_library(_gtest_main "${SRC_DIR}/src/gtest_main.cc") | ||
| add_library(_gtest_main "${SRC_DIR}/googletest/src/gtest_main.cc") | ||
| set_target_properties(_gtest_main PROPERTIES VERSION "1.0.0") | ||
| target_link_libraries(_gtest_main PUBLIC _gtest) | ||
|
|
||
| add_library(_gtest_all INTERFACE) | ||
| target_link_libraries(_gtest_all INTERFACE _gtest _gtest_main) | ||
| add_library(ch_contrib::gtest_all ALIAS _gtest_all) | ||
|
|
||
|
|
||
| add_library(_gmock "${SRC_DIR}/googlemock/src/gmock-all.cc") | ||
| set_target_properties(_gmock PROPERTIES VERSION "1.0.0") | ||
| target_compile_definitions (_gmock PUBLIC GTEST_HAS_POSIX_RE=0) | ||
| target_include_directories(_gmock SYSTEM PUBLIC "${SRC_DIR}/googlemock/include" "${SRC_DIR}/googletest/include") | ||
| target_include_directories(_gmock PRIVATE "${SRC_DIR}/googlemock") | ||
|
|
||
| add_library(_gmock_main "${SRC_DIR}/googlemock/src/gmock_main.cc") | ||
| set_target_properties(_gmock_main PROPERTIES VERSION "1.0.0") | ||
| target_link_libraries(_gmock_main PUBLIC _gmock) | ||
|
|
||
| add_library(_gmock_all INTERFACE) | ||
| target_link_libraries(_gmock_all INTERFACE _gmock _gmock_main) | ||
| add_library(ch_contrib::gmock_all ALIAS _gmock_all) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,26 +1,45 @@ | ||
| #include <Backups/BackupEntryFromAppendOnlyFile.h> | ||
| #include <Disks/IDisk.h> | ||
| #include <IO/LimitSeekableReadBuffer.h> | ||
|
|
||
|
|
||
| namespace DB | ||
| { | ||
|
|
||
| namespace | ||
| { | ||
| /// For append-only files we must calculate its size on the construction of a backup entry. | ||
| UInt64 calculateSize(const DiskPtr & disk, const String & file_path, bool copy_encrypted, std::optional<UInt64> unencrypted_file_size) | ||
| { | ||
| if (!unencrypted_file_size) | ||
| return copy_encrypted ? disk->getEncryptedFileSize(file_path) : disk->getFileSize(file_path); | ||
| else if (copy_encrypted) | ||
| return disk->getEncryptedFileSize(*unencrypted_file_size); | ||
| else | ||
| return *unencrypted_file_size; | ||
| } | ||
| } | ||
|
|
||
| BackupEntryFromAppendOnlyFile::BackupEntryFromAppendOnlyFile( | ||
| const DiskPtr & disk_, | ||
| const String & file_path_, | ||
| const ReadSettings & settings_, | ||
| const std::optional<UInt64> & file_size_, | ||
| const std::optional<UInt128> & checksum_, | ||
| const std::shared_ptr<TemporaryFileOnDisk> & temporary_file_) | ||
| : BackupEntryFromImmutableFile(disk_, file_path_, settings_, file_size_, checksum_, temporary_file_) | ||
| , limit(BackupEntryFromImmutableFile::getSize()) | ||
| const DiskPtr & disk_, const String & file_path_, bool copy_encrypted_, const std::optional<UInt64> & file_size_) | ||
| : disk(disk_) | ||
| , file_path(file_path_) | ||
| , data_source_description(disk->getDataSourceDescription()) | ||
| , copy_encrypted(copy_encrypted_ && data_source_description.is_encrypted) | ||
| , size(calculateSize(disk_, file_path_, copy_encrypted, file_size_)) | ||
| { | ||
| } | ||
|
|
||
| std::unique_ptr<SeekableReadBuffer> BackupEntryFromAppendOnlyFile::getReadBuffer() const | ||
| BackupEntryFromAppendOnlyFile::~BackupEntryFromAppendOnlyFile() = default; | ||
|
|
||
| std::unique_ptr<SeekableReadBuffer> BackupEntryFromAppendOnlyFile::getReadBuffer(const ReadSettings & read_settings) const | ||
| { | ||
| auto buf = BackupEntryFromImmutableFile::getReadBuffer(); | ||
| return std::make_unique<LimitSeekableReadBuffer>(std::move(buf), 0, limit); | ||
| std::unique_ptr<SeekableReadBuffer> buf; | ||
| if (copy_encrypted) | ||
| buf = disk->readEncryptedFile(file_path, read_settings.adjustBufferSize(size)); | ||
| else | ||
| buf = disk->readFile(file_path, read_settings.adjustBufferSize(size)); | ||
| return std::make_unique<LimitSeekableReadBuffer>(std::move(buf), 0, size); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,31 +1,41 @@ | ||
| #pragma once | ||
|
|
||
| #include <Backups/BackupEntryFromImmutableFile.h> | ||
| #include <Backups/BackupEntryWithChecksumCalculation.h> | ||
|
|
||
|
|
||
| namespace DB | ||
| { | ||
|
|
||
| /// Represents a file prepared to be included in a backup, assuming that until this backup entry is destroyed | ||
| /// the file can be appended with new data, but the bytes which are already in the file won't be changed. | ||
| class BackupEntryFromAppendOnlyFile : public BackupEntryFromImmutableFile | ||
| class BackupEntryFromAppendOnlyFile : public BackupEntryWithChecksumCalculation<IBackupEntry> | ||
| { | ||
| public: | ||
|
|
||
| /// The constructor is allowed to not set `file_size_` or `checksum_`, in that case it will be calculated from the data. | ||
| /// The constructor is allowed to not set `file_size_`, in that case it will be calculated from the data. | ||
| BackupEntryFromAppendOnlyFile( | ||
| const DiskPtr & disk_, | ||
| const String & file_path_, | ||
| const ReadSettings & settings_, | ||
| const std::optional<UInt64> & file_size_ = {}, | ||
| const std::optional<UInt128> & checksum_ = {}, | ||
| const std::shared_ptr<TemporaryFileOnDisk> & temporary_file_ = {}); | ||
| bool copy_encrypted_ = false, | ||
| const std::optional<UInt64> & file_size_ = {}); | ||
|
|
||
| ~BackupEntryFromAppendOnlyFile() override; | ||
|
|
||
| std::unique_ptr<SeekableReadBuffer> getReadBuffer(const ReadSettings & read_settings) const override; | ||
| UInt64 getSize() const override { return size; } | ||
|
|
||
| DataSourceDescription getDataSourceDescription() const override { return data_source_description; } | ||
| bool isEncryptedByDisk() const override { return copy_encrypted; } | ||
|
|
||
| UInt64 getSize() const override { return limit; } | ||
| std::unique_ptr<SeekableReadBuffer> getReadBuffer() const override; | ||
| bool isFromFile() const override { return true; } | ||
| DiskPtr getDisk() const override { return disk; } | ||
| String getFilePath() const override { return file_path; } | ||
|
|
||
| private: | ||
| const UInt64 limit; | ||
| const DiskPtr disk; | ||
| const String file_path; | ||
| const DataSourceDescription data_source_description; | ||
| const bool copy_encrypted; | ||
| const UInt64 size; | ||
| }; | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,53 +1,84 @@ | ||
| #include <Backups/BackupEntryFromImmutableFile.h> | ||
| #include <Disks/IDisk.h> | ||
| #include <Disks/IO/createReadBufferFromFileBase.h> | ||
| #include <Poco/File.h> | ||
| #include <Common/filesystemHelpers.h> | ||
|
|
||
|
|
||
| namespace DB | ||
| { | ||
|
|
||
| namespace | ||
| { | ||
| /// We mix the checksum calculated for non-encrypted data with IV generated to encrypt the file | ||
| /// to generate kind of a checksum for encrypted data. Of course it differs from the CityHash properly calculated for encrypted data. | ||
| UInt128 combineChecksums(UInt128 checksum1, UInt128 checksum2) | ||
| { | ||
| chassert(std::size(checksum2.items) == 2); | ||
| return CityHash_v1_0_2::CityHash128WithSeed(reinterpret_cast<const char *>(&checksum1), sizeof(checksum1), {checksum2.items[0], checksum2.items[1]}); | ||
| } | ||
| } | ||
|
|
||
| BackupEntryFromImmutableFile::BackupEntryFromImmutableFile( | ||
| const DiskPtr & disk_, | ||
| const String & file_path_, | ||
| const ReadSettings & settings_, | ||
| bool copy_encrypted_, | ||
| const std::optional<UInt64> & file_size_, | ||
| const std::optional<UInt128> & checksum_, | ||
| const std::shared_ptr<TemporaryFileOnDisk> & temporary_file_) | ||
| const std::optional<UInt128> & checksum_) | ||
| : disk(disk_) | ||
| , file_path(file_path_) | ||
| , settings(settings_) | ||
| , data_source_description(disk->getDataSourceDescription()) | ||
| , copy_encrypted(copy_encrypted_ && data_source_description.is_encrypted) | ||
| , file_size(file_size_) | ||
| , checksum(checksum_) | ||
| , temporary_file_on_disk(temporary_file_) | ||
| { | ||
| } | ||
|
|
||
| BackupEntryFromImmutableFile::~BackupEntryFromImmutableFile() = default; | ||
|
|
||
| UInt64 BackupEntryFromImmutableFile::getSize() const | ||
| std::unique_ptr<SeekableReadBuffer> BackupEntryFromImmutableFile::getReadBuffer(const ReadSettings & read_settings) const | ||
| { | ||
| std::lock_guard lock{get_file_size_mutex}; | ||
| if (!file_size) | ||
| file_size = disk->getFileSize(file_path); | ||
| return *file_size; | ||
| if (copy_encrypted) | ||
| return disk->readEncryptedFile(file_path, read_settings); | ||
| else | ||
| return disk->readFile(file_path, read_settings); | ||
| } | ||
|
|
||
| std::unique_ptr<SeekableReadBuffer> BackupEntryFromImmutableFile::getReadBuffer() const | ||
| UInt64 BackupEntryFromImmutableFile::getSize() const | ||
| { | ||
| return disk->readFile(file_path, settings); | ||
| std::lock_guard lock{size_and_checksum_mutex}; | ||
| if (!file_size_adjusted) | ||
| { | ||
| if (!file_size) | ||
| file_size = copy_encrypted ? disk->getEncryptedFileSize(file_path) : disk->getFileSize(file_path); | ||
| else if (copy_encrypted) | ||
| file_size = disk->getEncryptedFileSize(*file_size); | ||
| file_size_adjusted = true; | ||
| } | ||
| return *file_size; | ||
| } | ||
|
|
||
|
|
||
| DataSourceDescription BackupEntryFromImmutableFile::getDataSourceDescription() const | ||
| UInt128 BackupEntryFromImmutableFile::getChecksum() const | ||
| { | ||
| return disk->getDataSourceDescription(); | ||
| std::lock_guard lock{size_and_checksum_mutex}; | ||
| if (!checksum_adjusted) | ||
| { | ||
| if (!checksum) | ||
| checksum = BackupEntryWithChecksumCalculation<IBackupEntry>::getChecksum(); | ||
| else if (copy_encrypted) | ||
| checksum = combineChecksums(*checksum, disk->getEncryptedFileIV(file_path)); | ||
| checksum_adjusted = true; | ||
| } | ||
| return *checksum; | ||
| } | ||
|
|
||
| String BackupEntryFromImmutableFile::getFilePath() const | ||
| std::optional<UInt128> BackupEntryFromImmutableFile::getPartialChecksum(size_t prefix_length) const | ||
| { | ||
| return file_path; | ||
| if (prefix_length == 0) | ||
| return 0; | ||
|
|
||
| if (prefix_length >= getSize()) | ||
| return getChecksum(); | ||
|
|
||
| /// For immutable files we don't use partial checksums. | ||
| return std::nullopt; | ||
| } | ||
|
|
||
| } |
Oops, something went wrong.