Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Do not allow to merge data moving it against storage policy volume order #8549

Merged
merged 3 commits into from Jan 10, 2020
Merged

Do not allow to merge data moving it against storage policy volume order #8549

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7de9eb4
Do not allow to merge data moving it against storage policy volume or…
excitoon Jan 7, 2020
6f2a4bf
Added fallback behavior of reserving space on a volumes with smaller …
excitoon Jan 7, 2020
75e2dbd
Removed fallback behavior of reserving space on a volumes with smalle…
excitoon Jan 8, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
19 changes: 15 additions & 4 deletions dbms/src/Storages/MergeTree/MergeTreeData.cpp
View file
Open in desktop
Expand Up @@ -3216,18 +3216,20 @@ ReservationPtr MergeTreeData::tryReserveSpace(UInt64 expected_size, SpacePtr spa

ReservationPtr MergeTreeData::reserveSpacePreferringTTLRules(UInt64 expected_size,
const MergeTreeDataPart::TTLInfos & ttl_infos,
time_t time_of_move) const
time_t time_of_move,
size_t min_volume_index) const
{
expected_size = std::max(RESERVATION_MIN_ESTIMATION_SIZE, expected_size);

ReservationPtr reservation = tryReserveSpacePreferringTTLRules(expected_size, ttl_infos, time_of_move);
ReservationPtr reservation = tryReserveSpacePreferringTTLRules(expected_size, ttl_infos, time_of_move, min_volume_index);

return checkAndReturnReservation(expected_size, std::move(reservation));
}

ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_size,
const MergeTreeDataPart::TTLInfos & ttl_infos,
time_t time_of_move) const
time_t time_of_move,
size_t min_volume_index) const
{
expected_size = std::max(RESERVATION_MIN_ESTIMATION_SIZE, expected_size);

Expand All @@ -3253,10 +3255,19 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_
reservation = destination_ptr->reserve(expected_size);
if (reservation)
return reservation;
else
if (ttl_entry->destination_type == PartDestinationType::VOLUME)
LOG_WARNING(log, "Would like to reserve space on volume '"
<< ttl_entry->destination_name << "' by TTL rule of table '"
<< log_name << "' but there is not enough space");
else if (ttl_entry->destination_type == PartDestinationType::DISK)
LOG_WARNING(log, "Would like to reserve space on disk '"
<< ttl_entry->destination_name << "' by TTL rule of table '"
<< log_name << "' but there is not enough space");
}
}

reservation = storage_policy->reserve(expected_size);
reservation = storage_policy->reserve(expected_size, min_volume_index);

return reservation;
}
Expand Down
6 changes: 4 additions & 2 deletions dbms/src/Storages/MergeTree/MergeTreeData.h
View file
Open in desktop
Expand Up @@ -675,10 +675,12 @@ class MergeTreeData : public IStorage
/// Reserves space at least 1MB preferring best destination according to `ttl_infos`.
ReservationPtr reserveSpacePreferringTTLRules(UInt64 expected_size,
const MergeTreeDataPart::TTLInfos & ttl_infos,
time_t time_of_move) const;
time_t time_of_move,
size_t min_volume_index = 0) const;
ReservationPtr tryReserveSpacePreferringTTLRules(UInt64 expected_size,
const MergeTreeDataPart::TTLInfos & ttl_infos,
time_t time_of_move) const;
time_t time_of_move,
size_t min_volume_index = 0) const;
/// Choose disk with max available free space
/// Reserves 0 bytes
ReservationPtr makeEmptyReservationOnLargestDisk() { return storage_policy->makeEmptyReservationOnLargestDisk(); }
Expand Down
6 changes: 5 additions & 1 deletion dbms/src/Storages/StorageMergeTree.cpp
View file
Open in desktop
Expand Up @@ -328,10 +328,14 @@ struct CurrentlyMergingPartsTagger
else
{
MergeTreeDataPart::TTLInfos ttl_infos;
size_t max_volume_index = 0;
for (auto & part_ptr : future_part_.parts)
{
ttl_infos.update(part_ptr->ttl_infos);
max_volume_index = std::max(max_volume_index, storage.getStoragePolicy()->getVolumeIndexByDisk(part_ptr->disk));
}

reserved_space = storage.tryReserveSpacePreferringTTLRules(total_size, ttl_infos, time(nullptr));
reserved_space = storage.tryReserveSpacePreferringTTLRules(total_size, ttl_infos, time(nullptr), max_volume_index);
}
if (!reserved_space)
{
Expand Down
4 changes: 3 additions & 1 deletion dbms/src/Storages/StorageReplicatedMergeTree.cpp
View file
Open in desktop
Expand Up @@ -1053,12 +1053,14 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry)

/// Can throw an exception while reserving space.
MergeTreeDataPart::TTLInfos ttl_infos;
size_t max_volume_index = 0;
for (auto & part_ptr : parts)
{
ttl_infos.update(part_ptr->ttl_infos);
max_volume_index = std::max(max_volume_index, getStoragePolicy()->getVolumeIndexByDisk(part_ptr->disk));
}
ReservationPtr reserved_space = reserveSpacePreferringTTLRules(estimated_space_for_merge,
ttl_infos, time(nullptr));
ttl_infos, time(nullptr), max_volume_index);

auto table_lock = lockStructureForShare(false, RWLockImpl::NO_QUERY);

Expand Down