Skip to content

Commit

Permalink
demo
Browse files Browse the repository at this point in the history
  • Loading branch information
BohuTANG committed Sep 18, 2020
1 parent 97baa67 commit f67d98e
Show file tree
Hide file tree
Showing 4 changed files with 14 additions and 8 deletions.
2 changes: 1 addition & 1 deletion programs/server/Server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -361,7 +361,7 @@ int Server::main(const std::vector<std::string> & /*args*/)

global_context->setPath(path);

StatusFile status{path + "status", StatusFile::write_full_info};
//StatusFile status{path + "status", StatusFile::write_full_info};

SCOPE_EXIT({
/** Ask to cancel background jobs all table engines,
Expand Down
9 changes: 5 additions & 4 deletions src/Storages/MergeTree/MergeTreeData.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1834,7 +1834,7 @@ void MergeTreeData::renameTempPartAndAdd(MutableDataPartPtr & part, SimpleIncrem

void MergeTreeData::renameTempPartAndReplace(
MutableDataPartPtr & part, SimpleIncrement * increment, Transaction * out_transaction,
std::unique_lock<std::mutex> & lock, DataPartsVector * out_covered_parts)
std::unique_lock<std::mutex> & lock, DataPartsVector * out_covered_parts, bool share_storage)
{
if (out_transaction && &out_transaction->data != this)
throw Exception("MergeTreeData::Transaction for one table cannot be used with another. It is a bug.",
Expand Down Expand Up @@ -1897,7 +1897,8 @@ void MergeTreeData::renameTempPartAndReplace(
part->info = part_info;
part->is_temp = false;
part->state = DataPartState::PreCommitted;
part->renameTo(part_name, true);
if (!share_storage)
part->renameTo(part_name, true);

auto part_it = data_parts_indexes.insert(part).first;

Expand Down Expand Up @@ -1934,7 +1935,7 @@ void MergeTreeData::renameTempPartAndReplace(
}

MergeTreeData::DataPartsVector MergeTreeData::renameTempPartAndReplace(
MutableDataPartPtr & part, SimpleIncrement * increment, Transaction * out_transaction)
MutableDataPartPtr & part, SimpleIncrement * increment, Transaction * out_transaction, bool share_storage)
{
if (out_transaction && &out_transaction->data != this)
throw Exception("MergeTreeData::Transaction for one table cannot be used with another. It is a bug.",
Expand All @@ -1943,7 +1944,7 @@ MergeTreeData::DataPartsVector MergeTreeData::renameTempPartAndReplace(
DataPartsVector covered_parts;
{
auto lock = lockParts();
renameTempPartAndReplace(part, increment, out_transaction, lock, &covered_parts);
renameTempPartAndReplace(part, increment, out_transaction, lock, &covered_parts, share_storage);
}
return covered_parts;
}
Expand Down
4 changes: 2 additions & 2 deletions src/Storages/MergeTree/MergeTreeData.h
Original file line number Diff line number Diff line change
Expand Up @@ -432,12 +432,12 @@ class MergeTreeData : public IStorage
/// Returns all parts covered by the added part (in ascending order).
/// If out_transaction == nullptr, marks covered parts as Outdated.
DataPartsVector renameTempPartAndReplace(
MutableDataPartPtr & part, SimpleIncrement * increment = nullptr, Transaction * out_transaction = nullptr);
MutableDataPartPtr & part, SimpleIncrement * increment = nullptr, Transaction * out_transaction = nullptr, bool share_storage = false);

/// Low-level version of previous one, doesn't lock mutex
void renameTempPartAndReplace(
MutableDataPartPtr & part, SimpleIncrement * increment, Transaction * out_transaction, DataPartsLock & lock,
DataPartsVector * out_covered_parts = nullptr);
DataPartsVector * out_covered_parts = nullptr, bool share_storage = false);

/// Removes parts from the working set parts.
/// Parts in add must already be in data_parts with PreCommitted, Committed, or Outdated states.
Expand Down
7 changes: 6 additions & 1 deletion src/Storages/StorageReplicatedMergeTree.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3282,7 +3282,12 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora
if (!to_detached)
{
Transaction transaction(*this);
renameTempPartAndReplace(part, nullptr, &transaction);
// share_storage
/**
* share_storage should be determined by the config:
* <share_storage>true</share_storage>
*/
renameTempPartAndReplace(part, nullptr, &transaction, true);

/** NOTE
* Here, an error occurs if ALTER occurred with a change in the column type or column deletion,
Expand Down

5 comments on commit f67d98e

@compasses
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  1. 社区有这方面的考虑吗?
  2. 首先要跑在分布式文件系统之上吧,要支持 CH 内部的file api

@BohuTANG
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  1. 这个不太清楚
  2. 是的

@godliness
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@BohuTANG
社区目前支持了replicated merge tree on s3, 和你文档中介绍的基本一样, 👍
ClickHouse#16240

@BohuTANG
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@BohuTANG
社区目前支持了replicated merge tree on s3, 和你文档中介绍的基本一样,
ClickHouse#16240

嚯。
也请多关注我们云版 Better ClickHouse项目:
https://github.com/datafuselabs/datafuse

@godliness
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@BohuTANG NICE! 好好研读一下!

Please sign in to comment.