Skip to content

Commit

Permalink
Merge branch 'master' into add-read-system-table-test
Browse files Browse the repository at this point in the history
  • Loading branch information
@Lloyd-Pottiger
Lloyd-Pottiger committed Apr 10, 2023
2 parents 501b313 + e0c82c0 commit a89fb8c
Show file tree
Hide file tree
Showing 56 changed files with 1,834 additions and 958 deletions.
2 changes: 1 addition & 1 deletion contrib/tiflash-proxy
Submodule tiflash-proxy updated 5 files
+5 −1 proxy_components/proxy_ffi/src/basic_ffi_impls.rs
+780 −0 proxy_components/proxy_server/src/common.rs
+236 −0 proxy_components/proxy_server/src/common_override.rs
+4 −0 proxy_components/proxy_server/src/lib.rs
+168 −671 proxy_components/proxy_server/src/run.rs
8 changes: 8 additions & 0 deletions dbms/src/Columns/ColumnUtils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.

#include <Columns/ColumnNullable.h>
#include <Columns/ColumnUtils.h>
#include <DataTypes/DataTypeNullable.h>

namespace DB
{
Expand All @@ -30,4 +32,10 @@ bool columnEqual(const ColumnPtr & expected, const ColumnPtr & actual, String &
}
return true;
}
void convertColumnToNullable(ColumnWithTypeAndName & column)
{
column.type = makeNullable(column.type);
if (column.column)
column.column = makeNullable(column.column);
}
} // namespace DB
2 changes: 2 additions & 0 deletions dbms/src/Columns/ColumnUtils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@
#pragma once

#include <Columns/IColumn.h>
#include <Core/ColumnWithTypeAndName.h>

namespace DB
{
bool columnEqual(const ColumnPtr & expected, const ColumnPtr & actual, String & unequal_msg);
void convertColumnToNullable(ColumnWithTypeAndName & column);
} // namespace DB
2 changes: 1 addition & 1 deletion dbms/src/Common/ComputeLabelHolder.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ void ComputeLabelHolder::init(const Poco::Util::LayeredConfiguration & conf)
std::lock_guard lock(mu);
RUNTIME_ASSERT(!label_got, log, "Can't init after label got");
cluster_id = conf.getString(cluster_id_key, "unknown");
auto service_addr = conf.getString("flash.service_addr", "unknown");
auto service_addr = conf.getString("flash.service_addr", "0.0.0.0:3930");
std::replace(service_addr.begin(), service_addr.end(), ':', '_');
std::replace(service_addr.begin(), service_addr.end(), '.', '_');
process_id = fmt::format("compute_{}_{}", service_addr, microsecondsUTC());
Expand Down
8 changes: 0 additions & 8 deletions dbms/src/Common/Config/ConfigProcessor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,11 +84,3 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfig()

return LoadedConfig{configuration, false, config_doc};
}

void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config)
{
std::ofstream out(preprocessed_path);
cpptoml::toml_writer writer(out);
loaded_config.preprocessed_conf->accept(std::move(writer));
out.close();
}
2 changes: 0 additions & 2 deletions dbms/src/Common/Config/ConfigProcessor.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,6 @@ class ConfigProcessor

LoadedConfig loadConfig();

void savePreprocessedConfig(const LoadedConfig & loaded_config);

public:
/// Is the file named as result of config preprocessing, not as original files.
static bool isPreprocessedFile(const std::string & config_path);
Expand Down
6 changes: 3 additions & 3 deletions dbms/src/Common/Config/ConfigReloader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,9 @@ void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error)
}
}

if (force || new_files.isDifferOrNewerThan(files) || config_object_updated)
if (force
|| (new_files.valid() && new_files.isDifferOrNewerThan(files))
|| config_object_updated)
{
ConfigProcessor config_processor(path);
ConfigProcessor::LoadedConfig loaded_config;
Expand All @@ -103,8 +105,6 @@ void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error)
return;
}

config_processor.savePreprocessedConfig(loaded_config);

/** We should remember last modification time if and only if config was sucessfully loaded
* Otherwise a race condition could occur during config files update:
* File is contain raw (and non-valid) data, therefore config is not applied.
Expand Down
5 changes: 5 additions & 0 deletions dbms/src/Common/FileChangesTracker.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@ struct FilesChangesTracker
{
std::set<FileWithTimestamp> files;

bool valid() const
{
return !files.empty();
}

void addIfExists(const std::string & path)
{
if (!path.empty() && Poco::File(path).exists())
Expand Down
33 changes: 26 additions & 7 deletions dbms/src/Common/TiFlashMetrics.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,8 +218,8 @@ namespace DB
F(type_lock_conflict, {"type", "lock_conflict"}), F(type_delete_conflict, {"type", "delete_conflict"}), \
F(type_delete_risk, {"type", "delete_risk"})) \
M(tiflash_disaggregated_object_lock_request_duration_seconds, "Bucketed histogram of S3 object lock/delete request duration", Histogram, \
F(type_lock, {{"type", "cop"}}, ExpBuckets{0.001, 2, 20}), \
F(type_delete, {{"type", "batch"}}, ExpBuckets{0.001, 2, 20})) \
F(type_lock, {{"type", "lock"}}, ExpBuckets{0.001, 2, 20}), \
F(type_delete, {{"type", "delete"}}, ExpBuckets{0.001, 2, 20})) \
M(tiflash_disaggregated_read_tasks_count, "Total number of storage engine disaggregated read tasks", Counter) \
M(tiflash_disaggregated_breakdown_duration_seconds, "", Histogram, \
F(type_rpc_establish, {{"type", "rpc_establish"}}, ExpBuckets{0.01, 2, 20}), \
Expand Down Expand Up @@ -319,8 +319,26 @@ namespace DB
M(tiflash_storage_remote_stats, "The file stats on remote store", Gauge, \
F(type_total_size, {"type", "total_size"}), F(type_valid_size, {"type", "valid_size"}), \
F(type_num_files, {"type", "num_files"})) \
M(tiflash_storage_checkpoint_seconds, "PageStorage checkpoint elapsed time", \
Histogram, /* these command usually cost several seconds, increase the start bucket to 50ms */ \
F(type_dump_checkpoint_snapshot, {{"type", "dump_checkpoint_snapshot"}}, ExpBuckets{0.05, 2, 20}), \
F(type_dump_checkpoint_data, {{"type", "dump_checkpoint_data"}}, ExpBuckets{0.05, 2, 20}), \
F(type_upload_checkpoint, {{"type", "upload_checkpoint"}}, ExpBuckets{0.05, 2, 20}), \
F(type_copy_checkpoint_info, {{"type", "copy_checkpoint_info"}}, ExpBuckets{0.05, 2, 20})) \
M(tiflash_storage_checkpoint_flow, "The bytes flow cause by remote checkpoint", Counter, \
F(type_incremental, {"type", "incremental"}), F(type_compaction, {"type", "compaction"})) \
M(tiflash_storage_checkpoint_keys_by_types, "The keys flow cause by remote checkpoint", Counter, \
F(type_raftengine, {"type", "raftengine"}), F(type_kvengine, {"type", "kvengine"}), F(type_kvstore, {"type", "kvstore"}), \
F(type_data, {"type", "data"}), F(type_log, {"type", "log"}), F(type_meta, {"type", "kvstore"}), \
F(type_unknown, {"type", "unknown"})) \
M(tiflash_storage_checkpoint_flow_by_types, "The bytes flow cause by remote checkpoint", Counter, \
F(type_raftengine, {"type", "raftengine"}), F(type_kvengine, {"type", "kvengine"}), F(type_kvstore, {"type", "kvstore"}), \
F(type_data, {"type", "data"}), F(type_log, {"type", "log"}), F(type_meta, {"type", "kvstore"}), \
F(type_unknown, {"type", "unknown"})) \
M(tiflash_storage_page_data_by_types, "The existing bytes stored in UniPageStorage", Gauge, \
F(type_raftengine, {"type", "raftengine"}), F(type_kvengine, {"type", "kvengine"}), F(type_kvstore, {"type", "kvstore"}), \
F(type_data, {"type", "data"}), F(type_log, {"type", "log"}), F(type_meta, {"type", "kvstore"}), \
F(type_unknown, {"type", "unknown"})) \
M(tiflash_storage_s3_request_seconds, "S3 request duration in seconds", Histogram, \
F(type_put_object, {{"type", "put_object"}}, ExpBuckets{0.001, 2, 20}), \
F(type_copy_object, {{"type", "copy_object"}}, ExpBuckets{0.001, 2, 20}), \
Expand All @@ -331,6 +349,11 @@ namespace DB
F(type_list_objects, {{"type", "list_objects"}}, ExpBuckets{0.001, 2, 20}), \
F(type_delete_object, {{"type", "delete_object"}}, ExpBuckets{0.001, 2, 20}), \
F(type_head_object, {{"type", "head_object"}}, ExpBuckets{0.001, 2, 20})) \
M(tiflash_storage_s3_gc_status, "S3 GC status", Gauge, \
F(type_lifecycle_added, {{"type", "lifecycle_added"}}), \
F(type_lifecycle_failed, {{"type", "lifecycle_failed"}}), \
F(type_owner, {{"type", "owner"}}), \
F(type_running, {{"type", "running"}})) \
M(tiflash_storage_s3_gc_seconds, "S3 GC subprocess duration in seconds", \
Histogram, /* these command usually cost several seconds, increase the start bucket to 500ms */ \
F(type_total, {{"type", "total"}}, ExpBuckets{0.5, 2, 20}), \
Expand All @@ -339,11 +362,7 @@ namespace DB
F(type_clean_locks, {{"type", "clean_locks"}}, ExpBuckets{0.5, 2, 20}), \
F(type_clean_manifests, {{"type", "clean_manifests"}}, ExpBuckets{0.5, 2, 20}), \
F(type_scan_then_clean_data_files, {{"type", "scan_then_clean_data_files"}}, ExpBuckets{0.5, 2, 20}), \
F(type_clean_one_lock, {{"type", "clean_one_lock"}}, ExpBuckets{0.5, 2, 20})) \
M(tiflash_storage_checkpoint_seconds, "PageStorage checkpoint elapsed time", Histogram, \
F(type_dump_checkpoint_snapshot, {{"type", "dump_checkpoint_snapshot"}}, ExpBuckets{0.001, 2, 20}), \
F(type_dump_checkpoint_data, {{"type", "dump_checkpoint_data"}}, ExpBuckets{0.001, 2, 20}), \
F(type_upload_checkpoint, {{"type", "upload_checkpoint"}}, ExpBuckets{0.001, 2, 20}))
F(type_clean_one_lock, {{"type", "clean_one_lock"}}, ExpBuckets{0.5, 2, 20}))

// clang-format on

Expand Down

0 comments on commit a89fb8c

Please sign in to comment.