diff --git a/be/src/olap/olap_meta.cpp b/be/src/olap/olap_meta.cpp index 4d6b84e634ecaf..28ad378de8b053 100755 --- a/be/src/olap/olap_meta.cpp +++ b/be/src/olap/olap_meta.cpp @@ -55,10 +55,10 @@ OlapMeta::~OlapMeta() { for (auto handle : _handles) { delete handle; } - if (_db != NULL) { + if (_db != nullptr) { _db->Close(); delete _db; - _db= NULL; + _db= nullptr; } } @@ -79,7 +79,7 @@ OLAPStatus OlapMeta::init() { meta_column_family.prefix_extractor.reset(NewFixedPrefixTransform(PREFIX_LENGTH)); column_families.emplace_back(META_COLUMN_FAMILY, meta_column_family); Status s = DB::Open(options, db_path, column_families, &_handles, &_db); - if (!s.ok() || _db == NULL) { + if (!s.ok() || _db == nullptr) { LOG(WARNING) << "rocks db open failed, reason:" << s.ToString(); return OLAP_ERR_META_OPEN_DB; } diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 48d530e36f1b2a..602c7bef8b1fd8 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -45,12 +45,13 @@ OLAPStatus StorageEngine::_start_bg_worker() { [this] { _garbage_sweeper_thread_callback(nullptr); }); - + _garbage_sweeper_thread.detach(); // start thread for monitoring the tablet with io error _disk_stat_monitor_thread = std::thread( [this] { _disk_stat_monitor_thread_callback(nullptr); }); + _disk_stat_monitor_thread.detach(); // start be and ce threads for merge data int32_t base_compaction_num_threads = config::base_compaction_num_threads; @@ -61,6 +62,9 @@ OLAPStatus StorageEngine::_start_bg_worker() { _base_compaction_thread_callback(nullptr); }); } + for (auto& thread : _base_compaction_threads) { + thread.detach(); + } int32_t cumulative_compaction_num_threads = config::cumulative_compaction_num_threads; _cumulative_compaction_threads.reserve(cumulative_compaction_num_threads); @@ -70,11 +74,15 @@ OLAPStatus StorageEngine::_start_bg_worker() { _cumulative_compaction_thread_callback(nullptr); }); } + for (auto& thread : _cumulative_compaction_threads) { + thread.detach(); + } _fd_cache_clean_thread = std::thread( [this] { _fd_cache_clean_callback(nullptr); }); + _fd_cache_clean_thread.detach(); // path scan and gc thread if (config::path_gc_check) { @@ -89,6 +97,12 @@ OLAPStatus StorageEngine::_start_bg_worker() { _path_gc_thread_callback((void*)data_dir); }); } + for (auto& thread : _path_scan_threads) { + thread.detach(); + } + for (auto& thread : _path_gc_threads) { + thread.detach(); + } } VLOG(10) << "init finished."; diff --git a/be/src/olap/row_block.h b/be/src/olap/row_block.h index 19f86f8c5437ca..f33916088f595c 100644 --- a/be/src/olap/row_block.h +++ b/be/src/olap/row_block.h @@ -186,7 +186,7 @@ class RowBlock { uint32_t _capacity; RowBlockInfo _info; const TabletSchema* _schema; // 内部保存的schema句柄 - + bool _null_supported; size_t _field_count = 0; diff --git a/be/src/olap/row_cursor.cpp b/be/src/olap/row_cursor.cpp index 4410c1a541bc86..e590e45d30717f 100644 --- a/be/src/olap/row_cursor.cpp +++ b/be/src/olap/row_cursor.cpp @@ -81,8 +81,8 @@ OLAPStatus RowCursor::_init(const std::vector& schema, for (auto cid : _columns) { const TabletColumn& column = schema[cid]; _field_array[cid] = Field::create(column); - if (_field_array[cid] == NULL) { - OLAP_LOG_WARNING("Fail to create field."); + if (_field_array[cid] == nullptr) { + LOG(WARNING) << "Fail to create field."; return OLAP_ERR_INIT_FAILED; } _fixed_len += field_buf_lens[cid] + 1; //1 for null byte @@ -99,7 +99,7 @@ OLAPStatus RowCursor::_init(const std::vector& schema, _fixed_buf = new (nothrow) char[_fixed_len]; if (_fixed_buf == nullptr) { - OLAP_LOG_WARNING("Fail to malloc _fixed_buf."); + LOG(WARNING) << "Fail to malloc _fixed_buf."; return OLAP_ERR_MALLOC_ERROR; } _owned_fixed_buf = _fixed_buf; @@ -194,8 +194,8 @@ OLAPStatus RowCursor::init_scan_key(const TabletSchema& schema, } // variable_len for null bytes - _variable_buf = new (nothrow) char[_variable_len]; - if (_variable_buf == NULL) { + _variable_buf = new(nothrow) char[_variable_len]; + if (_variable_buf == nullptr) { OLAP_LOG_WARNING("Fail to malloc _variable_buf."); return OLAP_ERR_MALLOC_ERROR; } @@ -304,7 +304,7 @@ int RowCursor::cmp(const RowCursor& other) const { size_t common_prefix_count = min(_key_column_num, other._key_column_num); // 只有key column才会参与比较 for (size_t i = 0; i < common_prefix_count; ++i) { - if (_field_array[i] == NULL || other._field_array[i] == NULL) { + if (_field_array[i] == nullptr || other._field_array[i] == nullptr) { continue; } @@ -325,7 +325,7 @@ int RowCursor::index_cmp(const RowCursor& other) const { size_t common_prefix_count = min(_columns.size(), other._key_column_num); // 只有key column才会参与比较 for (size_t i = 0; i < common_prefix_count; ++i) { - if (_field_array[i] == NULL || other._field_array[i] == NULL) { + if (_field_array[i] == nullptr || other._field_array[i] == nullptr) { continue; } char* left = _field_array[i]->get_field_ptr(_fixed_buf); @@ -343,7 +343,7 @@ bool RowCursor::equal(const RowCursor& other) const { // 按field顺序从后往前比较，有利于尽快发现不同，提升比较性能 size_t common_prefix_count = min(_key_column_num, other._key_column_num); for (int i = common_prefix_count - 1; i >= 0; --i) { - if (_field_array[i] == NULL || other._field_array[i] == NULL) { + if (_field_array[i] == nullptr || other._field_array[i] == nullptr) { continue; } char* left = _field_array[i]->get_field_ptr(_fixed_buf); @@ -357,7 +357,7 @@ bool RowCursor::equal(const RowCursor& other) const { void RowCursor::finalize_one_merge() { for (size_t i = _key_column_num; i < _field_array.size(); ++i) { - if (_field_array[i] == NULL) { + if (_field_array[i] == nullptr) { continue; } char* dest = _field_array[i]->get_ptr(_fixed_buf); @@ -368,7 +368,7 @@ void RowCursor::finalize_one_merge() { void RowCursor::aggregate(const RowCursor& other) { // 只有value column才会参与aggregate for (size_t i = _key_column_num; i < _field_array.size(); ++i) { - if (_field_array[i] == NULL || other._field_array[i] == NULL) { + if (_field_array[i] == nullptr || other._field_array[i] == nullptr) { continue; } @@ -429,7 +429,7 @@ OlapTuple RowCursor::to_tuple() const { OlapTuple tuple; for (auto cid : _columns) { - if (_field_array[cid] != NULL) { + if (_field_array[cid] != nullptr) { Field* field = _field_array[cid]; char* src = field->get_ptr(_fixed_buf); if (field->is_null(_fixed_buf)) { @@ -476,7 +476,7 @@ string RowCursor::to_string(string sep) const { } Field* field = _field_array[cid]; - if (field != NULL) { + if (field != nullptr) { char* src = field->get_ptr(_fixed_buf); result.append(field->to_string(src)); } else { @@ -489,7 +489,7 @@ string RowCursor::to_string(string sep) const { OLAPStatus RowCursor::get_first_different_column_id(const RowCursor& other, size_t* first_diff_id) const { - if (first_diff_id == NULL) { + if (first_diff_id == nullptr) { OLAP_LOG_WARNING("input parameter 'first_diff_id' is NULL."); return OLAP_ERR_INPUT_PARAMETER_ERROR; } @@ -501,7 +501,7 @@ OLAPStatus RowCursor::get_first_different_column_id(const RowCursor& other, size_t i = 0; for (; i < _field_array.size(); ++i) { - if (_field_array[i] == NULL || other._field_array[i] == NULL) { + if (_field_array[i] == nullptr || other._field_array[i] == nullptr) { continue; } diff --git a/be/src/olap/row_cursor.h b/be/src/olap/row_cursor.h index 3101addf2c23e6..3066a4213988f4 100644 --- a/be/src/olap/row_cursor.h +++ b/be/src/olap/row_cursor.h @@ -78,13 +78,10 @@ class RowCursor { OLAPStatus init_scan_key(const TabletSchema& schema, const std::vector& keys); - OLAPStatus init_scan_key(const TabletSchema& schema, - const std::vector& field_lengths); - //allocate memory for string type, which include char, varchar, hyperloglog OLAPStatus allocate_memory_for_string_type(const TabletSchema& schema, MemPool* mem_pool = nullptr); - + // 两个RowCurosr做比较，返回-1，0，1 int cmp(const RowCursor& other) const; diff --git a/be/src/olap/rowset/column_data.cpp b/be/src/olap/rowset/column_data.cpp index 08e7151e187aed..302d879cd3ec5d 100644 --- a/be/src/olap/rowset/column_data.cpp +++ b/be/src/olap/rowset/column_data.cpp @@ -449,7 +449,7 @@ void ColumnData::set_read_params( auto res = _cursor.init(_segment_group->get_tablet_schema()); if (res != OLAP_SUCCESS) { - OLAP_LOG_WARNING("fail to init row_cursor"); + LOG(WARNING) << "fail to init row_cursor"; } _read_vector_batch.reset(new VectorizedRowBatch( diff --git a/be/src/olap/rowset/column_reader.cpp b/be/src/olap/rowset/column_reader.cpp index 5ecaef444477d2..cba9c2fa6f8709 100644 --- a/be/src/olap/rowset/column_reader.cpp +++ b/be/src/olap/rowset/column_reader.cpp @@ -485,18 +485,26 @@ ColumnReader::ColumnReader(uint32_t column_id, uint32_t column_unique_id) : _present_reader(NULL) { } -ColumnReader* ColumnReader::create(uint32_t column_id, + ColumnReader* ColumnReader::create(uint32_t column_id, const TabletSchema& schema, const UniqueIdToColumnIdMap& included, UniqueIdToColumnIdMap& segment_included, const UniqueIdEncodingMap& encodings) { - if (column_id >= schema.num_columns()) { + return create(column_id, schema.columns(), included, segment_included, encodings); +} + +ColumnReader* ColumnReader::create(uint32_t column_id, + const std::vector& schema, + const UniqueIdToColumnIdMap& included, + UniqueIdToColumnIdMap& segment_included, + const UniqueIdEncodingMap& encodings) { + if (column_id >= schema.size()) { LOG(WARNING) << "invalid column_id, column_id=" << column_id - << ", columns_size=" << schema.num_columns(); + << ", columns_size=" << schema.size(); return NULL; } - const TabletColumn& column = schema.column(column_id); + const TabletColumn& column = schema[column_id]; ColumnReader* reader = NULL; int32_t column_unique_id = column.unique_id(); diff --git a/be/src/olap/rowset/column_reader.h b/be/src/olap/rowset/column_reader.h index e5ad43a761f20c..8d928526e2233e 100644 --- a/be/src/olap/rowset/column_reader.h +++ b/be/src/olap/rowset/column_reader.h @@ -187,6 +187,12 @@ class ColumnReader { const UniqueIdToColumnIdMap& included, UniqueIdToColumnIdMap& segment_included, const UniqueIdEncodingMap& encodings); + + static ColumnReader* create(uint32_t column_id, + const std::vector& schema, + const UniqueIdToColumnIdMap& included, + UniqueIdToColumnIdMap& segment_included, + const UniqueIdEncodingMap& encodings); ColumnReader(uint32_t column_id, uint32_t column_unique_id); virtual ~ColumnReader(); diff --git a/be/src/olap/rowset/column_writer.cpp b/be/src/olap/rowset/column_writer.cpp index 110bc6f2d969ee..b8fa9ef2b00614 100755 --- a/be/src/olap/rowset/column_writer.cpp +++ b/be/src/olap/rowset/column_writer.cpp @@ -27,7 +27,7 @@ ColumnWriter* ColumnWriter::create(uint32_t column_id, OutStreamFactory* stream_factory, size_t num_rows_per_row_block, double bf_fpp) { - ColumnWriter* column_writer = NULL; + ColumnWriter* column_writer = nullptr; const TabletColumn& column = schema.column(column_id); switch (column.type()) { diff --git a/be/src/olap/rowset/column_writer.h b/be/src/olap/rowset/column_writer.h index 6613770c4ce737..fa3fb77b922830 100644 --- a/be/src/olap/rowset/column_writer.h +++ b/be/src/olap/rowset/column_writer.h @@ -57,6 +57,7 @@ class ColumnWriter { OutStreamFactory* stream_factory, size_t num_rows_per_row_block, double bf_fpp); + virtual ~ColumnWriter(); virtual OLAPStatus init(); diff --git a/be/src/olap/rowset/rowset_meta_manager.cpp b/be/src/olap/rowset/rowset_meta_manager.cpp index af63847dd4fc19..74a00594ddc60f 100644 --- a/be/src/olap/rowset/rowset_meta_manager.cpp +++ b/be/src/olap/rowset/rowset_meta_manager.cpp @@ -55,13 +55,12 @@ OLAPStatus RowsetMetaManager::get_rowset_meta(OlapMeta* meta, int64_t rowset_id, } OLAPStatus RowsetMetaManager::get_json_rowset_meta(OlapMeta* meta, int64_t rowset_id, std::string* json_rowset_meta) { - RowsetMeta rowset_meta; - RowsetMetaSharedPtr rowset_meta_ptr(&rowset_meta); + RowsetMetaSharedPtr rowset_meta_ptr(new(std::nothrow) RowsetMeta()); OLAPStatus status = get_rowset_meta(meta, rowset_id, rowset_meta_ptr); if (status != OLAP_SUCCESS) { return status; } - bool ret = rowset_meta.json_rowset_meta(json_rowset_meta); + bool ret = rowset_meta_ptr->json_rowset_meta(json_rowset_meta); if (!ret) { std::string error_msg = "get json rowset meta failed. rowset id:" + std::to_string(rowset_id); return OLAP_ERR_SERIALIZE_PROTOBUF_ERROR; @@ -69,7 +68,7 @@ OLAPStatus RowsetMetaManager::get_json_rowset_meta(OlapMeta* meta, int64_t rowse return OLAP_SUCCESS; } -OLAPStatus RowsetMetaManager::save(OlapMeta* meta, int64_t rowset_id, RowsetMetaSharedPtr rowset_meta) { +OLAPStatus RowsetMetaManager::save(OlapMeta* meta, int64_t rowset_id, RowsetMeta* rowset_meta) { std::string key = ROWSET_PREFIX + std::to_string(rowset_id); std::string value; bool ret = rowset_meta->serialize(&value); @@ -130,8 +129,7 @@ OLAPStatus RowsetMetaManager::load_json_rowset_meta(OlapMeta* meta, const std::s return OLAP_ERR_SERIALIZE_PROTOBUF_ERROR; } uint64_t rowset_id = rowset_meta.rowset_id(); - RowsetMetaSharedPtr rowset_meta_ptr(&rowset_meta); - OLAPStatus status = save(meta, rowset_id, rowset_meta_ptr); + OLAPStatus status = save(meta, rowset_id, &rowset_meta); return status; } diff --git a/be/src/olap/rowset/rowset_meta_manager.h b/be/src/olap/rowset/rowset_meta_manager.h index 14c207d89b0af1..67d52bfa7b69f5 100644 --- a/be/src/olap/rowset/rowset_meta_manager.h +++ b/be/src/olap/rowset/rowset_meta_manager.h @@ -35,7 +35,7 @@ class RowsetMetaManager { static OLAPStatus get_json_rowset_meta(OlapMeta* meta, int64_t rowset_id, std::string* json_rowset_meta); - static OLAPStatus save(OlapMeta* meta, int64_t rowset_id, RowsetMetaSharedPtr rowset_meta); + static OLAPStatus save(OlapMeta* meta, int64_t rowset_id, RowsetMeta* rowset_meta); static OLAPStatus save(OlapMeta* meta, int64_t rowset_id, const string& meta_binary); diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index b7e61a41430bc3..9004a6755182f0 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -96,7 +96,6 @@ Tablet::Tablet(TabletMeta* tablet_meta, DataDir* data_dir) _tablet_meta(tablet_meta), _schema(tablet_meta->tablet_schema()), _data_dir(data_dir) { - _is_dropped = false; _tablet_meta->set_data_dir(_data_dir); _tablet_path.append(_data_dir->path()); @@ -455,6 +454,10 @@ OLAPStatus Tablet::capture_rs_readers(const vector& version_path, return OLAP_SUCCESS; } +OLAPStatus Tablet::add_delete_predicate(const DeletePredicatePB& delete_predicate, int64_t version) { + return _tablet_meta->add_delete_predicate(delete_predicate, version); +} + bool Tablet::version_for_delete_predicate(const Version& version) { return _tablet_meta->version_for_delete_predicate(version); } diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index c1d2899ece6e69..aee6a446a3bbeb 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -69,7 +69,6 @@ class Tablet : public std::enable_shared_from_this { // operation for TabletState TabletState tablet_state() const { return _state; } inline OLAPStatus set_tablet_state(TabletState state); - void mark_dropped() { _is_dropped = true; } // Property encapsulated in TabletMeta inline const TabletMeta& tablet_meta(); @@ -142,6 +141,7 @@ class Tablet : public std::enable_shared_from_this { vector* rs_readers) const; DelPredicateArray delete_predicates() { return _tablet_meta->delete_predicates(); } + OLAPStatus add_delete_predicate(const DeletePredicatePB& delete_predicate, int64_t version); bool version_for_delete_predicate(const Version& version); bool version_for_load_deletion(const Version& version); @@ -222,7 +222,6 @@ class Tablet : public std::enable_shared_from_this { std::string _tablet_path; RowsetGraph _rs_graph; - bool _is_dropped; DorisInitOnce _init_once; RWMutex _meta_lock; Mutex _ingest_lock; diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index 6d7eb493cc4a88..ff5c176d7adfb4 100755 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -138,7 +138,6 @@ OLAPStatus TabletManager::add_tablet(TTabletId tablet_id, SchemaHash schema_hash _tablet_map[tablet_id].table_arr.sort(_sort_tablet_by_creation_time); _tablet_map_lock.unlock(); } else { - tablet->mark_dropped(); res = OLAP_ERR_ENGINE_INSERT_EXISTS_TABLE; } LOG(WARNING) << "add duplicated tablet. force=" << force << ", res=" << res @@ -707,7 +706,6 @@ OLAPStatus TabletManager::load_tablet_from_meta(DataDir* data_dir, TTabletId tab LOG(WARNING) << "tablet not in schema change state without delta is invalid." << "tablet=" << tablet->full_name(); // tablet state is invalid, drop tablet - tablet->mark_dropped(); return OLAP_ERR_TABLE_INDEX_VALIDATE_ERROR; } @@ -1096,10 +1094,12 @@ OLAPStatus TabletManager::_drop_tablet_directly_unlocked( for (list::iterator it = _tablet_map[tablet_id].table_arr.begin(); it != _tablet_map[tablet_id].table_arr.end();) { if ((*it)->equal(tablet_id, schema_hash)) { + TabletSharedPtr tablet = *it; + it = _tablet_map[tablet_id].table_arr.erase(it); if (!keep_files) { - (*it)->mark_dropped(); + LOG(INFO) << "remove tablet:" << tablet_id << " path:" << tablet->tablet_path(); + boost::filesystem::remove_all(tablet->tablet_path()); } - it = _tablet_map[tablet_id].table_arr.erase(it); } else { ++it; } diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 4d1da4d02913e6..02502c96107f0e 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -236,6 +236,7 @@ OLAPStatus TabletMeta::deserialize(const string& meta_binary) { OLAPStatus TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) { // init _tablet_state + _tablet_meta_pb = tablet_meta_pb; switch (tablet_meta_pb.tablet_state()) { case PB_NOTREADY: _tablet_state = TabletState::TABLET_NOTREADY; @@ -264,14 +265,14 @@ OLAPStatus TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) { for (auto& it : tablet_meta_pb.rs_metas()) { RowsetMetaSharedPtr rs_meta(new AlphaRowsetMeta()); rs_meta->init_from_pb(it); - if (rs_meta->has_delete_predicate()) { - add_delete_predicate(rs_meta->delete_predicate(), rs_meta->version().first); - } _rs_metas.push_back(std::move(rs_meta)); } for (auto& it : tablet_meta_pb.inc_rs_metas()) { RowsetMetaSharedPtr rs_meta(new AlphaRowsetMeta()); rs_meta->init_from_pb(it); + if (rs_meta->has_delete_predicate()) { + add_delete_predicate(rs_meta->delete_predicate(), rs_meta->version().first); + } _inc_rs_metas.push_back(std::move(rs_meta)); } diff --git a/be/src/olap/tablet_meta_manager.cpp b/be/src/olap/tablet_meta_manager.cpp index 9ac3945c4b8e18..3e80a87b4691be 100755 --- a/be/src/olap/tablet_meta_manager.cpp +++ b/be/src/olap/tablet_meta_manager.cpp @@ -76,7 +76,7 @@ OLAPStatus TabletMetaManager::get_json_header(DataDir* store, } OLAPStatus TabletMetaManager::save(DataDir* store, - TTabletId tablet_id, TSchemaHash schema_hash, const TabletMeta* tablet_meta, string header_prefix) { + TTabletId tablet_id, TSchemaHash schema_hash, const TabletMeta* tablet_meta, const string& header_prefix) { std::stringstream key_stream; key_stream << header_prefix << tablet_id << "_" << schema_hash; std::string key = key_stream.str(); @@ -87,7 +87,7 @@ OLAPStatus TabletMetaManager::save(DataDir* store, } OLAPStatus TabletMetaManager::save(DataDir* store, - TTabletId tablet_id, TSchemaHash schema_hash, const std::string& meta_binary, string header_prefix) { + TTabletId tablet_id, TSchemaHash schema_hash, const std::string& meta_binary, const string& header_prefix) { std::stringstream key_stream; key_stream << header_prefix << tablet_id << "_" << schema_hash; std::string key = key_stream.str(); @@ -96,7 +96,8 @@ OLAPStatus TabletMetaManager::save(DataDir* store, return meta->put(META_COLUMN_FAMILY_INDEX, key, meta_binary); } -OLAPStatus TabletMetaManager::remove(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash, string header_prefix) { +OLAPStatus TabletMetaManager::remove(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash, + const string& header_prefix) { std::stringstream key_stream; key_stream << header_prefix << tablet_id << "_" << schema_hash; std::string key = key_stream.str(); @@ -108,7 +109,7 @@ OLAPStatus TabletMetaManager::remove(DataDir* store, TTabletId tablet_id, TSchem } OLAPStatus TabletMetaManager::traverse_headers(OlapMeta* meta, - std::function const& func, string header_prefix) { + std::function const& func, const string& header_prefix) { auto traverse_header_func = [&func](const std::string& key, const std::string& value) -> bool { std::vector parts; // key format: "hdr_" + tablet_id + "_" + schema_hash @@ -117,8 +118,8 @@ OLAPStatus TabletMetaManager::traverse_headers(OlapMeta* meta, LOG(WARNING) << "invalid tablet_meta key:" << key << ", splitted size:" << parts.size(); return true; } - TTabletId tablet_id = std::stol(parts[1].c_str(), NULL, 10); - TSchemaHash schema_hash = std::stol(parts[2].c_str(), NULL, 10); + TTabletId tablet_id = std::stol(parts[1].c_str(), nullptr, 10); + TSchemaHash schema_hash = std::stol(parts[2].c_str(), nullptr, 10); return func(tablet_id, schema_hash, value); }; OLAPStatus status = meta->iterate(META_COLUMN_FAMILY_INDEX, header_prefix, traverse_header_func); diff --git a/be/src/olap/tablet_meta_manager.h b/be/src/olap/tablet_meta_manager.h index 2533a1361d5a90..a7d037d0dda466 100644 --- a/be/src/olap/tablet_meta_manager.h +++ b/be/src/olap/tablet_meta_manager.h @@ -39,15 +39,15 @@ class TabletMetaManager { TSchemaHash schema_hash, std::string* json_header); static OLAPStatus save(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash, - const TabletMeta* tablet_meta, string header_prefix = "tabletmeta_"); + const TabletMeta* tablet_meta, const string& header_prefix = "tabletmeta_"); static OLAPStatus save(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash, - const std::string& meta_binary, string header_prefix = "tabletmeta_"); + const std::string& meta_binary, const string& header_prefix = "tabletmeta_"); static OLAPStatus remove(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash, - string header_prefix = "tabletmeta_"); + const string& header_prefix = "tabletmeta_"); static OLAPStatus traverse_headers(OlapMeta* meta, - std::function const& func, string header_prefix = "tabletmeta_"); + std::function const& func, const string& header_prefix = "tabletmeta_"); static OLAPStatus load_json_header(DataDir* store, const std::string& header_path); diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index 49294f75790de2..d2cf7021d4c1f7 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -46,7 +46,7 @@ OLAPStatus TabletColumn::init_from_pb(const ColumnPB& column) { _precision = column.precision(); } if (column.has_frac()) { - _precision = column.frac(); + _frac = column.frac(); } _length = column.length(); _index_length = column.length(); diff --git a/be/src/olap/txn_manager.cpp b/be/src/olap/txn_manager.cpp index d2348bc3ba0bca..03d71e82b6b53a 100755 --- a/be/src/olap/txn_manager.cpp +++ b/be/src/olap/txn_manager.cpp @@ -175,7 +175,7 @@ OLAPStatus TxnManager::commit_txn( // it is under a single txn lock if (!is_recovery) { OLAPStatus save_status = RowsetMetaManager::save(meta, rowset_ptr->rowset_id(), - rowset_ptr->rowset_meta()); + rowset_ptr->rowset_meta().get()); if (save_status != OLAP_SUCCESS) { LOG(WARNING) << "save committed rowset failed. when commit txn rowset_id:" << rowset_ptr->rowset_id() @@ -226,7 +226,7 @@ OLAPStatus TxnManager::publish_txn(OlapMeta* meta, TPartitionId partition_id, TT rowset_ptr->set_version_and_version_hash(version, version_hash); OLAPStatus save_status = RowsetMetaManager::save(meta, rowset_ptr->rowset_id(), - rowset_ptr->rowset_meta()); + rowset_ptr->rowset_meta().get()); if (save_status != OLAP_SUCCESS) { LOG(WARNING) << "save committed rowset failed. when publish txn rowset_id:" << rowset_ptr->rowset_id() @@ -378,7 +378,7 @@ void TxnManager::get_txn_related_tablets(const TTransactionId transaction_id, // each tablet for (auto& load_info : load_info_map) { const TabletInfo& tablet_info = load_info.first; - tablet_infos->emplace(tablet_info,load_info.second.rowset); + tablet_infos->emplace(tablet_info, load_info.second.rowset); } } diff --git a/be/src/olap/txn_manager.h b/be/src/olap/txn_manager.h index dec6305ce7f353..54975786b601ac 100755 --- a/be/src/olap/txn_manager.h +++ b/be/src/olap/txn_manager.h @@ -86,7 +86,7 @@ class TxnManager { // remove a txn from txn manager // not persist rowset meta because OLAPStatus publish_txn(OlapMeta* meta, TPartitionId partition_id, TTransactionId transaction_id, - TTabletId tablet_id, SchemaHash schema_hash, + TTabletId tablet_id, SchemaHash schema_hash, Version& version, VersionHash& version_hash); // delete the txn from manager if it is not committed(not have a valid rowset) diff --git a/be/test/olap/bit_field_test.cpp b/be/test/olap/bit_field_test.cpp index 44859bfc1c6aed..b691845e6815f4 100755 --- a/be/test/olap/bit_field_test.cpp +++ b/be/test/olap/bit_field_test.cpp @@ -20,8 +20,8 @@ #include "olap/byte_buffer.h" #include "olap/out_stream.h" #include "olap/in_stream.h" -#include "olap/bit_field_reader.h" -#include "olap/bit_field_writer.h" +#include "olap/rowset/bit_field_reader.h" +#include "olap/rowset/bit_field_writer.h" #include "util/logging.h" namespace doris { diff --git a/be/test/olap/column_reader_test.cpp b/be/test/olap/column_reader_test.cpp index 6ff2f4c65ae9e9..500b7d8bd1c0ef 100644 --- a/be/test/olap/column_reader_test.cpp +++ b/be/test/olap/column_reader_test.cpp @@ -19,12 +19,13 @@ #include "olap/byte_buffer.h" #include "olap/stream_name.h" -#include "olap/column_reader.h" -#include "olap/column_writer.h" +#include "olap/rowset/column_reader.h" +#include "olap/rowset/column_writer.h" #include "olap/field.h" #include "olap/olap_define.h" #include "olap/olap_common.h" #include "olap/row_cursor.h" +#include "olap/row_block.h" #include "runtime/mem_pool.h" #include "runtime/string_value.hpp" #include "runtime/vectorized_row_batch.h" @@ -68,7 +69,7 @@ class TestColumn : public testing::Test { _stream_factory = new(std::nothrow) OutStreamFactory(COMPRESS_LZO, - OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE); + OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE); ASSERT_TRUE(_stream_factory != NULL); config::column_dictionary_key_ration_threshold = 30; config::column_dictionary_key_size_threshold = 1000; @@ -98,7 +99,7 @@ class TestColumn : public testing::Test { _length_buffers.clear(); } - void CreateColumnWriter(const std::vector &tablet_schema) { + void CreateColumnWriter(const TabletSchema& tablet_schema) { _column_writer = ColumnWriter::create( 0, tablet_schema, _stream_factory, 1024, BLOOM_FILTER_DEFAULT_FPP); @@ -106,7 +107,7 @@ class TestColumn : public testing::Test { ASSERT_EQ(_column_writer->init(), OLAP_SUCCESS); } - void CreateColumnReader(const std::vector &tablet_schema) { + void CreateColumnReader(const TabletSchema& tablet_schema) { UniqueIdEncodingMap encodings; encodings[0] = ColumnEncodingMessage(); encodings[0].set_kind(ColumnEncodingMessage::DIRECT); @@ -115,7 +116,7 @@ class TestColumn : public testing::Test { } void CreateColumnReader( - const std::vector &tablet_schema, + const TabletSchema& tablet_schema, UniqueIdEncodingMap &encodings) { UniqueIdToColumnIdMap included; included[0] = 0; @@ -123,19 +124,19 @@ class TestColumn : public testing::Test { segment_included[0] = 0; _column_reader = ColumnReader::create(0, - tablet_schema, - included, - segment_included, - encodings); + tablet_schema, + included, + segment_included, + encodings); ASSERT_TRUE(_column_reader != NULL); system("rm ./tmp_file"); ASSERT_EQ(OLAP_SUCCESS, - helper.open_with_mode("tmp_file", - O_CREAT | O_EXCL | O_WRONLY, - S_IRUSR | S_IWUSR)); + helper.open_with_mode("tmp_file", + O_CREAT | O_EXCL | O_WRONLY, + S_IRUSR | S_IWUSR)); std::vector off; std::vector length; std::vector buffer_size; @@ -186,42 +187,44 @@ class TestColumn : public testing::Test { for (int i = 0; i < off.size(); ++i) { ReadOnlyFileStream* in_stream = new (std::nothrow) ReadOnlyFileStream( - &helper, - &_shared_buffer, - off[i], - length[i], - lzo_decompress, - buffer_size[i], - &_stats); + &helper, + &_shared_buffer, + off[i], + length[i], + lzo_decompress, + buffer_size[i], + &_stats); ASSERT_EQ(OLAP_SUCCESS, in_stream->init()); _map_in_streams[name[i]] = in_stream; } ASSERT_EQ(_column_reader->init( - &_map_in_streams, - 1024, - _mem_pool.get(), - &_stats), OLAP_SUCCESS); + &_map_in_streams, + 1024, + _mem_pool.get(), + &_stats), OLAP_SUCCESS); } - void SetFieldInfo(FieldInfo &field_info, - std::string name, - FieldType type, - FieldAggregationMethod aggregation, - uint32_t length, - bool is_allow_null, - bool is_key) { - field_info.name = name; - field_info.type = type; - field_info.aggregation = aggregation; - field_info.length = length; - field_info.is_allow_null = is_allow_null; - field_info.is_key = is_key; - field_info.precision = 1000; - field_info.frac = 10000; - field_info.unique_id = 0; - field_info.is_bf_column = false; + void SetTabletSchemaWithOneColumn(std::string name, + std::string type, + std::string aggregation, + uint32_t length, + bool is_allow_null, + bool is_key, TabletSchema* tablet_schema) { + TabletSchemaPB tablet_schema_pb; + ColumnPB* column = tablet_schema_pb.add_column(); + column->set_unique_id(0); + column->set_name(name); + column->set_type(type); + column->set_is_key(is_key); + column->set_is_nullable(is_allow_null); + column->set_length(length); + column->set_aggregation(aggregation); + column->set_precision(1000); + column->set_frac(1000); + column->set_is_bf_column(false); + tablet_schema->init_from_pb(tablet_schema_pb); } void create_and_save_last_position() { @@ -260,23 +263,20 @@ class TestColumn : public testing::Test { TEST_F(TestColumn, VectorizedTinyColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("TinyColumn"), - OLAP_FIELD_TYPE_TINYINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 1, - false, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "TinyColumn", + "TINYINT", + "REPLACE", + 1, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -316,23 +316,22 @@ TEST_F(TestColumn, VectorizedTinyColumnWithoutPresent) { TEST_F(TestColumn, SeekTinyColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("TinyColumn"), - OLAP_FIELD_TYPE_TINYINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 1, - false, - true); - tablet_schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "TinyColumn", + "TINYINT", + "REPLACE", + 1, + false, + true, + &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -403,23 +402,22 @@ TEST_F(TestColumn, SeekTinyColumnWithoutPresent) { TEST_F(TestColumn, SkipTinyColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("TinyColumn"), - OLAP_FIELD_TYPE_TINYINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 1, - false, - true); - tablet_schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "TinyColumn", + "TINYINT", + "REPLACE", + 1, + false, + true, + &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -459,23 +457,21 @@ TEST_F(TestColumn, SkipTinyColumnWithoutPresent) { TEST_F(TestColumn, VectorizedTinyColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("TinyColumn"), - OLAP_FIELD_TYPE_TINYINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 1, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "TinyColumn", + "TINYINT", + "REPLACE", + 1, + true, + true, + &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -513,23 +509,21 @@ TEST_F(TestColumn, VectorizedTinyColumnWithPresent) { TEST_F(TestColumn, TinyColumnIndex) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("TinyColumn"), - OLAP_FIELD_TYPE_TINYINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 1, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "TinyColumn", + "TINYINT", + "REPLACE", + 1, + true, + true, + &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -567,23 +561,21 @@ TEST_F(TestColumn, TinyColumnIndex) { TEST_F(TestColumn, SeekTinyColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("TinyColumn"), - OLAP_FIELD_TYPE_TINYINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 1, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "TinyColumn", + "TINYINT", + "REPLACE", + 1, + true, + true, + &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -653,23 +645,20 @@ TEST_F(TestColumn, SeekTinyColumnWithPresent) { TEST_F(TestColumn, SkipTinyColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("TinyColumn"), - OLAP_FIELD_TYPE_TINYINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 1, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "TinyColumn", + "TINYINT", + "REPLACE", + 1, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -709,23 +698,20 @@ TEST_F(TestColumn, SkipTinyColumnWithPresent) { TEST_F(TestColumn, VectorizedShortColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("ShortColumn"), - OLAP_FIELD_TYPE_SMALLINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 2, - false, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "ShortColumn", + "SMALLINT", + "REPLACE", + 2, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -763,23 +749,20 @@ TEST_F(TestColumn, VectorizedShortColumnWithoutPresent) { TEST_F(TestColumn, SeekShortColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("ShortColumn"), - OLAP_FIELD_TYPE_SMALLINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 2, - false, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "ShortColumn", + "SMALLINT", + "REPLACE", + 2, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -848,23 +831,20 @@ TEST_F(TestColumn, SeekShortColumnWithoutPresent) { TEST_F(TestColumn, SkipShortColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("ShortColumn"), - OLAP_FIELD_TYPE_SMALLINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 2, - false, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "ShortColumn", + "SMALLINT", + "REPLACE", + 2, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -904,23 +884,20 @@ TEST_F(TestColumn, SkipShortColumnWithoutPresent) { TEST_F(TestColumn, SeekShortColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("ShortColumn"), - OLAP_FIELD_TYPE_SMALLINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 2, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "ShortColumn", + "SMALLINT", + "REPLACE", + 2, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -986,23 +963,21 @@ TEST_F(TestColumn, SeekShortColumnWithPresent) { TEST_F(TestColumn, VectorizedShortColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("ShortColumn"), - OLAP_FIELD_TYPE_SMALLINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 2, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "ShortColumn", + "SMALLINT", + "REPLACE", + 2, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -1042,23 +1017,20 @@ TEST_F(TestColumn, VectorizedShortColumnWithPresent) { TEST_F(TestColumn, SkipShortColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("ShortColumn"), - OLAP_FIELD_TYPE_SMALLINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 2, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "ShortColumn", + "SMALLINT", + "REPLACE", + 2, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -1097,23 +1069,20 @@ TEST_F(TestColumn, SkipShortColumnWithPresent) { TEST_F(TestColumn, VectorizedIntColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("IntColumn"), - OLAP_FIELD_TYPE_INT, - OLAP_FIELD_AGGREGATION_REPLACE, - 4, - false, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "IntColumn", + "INT", + "REPLACE", + 4, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -1151,23 +1120,20 @@ TEST_F(TestColumn, VectorizedIntColumnWithoutPresent) { TEST_F(TestColumn, VectorizedIntColumnMassWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("IntColumn"), - OLAP_FIELD_TYPE_INT, - OLAP_FIELD_AGGREGATION_REPLACE, - 4, - false, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "IntColumn", + "INT", + "REPLACE", + 4, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1207,23 +1173,20 @@ TEST_F(TestColumn, VectorizedIntColumnMassWithoutPresent) { TEST_F(TestColumn, VectorizedIntColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("IntColumn"), - OLAP_FIELD_TYPE_INT, - OLAP_FIELD_AGGREGATION_REPLACE, - 4, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "IntColumn", + "INT", + "REPLACE", + 4, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1264,23 +1227,20 @@ TEST_F(TestColumn, VectorizedIntColumnWithPresent) { TEST_F(TestColumn, VectorizedLongColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("LongColumnWithoutPresent"), - OLAP_FIELD_TYPE_BIGINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 8, - false, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "LongColumnWithoutPresent", + "BIGINT", + "REPLACE", + 8, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1319,23 +1279,20 @@ TEST_F(TestColumn, VectorizedLongColumnWithoutPresent) { TEST_F(TestColumn, VectorizedLongColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("LongColumnWithPresent"), - OLAP_FIELD_TYPE_BIGINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 8, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "LongColumnWithPresent", + "BIGINT", + "REPLACE", + 8, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1376,23 +1333,20 @@ TEST_F(TestColumn, VectorizedLongColumnWithPresent) { TEST_F(TestColumn, VectorizedFloatColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("FloatColumnWithoutPresent"), - OLAP_FIELD_TYPE_FLOAT, - OLAP_FIELD_AGGREGATION_REPLACE, - 4, - false, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "FloatColumnWithoutPresent", + "FLOAT", + "REPLACE", + 4, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1432,23 +1386,21 @@ TEST_F(TestColumn, VectorizedFloatColumnWithoutPresent) { TEST_F(TestColumn, VectorizedFloatColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("FloatColumnWithPresent"), - OLAP_FIELD_TYPE_FLOAT, - OLAP_FIELD_AGGREGATION_REPLACE, - 4, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "FloatColumnWithPresent", + "FLOAT", + "REPLACE", + 4, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1489,23 +1441,21 @@ TEST_F(TestColumn, VectorizedFloatColumnWithPresent) { TEST_F(TestColumn, SeekFloatColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("FloatColumnWithPresent"), - OLAP_FIELD_TYPE_FLOAT, - OLAP_FIELD_AGGREGATION_REPLACE, - 4, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "FloatColumnWithPresent", + "FLOAT", + "REPLACE", + 4, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1562,23 +1512,21 @@ TEST_F(TestColumn, SeekFloatColumnWithPresent) { TEST_F(TestColumn, SkipFloatColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("FloatColumnWithPresent"), - OLAP_FIELD_TYPE_FLOAT, - OLAP_FIELD_AGGREGATION_REPLACE, - 4, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "FloatColumnWithPresent", + "FLOAT", + "REPLACE", + 4, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1615,23 +1563,21 @@ TEST_F(TestColumn, SkipFloatColumnWithPresent) { TEST_F(TestColumn, VectorizedDoubleColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DoubleColumnWithoutPresent"), - OLAP_FIELD_TYPE_DOUBLE, - OLAP_FIELD_AGGREGATION_REPLACE, - 8, - false, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "DoubleColumnWithoutPresent", + "DOUBLE", + "REPLACE", + 8, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1671,23 +1617,21 @@ TEST_F(TestColumn, VectorizedDoubleColumnWithoutPresent) { TEST_F(TestColumn, VectorizedDoubleColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DoubleColumnWithPresent"), - OLAP_FIELD_TYPE_DOUBLE, - OLAP_FIELD_AGGREGATION_REPLACE, - 8, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "DoubleColumnWithPresent", + "DOUBLE", + "REPLACE", + 8, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1729,23 +1673,21 @@ TEST_F(TestColumn, VectorizedDoubleColumnWithPresent) { TEST_F(TestColumn, VectorizedDatetimeColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DatetimeColumnWithoutPresent"), - OLAP_FIELD_TYPE_DATETIME, - OLAP_FIELD_AGGREGATION_REPLACE, - 8, - false, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "DatetimeColumnWithoutPresent", + "DATETIME", + "REPLACE", + 8, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1778,23 +1720,21 @@ TEST_F(TestColumn, VectorizedDatetimeColumnWithoutPresent) { TEST_F(TestColumn, VectorizedDatetimeColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DatetimeColumnWithoutPresent"), - OLAP_FIELD_TYPE_DATETIME, - OLAP_FIELD_AGGREGATION_REPLACE, - 8, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "DatetimeColumnWithoutPresent", + "DATETIME", + "REPLACE", + 8, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1842,23 +1782,21 @@ TEST_F(TestColumn, VectorizedDatetimeColumnWithPresent) { TEST_F(TestColumn, VectorizedDateColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DateColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_DATE, - OLAP_FIELD_AGGREGATION_REPLACE, - 3, - false, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "DateColumnWithoutoutPresent", + "DATE", + "REPLACE", + 3, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1890,23 +1828,21 @@ TEST_F(TestColumn, VectorizedDateColumnWithoutPresent) { TEST_F(TestColumn, VectorizedDateColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DateColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_DATE, - OLAP_FIELD_AGGREGATION_REPLACE, - 3, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "DateColumnWithoutoutPresent", + "DATE", + "REPLACE", + 3, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1954,23 +1890,21 @@ TEST_F(TestColumn, VectorizedDateColumnWithPresent) { TEST_F(TestColumn, VectorizedDecimalColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DecimalColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_DECIMAL, - OLAP_FIELD_AGGREGATION_REPLACE, - 12, - false, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "DecimalColumnWithoutoutPresent", + "DECIMAL", + "REPLACE", + 12, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2015,23 +1949,21 @@ TEST_F(TestColumn, VectorizedDecimalColumnWithoutPresent) { TEST_F(TestColumn, VectorizedDecimalColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DecimalColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_DECIMAL, - OLAP_FIELD_AGGREGATION_REPLACE, - 12, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "DecimalColumnWithoutoutPresent", + "DECIMAL", + "REPLACE", + 12, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2076,23 +2008,21 @@ TEST_F(TestColumn, VectorizedDecimalColumnWithPresent) { TEST_F(TestColumn, SkipDecimalColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DecimalColumnWithPresent"), - OLAP_FIELD_TYPE_DECIMAL, - OLAP_FIELD_AGGREGATION_REPLACE, - 12, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "DecimalColumnWithPresent", + "DECIMAL", + "REPLACE", + 12, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2133,23 +2063,20 @@ TEST_F(TestColumn, SkipDecimalColumnWithPresent) { TEST_F(TestColumn, SeekDecimalColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DecimalColumnWithPresent"), - OLAP_FIELD_TYPE_DECIMAL, - OLAP_FIELD_AGGREGATION_REPLACE, - 12, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "DecimalColumnWithPresent", + "DECIMAL", + "REPLACE", + 12, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2217,17 +2144,15 @@ TEST_F(TestColumn, SeekDecimalColumnWithPresent) { TEST_F(TestColumn, VectorizedLargeIntColumnWithoutPresent) { // init tablet schema - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("LargeIntColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_LARGEINT, - OLAP_FIELD_AGGREGATION_SUM, - 16, - false, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "LargeIntColumnWithoutoutPresent", + "LARGEINT", + "SUM", + 16, + false, + true, &tablet_schema); // test data string value1 = "100000000000000000000000000000000000000"; string value2 = "-170141183460469231731687303715884105728"; @@ -2237,7 +2162,7 @@ TEST_F(TestColumn, VectorizedLargeIntColumnWithoutPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2281,16 +2206,16 @@ TEST_F(TestColumn, VectorizedLargeIntColumnWithoutPresent) { TEST_F(TestColumn, VectorizedLargeIntColumnWithPresent) { // init tablet schema - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("LargeIntColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_LARGEINT, - OLAP_FIELD_AGGREGATION_SUM, - 16, - true, - true); - tablet_schema.push_back(field_info); + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "LargeIntColumnWithoutoutPresent", + "LARGEINT", + "SUM", + 16, + true, + true, &tablet_schema); + // test data string value1 = "100000000000000000000000000000000000000"; @@ -2301,7 +2226,7 @@ TEST_F(TestColumn, VectorizedLargeIntColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2360,17 +2285,15 @@ TEST_F(TestColumn, VectorizedLargeIntColumnWithPresent) { TEST_F(TestColumn, SkipLargeIntColumnWithPresent) { // init tablet schema - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("LargeIntColumnWithPresent"), - OLAP_FIELD_TYPE_LARGEINT, - OLAP_FIELD_AGGREGATION_SUM, - 16, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "LargeIntColumnWithPresent", + "LARGEINT", + "SUM", + 16, + true, + true, &tablet_schema); // test data string value1 = "100000000000000000000000000000000000000"; string value2 = "-170141183460469231731687303715884105728"; @@ -2380,7 +2303,7 @@ TEST_F(TestColumn, SkipLargeIntColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2421,16 +2344,15 @@ TEST_F(TestColumn, SkipLargeIntColumnWithPresent) { TEST_F(TestColumn, VectorizedDirectVarcharColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DirectVarcharColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, - 10, - false, - true); - tablet_schema.push_back(field_info); + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "DirectVarcharColumnWithoutoutPresent", + "VARCHAR", + "REPLACE", + 10, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); @@ -2438,7 +2360,7 @@ TEST_F(TestColumn, VectorizedDirectVarcharColumnWithoutPresent) { write_row.init(tablet_schema); write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2495,24 +2417,21 @@ TEST_F(TestColumn, VectorizedDirectVarcharColumnWithoutPresent) { TEST_F(TestColumn, VectorizedDirectVarcharColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DirectVarcharColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, - 10, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "DirectVarcharColumnWithoutoutPresent", + "VARCHAR", + "REPLACE", + 10, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2555,24 +2474,22 @@ TEST_F(TestColumn, VectorizedDirectVarcharColumnWithPresent) { TEST_F(TestColumn, SkipDirectVarcharColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DirectVarcharColumnWithPresent"), - OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, - 10, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "DirectVarcharColumnWithPresent", + "VARCHAR", + "REPLACE", + 10, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2615,24 +2532,22 @@ TEST_F(TestColumn, SkipDirectVarcharColumnWithPresent) { TEST_F(TestColumn, SeekDirectVarcharColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DirectVarcharColumnWithPresent"), - OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, - 10, - false, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "DirectVarcharColumnWithPresent", + "VARCHAR", + "REPLACE", + 10, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2697,24 +2612,22 @@ TEST_F(TestColumn, SeekDirectVarcharColumnWithoutPresent) { TEST_F(TestColumn, SeekDirectVarcharColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DirectVarcharColumnWithPresent"), - OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, - 10, - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "DirectVarcharColumnWithPresent", + "VARCHAR", + "REPLACE", + 10, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2779,24 +2692,21 @@ TEST_F(TestColumn, SeekDirectVarcharColumnWithPresent) { TEST_F(TestColumn, VectorizedStringColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("VarcharColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_CHAR, - OLAP_FIELD_AGGREGATION_REPLACE, - strlen("abcde"), - false, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "VarcharColumnWithoutoutPresent", + "CHAR", + "REPLACE", + strlen("abcde"), + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2853,24 +2763,21 @@ TEST_F(TestColumn, VectorizedStringColumnWithoutPresent) { TEST_F(TestColumn, VectorizedStringColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("VarcharColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_CHAR, - OLAP_FIELD_AGGREGATION_REPLACE, - strlen("abcde"), - true, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "VarcharColumnWithoutoutPresent", + "CHAR", + "REPLACE", + strlen("abcde"), + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2911,24 +2818,21 @@ TEST_F(TestColumn, VectorizedStringColumnWithPresent) { TEST_F(TestColumn, VectorizedStringColumnWithoutoutPresent2) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("VarcharColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_CHAR, - OLAP_FIELD_AGGREGATION_REPLACE, - 20, - false, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "VarcharColumnWithoutoutPresent", + "CHAR", + "REPLACE", + 20, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -3005,24 +2909,21 @@ TEST_F(TestColumn, VectorizedStringColumnWithoutoutPresent2) { TEST_F(TestColumn, VectorizedDirectVarcharColumnWith65533) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DirectVarcharColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, - 65535, - false, - true); - tablet_schema.push_back(field_info); - + TabletSchema tablet_schema; + SetTabletSchemaWithOneColumn( + "DirectVarcharColumnWithoutoutPresent", + "VARCHAR", + "REPLACE", + 65535, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -3084,4 +2985,3 @@ int main(int argc, char** argv) { ret = RUN_ALL_TESTS(); return ret; } - diff --git a/be/test/olap/comparison_predicate_test.cpp b/be/test/olap/comparison_predicate_test.cpp index 3494b64ae8b2ad..8d582a70c09fc2 100644 --- a/be/test/olap/comparison_predicate_test.cpp +++ b/be/test/olap/comparison_predicate_test.cpp @@ -99,24 +99,28 @@ public: \ delete _vectorized_batch; \ } \ } \ - void SetFieldInfo(FieldInfo &field_info, std::string name, \ - FieldType type, FieldAggregationMethod aggregation, \ - uint32_t length, bool is_allow_null, bool is_key) { \ - field_info.name = name; \ - field_info.type = type; \ - field_info.aggregation = aggregation; \ - field_info.length = length; \ - field_info.is_allow_null = is_allow_null; \ - field_info.is_key = is_key; \ - field_info.precision = 1000; \ - field_info.frac = 10000; \ - field_info.unique_id = 0; \ - field_info.is_bf_column = false; \ + void SetTabletSchema(std::string name, \ + const std::string& type, const std::string& aggregation, \ + uint32_t length, bool is_allow_null, bool is_key, TabletSchema* tablet_schema) { \ + TabletSchemaPB tablet_schema_pb; \ + static int id = 0; \ + ColumnPB* column = tablet_schema_pb.add_column(); \ + column->set_unique_id(++id); \ + column->set_name(name); \ + column->set_type(type); \ + column->set_is_key(is_key); \ + column->set_is_nullable(is_allow_null); \ + column->set_length(length); \ + column->set_aggregation(aggregation); \ + column->set_precision(1000); \ + column->set_frac(1000); \ + column->set_is_bf_column(false); \ + tablet_schema->init_from_pb(tablet_schema_pb); \ } \ - void InitVectorizedBatch(const std::vector& schema, \ + void InitVectorizedBatch(const TabletSchema* tablet_schema, \ const std::vector&ids, \ int size) { \ - _vectorized_batch = new VectorizedRowBatch(schema, ids, size); \ + _vectorized_batch = new VectorizedRowBatch(tablet_schema, ids, size); \ _vectorized_batch->set_size(size); \ } \ std::unique_ptr _mem_tracker; \ @@ -129,17 +133,15 @@ TEST_PREDICATE_DEFINITION(TestLessPredicate) #define TEST_EQUAL_PREDICATE(TYPE, TYPE_NAME, FIELD_TYPE) \ TEST_F(TestEqualPredicate, TYPE_NAME##_COLUMN) { \ - std::vector schema; \ - FieldInfo field_info; \ - SetFieldInfo(field_info, std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, \ - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); \ - schema.push_back(field_info); \ + TabletSchema tablet_schema; \ + SetTabletSchema(std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, \ + "REPLACE", 1, false, true, &tablet_schema); \ int size = 10; \ std::vector return_columns; \ - for (int i = 0; i < schema.size(); ++i) { \ + for (int i = 0; i < tablet_schema.num_columns(); ++i) { \ return_columns.push_back(i); \ } \ - InitVectorizedBatch(schema, return_columns, size); \ + InitVectorizedBatch(&tablet_schema, return_columns, size); \ ColumnVector* col_vector = _vectorized_batch->column(0); \ \ /* for no nulls */ \ @@ -176,24 +178,22 @@ TEST_F(TestEqualPredicate, TYPE_NAME##_COLUMN) { \ ASSERT_EQ(*(col_data + sel[0]), 5); \ } \ -TEST_EQUAL_PREDICATE(int8_t, TINYINT, OLAP_FIELD_TYPE_TINYINT) -TEST_EQUAL_PREDICATE(int16_t, SMALLINT, OLAP_FIELD_TYPE_SMALLINT) -TEST_EQUAL_PREDICATE(int32_t, INT, OLAP_FIELD_TYPE_INT) -TEST_EQUAL_PREDICATE(int64_t, BIGINT, OLAP_FIELD_TYPE_BIGINT) -TEST_EQUAL_PREDICATE(int128_t, LARGEINT, OLAP_FIELD_TYPE_LARGEINT) +TEST_EQUAL_PREDICATE(int8_t, TINYINT, "TINYINT") +TEST_EQUAL_PREDICATE(int16_t, SMALLINT, "SMALLINT") +TEST_EQUAL_PREDICATE(int32_t, INT, "INT") +TEST_EQUAL_PREDICATE(int64_t, BIGINT, "BIGINT") +TEST_EQUAL_PREDICATE(int128_t, LARGEINT, "LARGEINT") TEST_F(TestEqualPredicate, FLOAT_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("FLOAT_COLUMN"), OLAP_FIELD_TYPE_FLOAT, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("FLOAT_COLUMN"), "FLOAT", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -231,17 +231,15 @@ TEST_F(TestEqualPredicate, FLOAT_COLUMN) { } TEST_F(TestEqualPredicate, DOUBLE_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DOUBLE_COLUMN"), OLAP_FIELD_TYPE_DOUBLE, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DOUBLE_COLUMN"), "DOUBLE", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -279,17 +277,15 @@ TEST_F(TestEqualPredicate, DOUBLE_COLUMN) { } TEST_F(TestEqualPredicate, DECIMAL_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DECIMAL_COLUMN"), OLAP_FIELD_TYPE_DECIMAL, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DECIMAL_COLUMN"), "DECIMAL", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -330,17 +326,15 @@ TEST_F(TestEqualPredicate, DECIMAL_COLUMN) { } TEST_F(TestEqualPredicate, STRING_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("STRING_COLUMN"), OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("STRING_COLUMN"), "VARCHAR", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -398,17 +392,15 @@ TEST_F(TestEqualPredicate, STRING_COLUMN) { } TEST_F(TestEqualPredicate, DATE_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DATE_COLUMN"), OLAP_FIELD_TYPE_DATE, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DATE_COLUMN"), "DATA", + "REPLACE", 1, false, true, &tablet_schema); int size = 6; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -461,17 +453,15 @@ TEST_F(TestEqualPredicate, DATE_COLUMN) { } TEST_F(TestEqualPredicate, DATETIME_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DATETIME_COLUMN"), OLAP_FIELD_TYPE_DATETIME, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DATETIME_COLUMN"), "DATETIME", + "REPLACE", 1, false, true, &tablet_schema); int size = 6; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -525,17 +515,15 @@ TEST_F(TestEqualPredicate, DATETIME_COLUMN) { #define TEST_LESS_PREDICATE(TYPE, TYPE_NAME, FIELD_TYPE) \ TEST_F(TestLessPredicate, TYPE_NAME##_COLUMN) { \ - std::vector schema; \ - FieldInfo field_info; \ - SetFieldInfo(field_info, std::string("TYPE_NAME_COLUMN"), FIELD_TYPE, \ - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); \ - schema.push_back(field_info); \ + TabletSchema tablet_schema; \ + SetTabletSchema(std::string("TYPE_NAME_COLUMN"), FIELD_TYPE, \ + "REPLACE", 1, false, true, &tablet_schema); \ int size = 10; \ std::vector return_columns; \ - for (int i = 0; i < schema.size(); ++i) { \ + for (int i = 0; i < tablet_schema.num_columns(); ++i) { \ return_columns.push_back(i); \ } \ - InitVectorizedBatch(schema, return_columns, size); \ + InitVectorizedBatch(&tablet_schema, return_columns, size); \ ColumnVector* col_vector = _vectorized_batch->column(0); \ \ /* for no nulls */ \ @@ -580,24 +568,22 @@ TEST_F(TestLessPredicate, TYPE_NAME##_COLUMN) { \ ASSERT_EQ(sum, 4); \ } \ -TEST_LESS_PREDICATE(int8_t, TINYINT, OLAP_FIELD_TYPE_TINYINT) -TEST_LESS_PREDICATE(int16_t, SMALLINT, OLAP_FIELD_TYPE_SMALLINT) -TEST_LESS_PREDICATE(int32_t, INT, OLAP_FIELD_TYPE_INT) -TEST_LESS_PREDICATE(int64_t, BIGINT, OLAP_FIELD_TYPE_BIGINT) -TEST_LESS_PREDICATE(int128_t, LARGEINT, OLAP_FIELD_TYPE_LARGEINT) +TEST_LESS_PREDICATE(int8_t, TINYINT, "TINYINT") +TEST_LESS_PREDICATE(int16_t, SMALLINT, "SMALLINT") +TEST_LESS_PREDICATE(int32_t, INT, "INT") +TEST_LESS_PREDICATE(int64_t, BIGINT, "BIGINT") +TEST_LESS_PREDICATE(int128_t, LARGEINT, "LARGEINT") TEST_F(TestLessPredicate, FLOAT_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("FLOAT_COLUMN"), OLAP_FIELD_TYPE_FLOAT, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("FLOAT_COLUMN"), "FLOAT", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -644,17 +630,15 @@ TEST_F(TestLessPredicate, FLOAT_COLUMN) { } TEST_F(TestLessPredicate, DOUBLE_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DOUBLE_COLUMN"), OLAP_FIELD_TYPE_DOUBLE, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DOUBLE_COLUMN"), "DOUBLE", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -701,17 +685,15 @@ TEST_F(TestLessPredicate, DOUBLE_COLUMN) { } TEST_F(TestLessPredicate, DECIMAL_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DECIMAL_COLUMN"), OLAP_FIELD_TYPE_DECIMAL, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DECIMAL_COLUMN"), "DECIMAL", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -762,17 +744,15 @@ TEST_F(TestLessPredicate, DECIMAL_COLUMN) { } TEST_F(TestLessPredicate, STRING_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("STRING_COLUMN"), OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("STRING_COLUMN"), "VARCHAR", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -828,17 +808,15 @@ TEST_F(TestLessPredicate, STRING_COLUMN) { } TEST_F(TestLessPredicate, DATE_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DATE_COLUMN"), OLAP_FIELD_TYPE_DATE, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DATE_COLUMN"), "DATE", + "REPLACE", 1, false, true, &tablet_schema); int size = 6; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -888,17 +866,16 @@ TEST_F(TestLessPredicate, DATE_COLUMN) { } TEST_F(TestLessPredicate, DATETIME_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DATETIME_COLUMN"), OLAP_FIELD_TYPE_DATETIME, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + TabletColumn tablet_column; + SetTabletSchema(std::string("DATETIME_COLUMN"), "DATETIME", + "REPLACE", 1, false, true, &tablet_schema); int size = 6; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls diff --git a/be/test/olap/delete_handler_test.cpp b/be/test/olap/delete_handler_test.cpp index 818ae03dc91155..ef840bd32ccef5 100644 --- a/be/test/olap/delete_handler_test.cpp +++ b/be/test/olap/delete_handler_test.cpp @@ -151,8 +151,6 @@ void set_default_create_tablet_request(TCreateTabletReq* request) { void set_default_push_request(TPushReq* request) { request->tablet_id = 10003; request->schema_hash = 270068375; - request->__set_version(2); - request->__set_version_hash(1); request->timeout = 86400; request->push_type = TPushType::LOAD; } @@ -173,26 +171,16 @@ class TestDeleteConditionHandler : public testing::Test { set_default_create_tablet_request(&_create_tablet); res = k_engine->create_tablet(_create_tablet); ASSERT_EQ(OLAP_SUCCESS, res); - tablet = k_engine->get_tablet( + tablet = k_engine->tablet_manager()->get_tablet( _create_tablet.tablet_id, _create_tablet.tablet_schema.schema_hash); ASSERT_TRUE(tablet.get() != NULL); _tablet_path = tablet->tablet_path(); } - OLAPStatus push_empty_delta(int32_t version) { - // push data - TPushReq push_req; - set_default_push_request(&push_req); - push_req.version = version; - push_req.version_hash = version; - std::vector tablets_info; - return k_engine->push(push_req, &tablets_info); - } - void TearDown() { // Remove all dir. tablet.reset(); - StorageEngine::get_instance()->drop_tablet( + StorageEngine::instance()->tablet_manager()->drop_tablet( _create_tablet.tablet_id, _create_tablet.tablet_schema.schema_hash); while (0 == access(_tablet_path.c_str(), F_OK)) { sleep(1); @@ -229,82 +217,23 @@ TEST_F(TestDeleteConditionHandler, StoreCondSucceed) { condition.condition_values.push_back("5"); conditions.push_back(condition); - success_res = _delete_condition_handler.store_cond(tablet, 3, conditions); + DeletePredicatePB del_pred; + success_res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); ASSERT_EQ(OLAP_SUCCESS, success_res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(3)); // 验证存储在header中的过滤条件正确 - const DelPredicateArray& delete_conditions = tablet->delete_data_conditions(); - ASSERT_EQ(size_t(1), delete_conditions.size()); - EXPECT_EQ(3, delete_conditions.Get(0).version()); - ASSERT_EQ(size_t(3), delete_conditions.Get(0).sub_conditions_size()); - EXPECT_STREQ("k1=1", delete_conditions.Get(0).sub_conditions(0).c_str()); - EXPECT_STREQ("k2>>3", delete_conditions.Get(0).sub_conditions(1).c_str()); - EXPECT_STREQ("k2<=5", delete_conditions.Get(0).sub_conditions(2).c_str()); - - // 再次存储相同版本号(版本号为3)的过滤条件 - conditions.clear(); - condition.column_name = "k1"; - condition.condition_op = "!="; - condition.condition_values.clear(); - condition.condition_values.push_back("1"); - conditions.push_back(condition); - - success_res = _delete_condition_handler.store_cond(tablet, 3, conditions); - ASSERT_EQ(OLAP_SUCCESS, success_res); - - // 验证存储相同版本号的过滤条件情况下，新的过滤条件替换掉旧的过滤条件 - const DelPredicateArray& new_delete_conditions = tablet->delete_data_conditions(); - ASSERT_EQ(size_t(1), new_delete_conditions.size()); - EXPECT_EQ(3, new_delete_conditions.Get(0).version()); - ASSERT_EQ(size_t(1), new_delete_conditions.Get(0).sub_conditions_size()); - EXPECT_STREQ("k1!=1", new_delete_conditions.Get(0).sub_conditions(0).c_str()); - - // 第三次存储不同版本号(版本号为4)的过滤条件 - conditions.clear(); - condition.column_name = "k1"; - condition.condition_op = "!="; - condition.condition_values.clear(); - condition.condition_values.push_back("1"); - conditions.push_back(condition); - - condition.column_name = "k1"; - condition.condition_op = "!="; - condition.condition_values.clear(); - condition.condition_values.push_back("2"); - conditions.push_back(condition); - - success_res = _delete_condition_handler.store_cond(tablet, 4, conditions); - ASSERT_EQ(OLAP_SUCCESS, success_res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(4)); - - const DelPredicateArray& all_delete_conditions = tablet->delete_data_conditions(); - ASSERT_EQ(size_t(2), all_delete_conditions.size()); - EXPECT_EQ(3, all_delete_conditions.Get(0).version()); - ASSERT_EQ(size_t(1), all_delete_conditions.Get(0).sub_conditions_size()); - EXPECT_STREQ("k1!=1", all_delete_conditions.Get(0).sub_conditions(0).c_str()); - EXPECT_EQ(4, all_delete_conditions.Get(1).version()); - ASSERT_EQ(size_t(2), all_delete_conditions.Get(1).sub_conditions_size()); - EXPECT_STREQ("k1!=1", all_delete_conditions.Get(1).sub_conditions(0).c_str()); - EXPECT_STREQ("k1!=2", all_delete_conditions.Get(1).sub_conditions(1).c_str()); + ASSERT_EQ(size_t(3), del_pred.sub_predicates_size()); + EXPECT_STREQ("k1=1", del_pred.sub_predicates(0).c_str()); + EXPECT_STREQ("k2>>3", del_pred.sub_predicates(1).c_str()); + EXPECT_STREQ("k2<=5", del_pred.sub_predicates(2).c_str()); } -// 检测参数不正确的情况，包括：空的过滤条件字符串，以及负的版本号 +// 检测参数不正确的情况，包括：空的过滤条件字符串 TEST_F(TestDeleteConditionHandler, StoreCondInvalidParameters) { // 空的过滤条件 std::vector conditions; - OLAPStatus failed_res = _delete_condition_handler.store_cond(tablet, 3, conditions); - ASSERT_EQ(OLAP_ERR_DELETE_INVALID_PARAMETERS, failed_res); - - // 负的版本号: -10 - TCondition condition; - condition.column_name = "k1"; - condition.condition_op = "="; - condition.condition_values.clear(); - condition.condition_values.push_back("2"); - conditions.push_back(condition); - - failed_res = _delete_condition_handler.store_cond(tablet, -10, conditions); + DeletePredicatePB del_pred; + OLAPStatus failed_res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred);; ASSERT_EQ(OLAP_ERR_DELETE_INVALID_PARAMETERS, failed_res); } @@ -318,8 +247,8 @@ TEST_F(TestDeleteConditionHandler, StoreCondNonexistentColumn) { condition.condition_values.clear(); condition.condition_values.push_back("2"); conditions.push_back(condition); - - OLAPStatus failed_res = _delete_condition_handler.store_cond(tablet, 3, conditions); + DeletePredicatePB del_pred; + OLAPStatus failed_res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred);; ASSERT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, failed_res); // 'v'是value列 @@ -330,138 +259,10 @@ TEST_F(TestDeleteConditionHandler, StoreCondNonexistentColumn) { condition.condition_values.push_back("5"); conditions.push_back(condition); - failed_res = _delete_condition_handler.store_cond(tablet, 3, conditions); + failed_res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred);; ASSERT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, failed_res); } -// 只删除特定版本的过滤条件 -TEST_F(TestDeleteConditionHandler, DeleteCondRemoveOneCondition) { - OLAPStatus res; - std::vector conditions; - TCondition condition; - condition.column_name = "k1"; - condition.condition_op = "="; - condition.condition_values.clear(); - condition.condition_values.push_back("1"); - conditions.push_back(condition); - - condition.column_name = "k2"; - condition.condition_op = ">"; - condition.condition_values.clear(); - condition.condition_values.push_back("3"); - conditions.push_back(condition); - - condition.column_name = "k2"; - condition.condition_op = "<="; - condition.condition_values.clear(); - condition.condition_values.push_back("5"); - conditions.push_back(condition); - - res = _delete_condition_handler.store_cond(tablet, 3, conditions); - ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(3)); - - conditions.clear(); - condition.column_name = "k1"; - condition.condition_op = "!="; - condition.condition_values.clear(); - condition.condition_values.push_back("1"); - conditions.push_back(condition); - - res = _delete_condition_handler.store_cond(tablet, 4, conditions); - ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(4)); - - conditions.clear(); - condition.column_name = "k2"; - condition.condition_op = ">="; - condition.condition_values.clear(); - condition.condition_values.push_back("1"); - conditions.push_back(condition); - - res = _delete_condition_handler.store_cond(tablet, 5, conditions); - ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(5)); - - // 删除版本号为8的过滤条件 - res = _delete_condition_handler.delete_cond(tablet, 5, false); - ASSERT_EQ(OLAP_SUCCESS, res); - - const DelPredicateArray& all_delete_conditions = tablet->delete_data_conditions(); - ASSERT_EQ(size_t(2), all_delete_conditions.size()); - - EXPECT_EQ(3, all_delete_conditions.Get(0).version()); - ASSERT_EQ(size_t(3), all_delete_conditions.Get(0).sub_conditions_size()); - EXPECT_STREQ("k1=1", all_delete_conditions.Get(0).sub_conditions(0).c_str()); - EXPECT_STREQ("k2>>3", all_delete_conditions.Get(0).sub_conditions(1).c_str()); - EXPECT_STREQ("k2<=5", all_delete_conditions.Get(0).sub_conditions(2).c_str()); - - EXPECT_EQ(4, all_delete_conditions.Get(1).version()); - ASSERT_EQ(size_t(1), all_delete_conditions.Get(1).sub_conditions_size()); - EXPECT_STREQ("k1!=1", all_delete_conditions.Get(1).sub_conditions(0).c_str()); -} - -// 删除特定版本以及版本比它小的过滤条件 -TEST_F(TestDeleteConditionHandler, DeleteCondRemovBelowCondition) { - OLAPStatus res; - std::vector conditions; - TCondition condition; - condition.column_name = "k1"; - condition.condition_op = "="; - condition.condition_values.clear(); - condition.condition_values.push_back("1"); - conditions.push_back(condition); - - condition.column_name = "k2"; - condition.condition_op = ">"; - condition.condition_values.clear(); - condition.condition_values.push_back("3"); - conditions.push_back(condition); - - condition.column_name = "k2"; - condition.condition_op = "<="; - condition.condition_values.clear(); - condition.condition_values.push_back("5"); - conditions.push_back(condition); - - res = _delete_condition_handler.store_cond(tablet, 3, conditions); - ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(3)); - - conditions.clear(); - condition.column_name = "k1"; - condition.condition_op = "!="; - condition.condition_values.clear(); - condition.condition_values.push_back("1"); - conditions.push_back(condition); - - res = _delete_condition_handler.store_cond(tablet, 4, conditions); - ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(4)); - - conditions.clear(); - condition.column_name = "k2"; - condition.condition_op = ">="; - condition.condition_values.clear(); - condition.condition_values.push_back("1"); - conditions.push_back(condition); - - res = _delete_condition_handler.store_cond(tablet, 5, conditions); - ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(5)); - - // 删除版本号为7以及版本号小于7的过滤条件 - res = _delete_condition_handler.delete_cond(tablet, 4, true); - ASSERT_EQ(OLAP_SUCCESS, res); - - const DelPredicateArray& all_delete_conditions = tablet->delete_data_conditions(); - ASSERT_EQ(size_t(1), all_delete_conditions.size()); - - EXPECT_EQ(5, all_delete_conditions.Get(0).version()); - ASSERT_EQ(size_t(1), all_delete_conditions.Get(0).sub_conditions_size()); - EXPECT_STREQ("k2>=1", all_delete_conditions.Get(0).sub_conditions(0).c_str()); -} - // 测试删除条件值不符合类型要求 class TestDeleteConditionHandler2 : public testing::Test { protected: @@ -479,7 +280,7 @@ class TestDeleteConditionHandler2 : public testing::Test { set_default_create_tablet_request(&_create_tablet); res = k_engine->create_tablet(_create_tablet); ASSERT_EQ(OLAP_SUCCESS, res); - tablet = k_engine->get_tablet( + tablet = k_engine->tablet_manager()->get_tablet( _create_tablet.tablet_id, _create_tablet.tablet_schema.schema_hash); ASSERT_TRUE(tablet.get() != NULL); _tablet_path = tablet->tablet_path(); @@ -488,7 +289,7 @@ class TestDeleteConditionHandler2 : public testing::Test { void TearDown() { // Remove all dir. tablet.reset(); - StorageEngine::get_instance()->drop_tablet( + StorageEngine::instance()->tablet_manager()->drop_tablet( _create_tablet.tablet_id, _create_tablet.tablet_schema.schema_hash); while (0 == access(_tablet_path.c_str(), F_OK)) { sleep(1); @@ -499,6 +300,7 @@ class TestDeleteConditionHandler2 : public testing::Test { std::string _tablet_path; TabletSharedPtr tablet; TCreateTabletReq _create_tablet; + DeleteConditionHandler _delete_condition_handler; }; TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { @@ -532,8 +334,9 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { condition.condition_values.push_back("-1"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 2, conditions); - EXPECT_EQ(OLAP_SUCCESS, res); + DeletePredicatePB del_pred; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); + ASSERT_EQ(OLAP_SUCCESS, res); // k5类型为int128 conditions.clear(); @@ -543,8 +346,9 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { condition.condition_values.push_back("1"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 2, conditions); - EXPECT_EQ(OLAP_SUCCESS, res); + DeletePredicatePB del_pred_2; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_2); + ASSERT_EQ(OLAP_SUCCESS, res); // k9类型为decimal, precision=6, frac=3 conditions.clear(); @@ -554,23 +358,27 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { condition.condition_values.push_back("2.3"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 2, conditions); - EXPECT_EQ(OLAP_SUCCESS, res); + DeletePredicatePB del_pred_3; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_3); + ASSERT_EQ(OLAP_SUCCESS, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2"); - res = cond_handler.store_cond(tablet, 2, conditions); - EXPECT_EQ(OLAP_SUCCESS, res); + DeletePredicatePB del_pred_4; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_4); + ASSERT_EQ(OLAP_SUCCESS, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-2"); - res = cond_handler.store_cond(tablet, 2, conditions); - EXPECT_EQ(OLAP_SUCCESS, res); + DeletePredicatePB del_pred_5; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_5); + ASSERT_EQ(OLAP_SUCCESS, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-2.3"); - res = cond_handler.store_cond(tablet, 2, conditions); - EXPECT_EQ(OLAP_SUCCESS, res); + DeletePredicatePB del_pred_6; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_6); + ASSERT_EQ(OLAP_SUCCESS, res); // k10,k11类型分别为date, datetime conditions.clear(); @@ -586,8 +394,9 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { condition.condition_values.push_back("2014-01-01 00:00:00"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 2, conditions); - EXPECT_EQ(OLAP_SUCCESS, res); + DeletePredicatePB del_pred_7; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_7); + ASSERT_EQ(OLAP_SUCCESS, res); // k12,k13类型分别为string(64), varchar(64) conditions.clear(); @@ -603,8 +412,9 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { condition.condition_values.push_back("YWFhYQ=="); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 2, conditions); - EXPECT_EQ(OLAP_SUCCESS, res); + DeletePredicatePB del_pred_8; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_8); + ASSERT_EQ(OLAP_SUCCESS, res); } TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { @@ -620,133 +430,155 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { condition.condition_values.push_back("1000"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_1; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_1); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k1的值越下界，k1类型为int8 conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-1000"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_2; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_2); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k2的值越上界，k2类型为int16 conditions[0].condition_values.clear(); conditions[0].column_name = "k2"; conditions[0].condition_values.push_back("32768"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_3; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_3); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k2的值越下界，k2类型为int16 conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-32769"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_4; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_4); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k3的值越上界，k3类型为int32 conditions[0].condition_values.clear(); conditions[0].column_name = "k3"; conditions[0].condition_values.push_back("2147483648"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_5; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_5); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k3的值越下界，k3类型为int32 conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-2147483649"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_6; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_6); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k4的值越上界，k2类型为int64 conditions[0].condition_values.clear(); conditions[0].column_name = "k4"; conditions[0].condition_values.push_back("9223372036854775808"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_7; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_7); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k4的值越下界，k1类型为int64 conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-9223372036854775809"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_8; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_8); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k5的值越上界，k5类型为int128 conditions[0].condition_values.clear(); conditions[0].column_name = "k5"; conditions[0].condition_values.push_back("170141183460469231731687303715884105728"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_9; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_9); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k5的值越下界，k5类型为int128 conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-170141183460469231731687303715884105729"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_10; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_10); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k9整数部分长度过长，k9类型为decimal, precision=6, frac=3 conditions[0].condition_values.clear(); conditions[0].column_name = "k9"; - conditions[0].condition_values.push_back("1234.5"); - res = cond_handler.store_cond(tablet, 2, conditions); + conditions[0].condition_values.push_back("12347876.5"); + DeletePredicatePB del_pred_11; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_11); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k9小数部分长度过长，k9类型为decimal, precision=6, frac=3 conditions[0].condition_values.clear(); - conditions[0].condition_values.push_back("1.2345"); - res = cond_handler.store_cond(tablet, 2, conditions); + conditions[0].condition_values.push_back("1.2345678"); + DeletePredicatePB del_pred_12; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_12); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k9没有小数部分，但包含小数点 conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("1."); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_13; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_13); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k10类型的过滤值不符合对应格式，k10为date conditions[0].condition_values.clear(); conditions[0].column_name = "k10"; conditions[0].condition_values.push_back("20130101"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_14; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_14); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-64-01"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_15; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_15); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-01-40"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_16; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_16); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k11类型的过滤值不符合对应格式，k11为datetime conditions[0].condition_values.clear(); conditions[0].column_name = "k11"; conditions[0].condition_values.push_back("20130101 00:00:00"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_17; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_17); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-64-01 00:00:00"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_18; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_18); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-01-40 00:00:00"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_19; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_19); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-01-01 24:00:00"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_20; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_20); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-01-01 00:60:00"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_21; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_21); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-01-01 00:00:60"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_22; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_22); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k12和k13类型的过滤值过长，k12,k13类型分别为string(64), varchar(64) @@ -755,7 +587,8 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].condition_values.push_back("YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYW" "FhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYW" "FhYWFhYWFhYWFhYWFhYWFhYWFhYWE=;k13=YWFhYQ=="); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_23; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_23); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); conditions[0].condition_values.clear(); @@ -763,7 +596,8 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].condition_values.push_back("YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYW" "FhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYW" "FhYWFhYWFhYWFhYWFhYWFhYWFhYWE=;k13=YWFhYQ=="); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_24; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_24); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); } @@ -783,30 +617,20 @@ class TestDeleteHandler : public testing::Test { set_default_create_tablet_request(&_create_tablet); res = k_engine->create_tablet(_create_tablet); ASSERT_EQ(OLAP_SUCCESS, res); - tablet = k_engine->get_tablet( + tablet = k_engine->tablet_manager()->get_tablet( _create_tablet.tablet_id, _create_tablet.tablet_schema.schema_hash); - ASSERT_TRUE(tablet.get() != NULL); + ASSERT_TRUE(tablet != nullptr); _tablet_path = tablet->tablet_path(); _data_row_cursor.init(tablet->tablet_schema()); _data_row_cursor.allocate_memory_for_string_type(tablet->tablet_schema()); } - OLAPStatus push_empty_delta(int32_t version) { - // push data - TPushReq push_req; - set_default_push_request(&push_req); - push_req.version = version; - push_req.version_hash = version; - std::vector tablets_info; - return k_engine->push(push_req, &tablets_info); - } - void TearDown() { // Remove all dir. tablet.reset(); _delete_handler.finalize(); - StorageEngine::get_instance()->drop_tablet( + StorageEngine::instance()->tablet_manager()->drop_tablet( _create_tablet.tablet_id, _create_tablet.tablet_schema.schema_hash); while (0 == access(_tablet_path.c_str(), F_OK)) { sleep(1); @@ -819,6 +643,7 @@ class TestDeleteHandler : public testing::Test { TabletSharedPtr tablet; TCreateTabletReq _create_tablet; DeleteHandler _delete_handler; + DeleteConditionHandler _delete_condition_handler; }; TEST_F(TestDeleteHandler, InitSuccess) { @@ -826,10 +651,6 @@ TEST_F(TestDeleteHandler, InitSuccess) { std::vector conditions; DeleteConditionHandler delete_condition_handler; - // Header中还没有删除条件 - res = _delete_handler.init(tablet, 2); - ASSERT_EQ(OLAP_SUCCESS, res); - // 往头文件中添加过滤条件 TCondition condition; condition.column_name = "k1"; @@ -850,9 +671,11 @@ TEST_F(TestDeleteHandler, InitSuccess) { condition.condition_values.push_back("5"); conditions.push_back(condition); - res = delete_condition_handler.store_cond(tablet, 3, conditions); + DeletePredicatePB del_pred; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); + ASSERT_EQ(OLAP_SUCCESS, res); + res = tablet->add_delete_predicate(del_pred, 1); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(3)); conditions.clear(); condition.column_name = "k1"; @@ -861,9 +684,11 @@ TEST_F(TestDeleteHandler, InitSuccess) { condition.condition_values.push_back("3"); conditions.push_back(condition); - res = delete_condition_handler.store_cond(tablet, 4, conditions); + DeletePredicatePB del_pred_2; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_2); + ASSERT_EQ(OLAP_SUCCESS, res); + res = tablet->add_delete_predicate(del_pred_2, 2); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(4)); conditions.clear(); condition.column_name = "k2"; @@ -872,9 +697,11 @@ TEST_F(TestDeleteHandler, InitSuccess) { condition.condition_values.push_back("1"); conditions.push_back(condition); - res = delete_condition_handler.store_cond(tablet, 5, conditions); + DeletePredicatePB del_pred_3; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_3); + ASSERT_EQ(OLAP_SUCCESS, res); + res = tablet->add_delete_predicate(del_pred_3, 3); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(5)); conditions.clear(); condition.column_name = "k2"; @@ -883,19 +710,23 @@ TEST_F(TestDeleteHandler, InitSuccess) { condition.condition_values.push_back("3"); conditions.push_back(condition); - res = delete_condition_handler.store_cond(tablet, 6, conditions); + DeletePredicatePB del_pred_4; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_4); + ASSERT_EQ(OLAP_SUCCESS, res); + res = tablet->add_delete_predicate(del_pred_4, 4); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(6)); // 从header文件中取出版本号小于等于7的过滤条件 - _delete_handler.finalize(); - res = _delete_handler.init(tablet, 4); + res = _delete_handler.init(tablet->tablet_schema(), tablet->delete_predicates(), 4); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(2, _delete_handler.conditions_num()); + ASSERT_EQ(4, _delete_handler.conditions_num()); vector conds_version = _delete_handler.get_conds_version(); + EXPECT_EQ(4, conds_version.size()); sort(conds_version.begin(), conds_version.end()); - EXPECT_EQ(3, conds_version[0]); - EXPECT_EQ(4, conds_version[1]); + EXPECT_EQ(1, conds_version[0]); + EXPECT_EQ(2, conds_version[1]); + EXPECT_EQ(3, conds_version[2]); + EXPECT_EQ(4, conds_version[3]); _delete_handler.finalize(); } @@ -922,12 +753,15 @@ TEST_F(TestDeleteHandler, FilterDataSubconditions) { condition.condition_values.push_back("4"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 3, conditions); + DeletePredicatePB del_pred; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(3)); + res = tablet->add_delete_predicate(del_pred, 1); // 指定版本号为10以载入Header中的所有过滤条件(在这个case中，只有过滤条件1) - _delete_handler.init(tablet, 10); + res = _delete_handler.init(tablet->tablet_schema(), tablet->delete_predicates(), 4); + ASSERT_EQ(OLAP_SUCCESS, res); + ASSERT_EQ(1, _delete_handler.conditions_num()); // 构造一行测试数据 vector data_str; @@ -980,9 +814,10 @@ TEST_F(TestDeleteHandler, FilterDataConditions) { condition.condition_values.push_back("4"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 3, conditions); + DeletePredicatePB del_pred; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(3)); + res = tablet->add_delete_predicate(del_pred, 1); // 过滤条件2 conditions.clear(); @@ -992,9 +827,10 @@ TEST_F(TestDeleteHandler, FilterDataConditions) { condition.condition_values.push_back("3"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 4, conditions); + DeletePredicatePB del_pred_2; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_2); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(4)); + res = tablet->add_delete_predicate(del_pred_2, 2); // 过滤条件3 conditions.clear(); @@ -1004,12 +840,15 @@ TEST_F(TestDeleteHandler, FilterDataConditions) { condition.condition_values.push_back("5"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 5, conditions); + DeletePredicatePB del_pred_3; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_3); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(5)); + res = tablet->add_delete_predicate(del_pred_3, 3); - // 指定版本号为10以载入Header中的所有三条过滤条件 - _delete_handler.init(tablet, 10); + // 指定版本号为4以载入meta中的所有过滤条件(在这个case中，只有过滤条件1) + res = _delete_handler.init(tablet->tablet_schema(), tablet->delete_predicates(), 4); + ASSERT_EQ(OLAP_SUCCESS, res); + ASSERT_EQ(3, _delete_handler.conditions_num()); vector data_str; data_str.push_back("4"); @@ -1027,7 +866,7 @@ TEST_F(TestDeleteHandler, FilterDataConditions) { res = _data_row_cursor.from_tuple(tuple); ASSERT_EQ(OLAP_SUCCESS, res); // 这行数据会因为过滤条件3而被过滤 - ASSERT_TRUE(_delete_handler.is_filter_data(1, _data_row_cursor)); + ASSERT_TRUE(_delete_handler.is_filter_data(3, _data_row_cursor)); _delete_handler.finalize(); } @@ -1053,9 +892,10 @@ TEST_F(TestDeleteHandler, FilterDataVersion) { condition.condition_values.push_back("4"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 3, conditions); + DeletePredicatePB del_pred; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(3)); + res = tablet->add_delete_predicate(del_pred, 3); // 过滤条件2 conditions.clear(); @@ -1065,12 +905,15 @@ TEST_F(TestDeleteHandler, FilterDataVersion) { condition.condition_values.push_back("3"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 4, conditions); + DeletePredicatePB del_pred_2; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_2); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(4)); + res = tablet->add_delete_predicate(del_pred_2, 4); - // 指定版本号为10以载入Header中的所有过滤条件(过滤条件1，过滤条件2) - _delete_handler.init(tablet, 10); + // 指定版本号为4以载入meta中的所有过滤条件(过滤条件1，过滤条件2) + res = _delete_handler.init(tablet->tablet_schema(), tablet->delete_predicates(), 4); + ASSERT_EQ(OLAP_SUCCESS, res); + ASSERT_EQ(2, _delete_handler.conditions_num()); // 构造一行测试数据 vector data_str; @@ -1088,9 +931,9 @@ TEST_F(TestDeleteHandler, FilterDataVersion) { OlapTuple tuple(data_str); res = _data_row_cursor.from_tuple(tuple); ASSERT_EQ(OLAP_SUCCESS, res); - // 如果数据版本小于6，则过滤条件1生效，这条数据被过滤 - ASSERT_TRUE(_delete_handler.is_filter_data(1, _data_row_cursor)); - // 如果数据版本大于6，则过滤条件1会被跳过 + // 如果数据版本小于3，则过滤条件1生效，这条数据被过滤 + ASSERT_TRUE(_delete_handler.is_filter_data(2, _data_row_cursor)); + // 如果数据版本大于3，则过滤条件1会被跳过 ASSERT_FALSE(_delete_handler.is_filter_data(4, _data_row_cursor)); _delete_handler.finalize(); @@ -1099,11 +942,6 @@ TEST_F(TestDeleteHandler, FilterDataVersion) { } // namespace doris int main(int argc, char** argv) { - std::string conffile = std::string(getenv("DORIS_HOME")) + "/conf/be.conf"; - if (!doris::config::init(conffile.c_str(), false)) { - fprintf(stderr, "error read config file. \n"); - return -1; - } doris::init_glog("be-test"); int ret = doris::OLAP_SUCCESS; testing::InitGoogleTest(&argc, argv); diff --git a/be/test/olap/delta_writer_test.cpp b/be/test/olap/delta_writer_test.cpp index 4e954570c54c5b..9759382374707b 100644 --- a/be/test/olap/delta_writer_test.cpp +++ b/be/test/olap/delta_writer_test.cpp @@ -59,15 +59,17 @@ void set_up() { } void tear_down() { + delete k_engine; + k_engine = nullptr; system("rm -rf ./data_test"); remove_all_dir(std::string(getenv("DORIS_HOME")) + UNUSED_PREFIX); } -void create_tablet_request(TCreateTabletReq* request) { - request->tablet_id = 10003; +void create_tablet_request(int64_t tablet_id, int32_t schema_hash, TCreateTabletReq* request) { + request->tablet_id = tablet_id; request->__set_version(1); request->__set_version_hash(0); - request->tablet_schema.schema_hash = 270068375; + request->tablet_schema.schema_hash = schema_hash; request->tablet_schema.short_key_column_count = 6; request->tablet_schema.keys_type = TKeysType::AGG_KEYS; request->tablet_schema.storage_type = TStorageType::COLUMN; @@ -268,22 +270,20 @@ class TestDeltaWriter : public ::testing::Test { void SetUp() { // Create local data dir for StorageEngine. - char buffer[MAX_PATH_LEN]; - getcwd(buffer, MAX_PATH_LEN); - config::storage_root_path = std::string(buffer) + "/data_push"; - remove_all_dir(config::storage_root_path); - ASSERT_EQ(create_dir(config::storage_root_path), OLAP_SUCCESS); + std::cout << "setup" << std::endl; } void TearDown(){ // Remove all dir. - ASSERT_EQ(OLAP_SUCCESS, remove_all_dir(config::storage_root_path)); + std::cout << "tear down" << std::endl; + //doris::tear_down(); + //ASSERT_EQ(OLAP_SUCCESS, remove_all_dir(config::storage_root_path)); } }; - +/* TEST_F(TestDeltaWriter, open) { TCreateTabletReq request; - create_tablet_request(&request); + create_tablet_request(10003, 270068375, &request); OLAPStatus res = k_engine->create_tablet(request); ASSERT_EQ(OLAP_SUCCESS, res); @@ -308,13 +308,13 @@ TEST_F(TestDeltaWriter, open) { TDropTabletReq drop_request; auto tablet_id = 10003; auto schema_hash = 270068375; - res = k_engine->drop_tablet(tablet_id, schema_hash); + res = k_engine->tablet_manager()->drop_tablet(tablet_id, schema_hash); ASSERT_EQ(OLAP_SUCCESS, res); } - +*/ TEST_F(TestDeltaWriter, write) { TCreateTabletReq request; - create_tablet_request(&request); + create_tablet_request(10004, 270068376, &request); OLAPStatus res = k_engine->create_tablet(request); ASSERT_EQ(OLAP_SUCCESS, res); @@ -327,8 +327,8 @@ TEST_F(TestDeltaWriter, write) { PUniqueId load_id; load_id.set_hi(0); load_id.set_lo(0); - WriteRequest write_req = {10003, 270068375, WriteType::LOAD, - 20001, 30001, load_id, false, tuple_desc}; + WriteRequest write_req = {10004, 270068376, WriteType::LOAD, + 20002, 30002, load_id, false, tuple_desc}; DeltaWriter* delta_writer = nullptr; DeltaWriter::open(&write_req, &delta_writer); ASSERT_NE(delta_writer, nullptr); @@ -381,7 +381,7 @@ TEST_F(TestDeltaWriter, write) { var_ptr = (StringValue*)(tuple->get_slot(slots[18]->tuple_offset())); var_ptr->ptr = arena.Allocate(5); memcpy(var_ptr->ptr, "abcde", 5); - var_ptr->len = 5; + var_ptr->len = 5; DecimalValue val_decimal(1.1); *(DecimalValue*)(tuple->get_slot(slots[19]->tuple_offset())) = val_decimal; @@ -391,252 +391,38 @@ TEST_F(TestDeltaWriter, write) { } res = delta_writer->close(nullptr); - ASSERT_EQ(res, OLAP_SUCCESS); - - // publish version success - TabletSharedPtr tablet = TabletManager::instance()->get_tablet(write_req.tablet_id, write_req.schema_hash); - TPublishVersionRequest publish_req; - publish_req.transaction_id = write_req.transaction_id; - TPartitionVersionInfo info; - info.partition_id = write_req.partition_id; - info.version = tablet->rowset_with_max_version()->end_version() + 1; - info.version_hash = tablet->rowset_with_max_version()->version_hash() + 1; - std::vector partition_version_infos; - partition_version_infos.push_back(info); - publish_req.partition_version_infos = partition_version_infos; - std::vector error_tablet_ids; - res = k_engine->publish_version(publish_req, &error_tablet_ids); - - ASSERT_EQ(1, tablet->get_num_rows()); - - auto tablet_id = 10003; - auto schema_hash = 270068375; - res = k_engine->drop_tablet(tablet_id, schema_hash); - ASSERT_EQ(OLAP_SUCCESS, res); -} - -// ######################### ALTER TABLE TEST BEGIN ######################### - -void schema_change_request(const TCreateTabletReq& base_request, TCreateTabletReq* request) { - //linked schema change, add a value column - request->tablet_id = base_request.tablet_id + 1; - request->__set_version(base_request.version); - request->__set_version_hash(base_request.version_hash); - request->tablet_schema.schema_hash = base_request.tablet_schema.schema_hash + 1; - request->tablet_schema.short_key_column_count = 3; - request->tablet_schema.storage_type = TStorageType::COLUMN; - - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[2]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[3]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[4]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[5]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[6]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[7]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[8]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[9]); - - TColumn v0; - v0.column_name = "v0"; - v0.column_type.type = TPrimitiveType::BIGINT; - v0.__set_is_key(false); - v0.__set_default_value("0"); - v0.__set_aggregation_type(TAggregationType::SUM); - request->tablet_schema.columns.push_back(v0); - - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[10]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[11]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[12]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[13]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[14]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[15]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[16]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[17]); - - TColumn v9; - v9.column_name = "v9"; - v9.__set_is_key(false); - v9.column_type.type = TPrimitiveType::VARCHAR; - v9.column_type.__set_len(130); - v9.__set_aggregation_type(TAggregationType::REPLACE); - request->tablet_schema.columns.push_back(v9); - - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[19]); -} - -AlterTableStatus show_alter_tablet_status(const TAlterTabletReq& request) { - AlterTableStatus status = ALTER_TABLE_RUNNING; - uint32_t max_retry = MAX_RETRY_TIMES; - while (max_retry > 0) { - status = k_engine->show_alter_tablet_status( - request.base_tablet_id, request.base_schema_hash); - if (status != ALTER_TABLE_RUNNING) { break; } - LOG(INFO) << "doing alter tablet......"; - --max_retry; - sleep(1); - } - return status; -} - -class TestSchemaChange : public ::testing::Test { -public: - TestSchemaChange() { } - ~TestSchemaChange() { } - - void SetUp() { - // Create local data dir for StorageEngine. - char buffer[MAX_PATH_LEN]; - getcwd(buffer, MAX_PATH_LEN); - config::storage_root_path = std::string(buffer) + "/data_schema_change"; - remove_all_dir(config::storage_root_path); - ASSERT_EQ(create_dir(config::storage_root_path), OLAP_SUCCESS); - } - - void TearDown(){ - // Remove all dir. - ASSERT_EQ(OLAP_SUCCESS, remove_all_dir(config::storage_root_path)); - } -}; - -TEST_F(TestSchemaChange, schema_change) { - OLAPStatus res = OLAP_SUCCESS; - AlterTableStatus status = ALTER_TABLE_WAITING; - - // 1. Prepare for schema change. - // create base tablet - TCreateTabletReq create_base_tablet; - create_tablet_request(&create_base_tablet); - res = k_engine->create_tablet(create_base_tablet); ASSERT_EQ(OLAP_SUCCESS, res); - TDescriptorTable tdesc_tbl = create_descriptor_tablet(); - ObjectPool obj_pool; - DescriptorTbl* desc_tbl = nullptr; - DescriptorTbl::create(&obj_pool, tdesc_tbl, &desc_tbl); - TupleDescriptor* tuple_desc = desc_tbl->get_tuple_descriptor(0); - - PUniqueId load_id; - load_id.set_hi(0); - load_id.set_lo(0); - WriteRequest write_req = {10003, 270068375, WriteType::LOAD, - 20001, 30001, load_id, false, tuple_desc}; - DeltaWriter* delta_writer = nullptr; - DeltaWriter::open(&write_req, &delta_writer); - ASSERT_NE(delta_writer, nullptr); - - const std::vector& slots = tuple_desc->slots(); - Arena arena; - // streaming load data - { - Tuple* tuple = reinterpret_cast(arena.Allocate(tuple_desc->byte_size())); - memset(tuple, 0, tuple_desc->byte_size()); - *(int8_t*)(tuple->get_slot(slots[0]->tuple_offset())) = -127; - *(int16_t*)(tuple->get_slot(slots[1]->tuple_offset())) = -32767; - *(int32_t*)(tuple->get_slot(slots[2]->tuple_offset())) = -2147483647; - *(int64_t*)(tuple->get_slot(slots[3]->tuple_offset())) = -9223372036854775807L; - - int128_t large_int_value = -90000; - memcpy(tuple->get_slot(slots[4]->tuple_offset()), &large_int_value, sizeof(int128_t)); - - ((DateTimeValue*)(tuple->get_slot(slots[5]->tuple_offset())))->from_date_str("2048-11-10", 10); - ((DateTimeValue*)(tuple->get_slot(slots[6]->tuple_offset())))->from_date_str("2636-08-16 19:39:43", 19); - - StringValue* char_ptr = (StringValue*)(tuple->get_slot(slots[7]->tuple_offset())); - char_ptr->ptr = arena.Allocate(4); - memcpy(char_ptr->ptr, "abcd", 4); - char_ptr->len = 4; - - StringValue* var_ptr = (StringValue*)(tuple->get_slot(slots[8]->tuple_offset())); - var_ptr->ptr = arena.Allocate(5); - memcpy(var_ptr->ptr, "abcde", 5); - var_ptr->len = 5; - - DecimalValue decimal_value(1.1); - *(DecimalValue*)(tuple->get_slot(slots[9]->tuple_offset())) = decimal_value; - - *(int8_t*)(tuple->get_slot(slots[10]->tuple_offset())) = -127; - *(int16_t*)(tuple->get_slot(slots[11]->tuple_offset())) = -32767; - *(int32_t*)(tuple->get_slot(slots[12]->tuple_offset())) = -2147483647; - *(int64_t*)(tuple->get_slot(slots[13]->tuple_offset())) = -9223372036854775807L; - - memcpy(tuple->get_slot(slots[14]->tuple_offset()), &large_int_value, sizeof(int128_t)); - - ((DateTimeValue*)(tuple->get_slot(slots[15]->tuple_offset())))->from_date_str("2048-11-10", 10); - ((DateTimeValue*)(tuple->get_slot(slots[16]->tuple_offset())))->from_date_str("2636-08-16 19:39:43", 19); - - char_ptr = (StringValue*)(tuple->get_slot(slots[17]->tuple_offset())); - char_ptr->ptr = arena.Allocate(4); - memcpy(char_ptr->ptr, "abcd", 4); - char_ptr->len = 4; - - var_ptr = (StringValue*)(tuple->get_slot(slots[18]->tuple_offset())); - var_ptr->ptr = arena.Allocate(5); - memcpy(var_ptr->ptr, "abcde", 5); - var_ptr->len = 5; - - DecimalValue val_decimal(1.1); - *(DecimalValue*)(tuple->get_slot(slots[19]->tuple_offset())) = val_decimal; - - res = delta_writer->write(tuple); + // publish version success + TabletSharedPtr tablet = k_engine->tablet_manager()->get_tablet(write_req.tablet_id, write_req.schema_hash); + std::cout << "before publish, tablet row nums:" << tablet->num_rows() << std::endl; + OlapMeta* meta = tablet->data_dir()->get_meta(); + Version version; + version.first = tablet->rowset_with_max_version()->end_version() + 1; + version.second = tablet->rowset_with_max_version()->end_version() + 1; + std::cout << "start to add rowset version:" << version.first << "-" << version.second << std::endl; + VersionHash version_hash = 2; + std::map tablet_related_rs; + StorageEngine::instance()->txn_manager()->get_txn_related_tablets(write_req.txn_id, write_req.partition_id, &tablet_related_rs); + for (auto& tablet_rs : tablet_related_rs) { + std::cout << "start to publish txn" << std::endl; + RowsetSharedPtr rowset = tablet_rs.second; + res = k_engine->txn_manager()->publish_txn(meta, write_req.partition_id, write_req.txn_id, + write_req.tablet_id, write_req.schema_hash, + version, version_hash); + ASSERT_EQ(OLAP_SUCCESS, res); + std::cout << "start to add inc rowset:" << rowset->rowset_id() << ", num rows:" << rowset->num_rows() + << ", version:" << rowset->version().first << "-" << rowset->version().second + << ", version_hash:" << rowset->version_hash() + << std::endl; + res = tablet->add_inc_rowset(rowset); ASSERT_EQ(OLAP_SUCCESS, res); } + ASSERT_EQ(1, tablet->num_rows()); - // publish version - res = delta_writer->close(nullptr); - ASSERT_EQ(res, OLAP_SUCCESS); - - // publish version success - TabletSharedPtr tablet = TabletManager::instance()->get_tablet(write_req.tablet_id, write_req.schema_hash); - TPublishVersionRequest publish_req; - publish_req.transaction_id = write_req.transaction_id; - TPartitionVersionInfo info; - info.partition_id = write_req.partition_id; - info.version = tablet->rowset_with_max_version()->end_version() + 1; - info.version_hash = tablet->rowset_with_max_version()->version_hash() + 1; - std::vector partition_version_infos; - partition_version_infos.push_back(info); - publish_req.partition_version_infos = partition_version_infos; - std::vector error_tablet_ids; - res = k_engine->publish_version(publish_req, &error_tablet_ids); - ASSERT_EQ(res, OLAP_SUCCESS); - - // 1. set add column request - TCreateTabletReq create_new_tablet; - schema_change_request(create_base_tablet, &create_new_tablet); - TAlterTabletReq request; - request.__set_base_tablet_id(create_base_tablet.tablet_id); - request.__set_base_schema_hash(create_base_tablet.tablet_schema.schema_hash); - request.__set_new_tablet_req(create_new_tablet); - - // 2. Submit schema change - request.base_schema_hash = create_base_tablet.tablet_schema.schema_hash; - res = k_engine->schema_change(request); - ASSERT_EQ(OLAP_SUCCESS, res); - - // 3. Verify schema change result. - // show schema change status - status = show_alter_tablet_status(request); - ASSERT_EQ(ALTER_TABLE_FINISHED, status); - - // check new tablet information - TTabletInfo tablet_info; - tablet_info.tablet_id = create_new_tablet.tablet_id; - tablet_info.schema_hash = create_new_tablet.tablet_schema.schema_hash; - res = k_engine->report_tablet_info(&tablet_info); - ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(info.version, tablet_info.version); - ASSERT_EQ(info.version_hash, tablet_info.version_hash); - ASSERT_EQ(1, tablet_info.row_count); - - // 4. Retry the same schema change request. - res = k_engine->schema_change(request); - ASSERT_EQ(OLAP_SUCCESS, res); - status = k_engine->show_alter_tablet_status( - request.base_tablet_id, request.base_schema_hash); - ASSERT_EQ(ALTER_TABLE_FINISHED, status); - - auto tablet_id = create_new_tablet.tablet_id; - auto schema_hash = create_new_tablet.tablet_schema.schema_hash; - res = k_engine->drop_tablet(tablet_id, schema_hash); + auto tablet_id = 10003; + auto schema_hash = 270068375; + res = k_engine->tablet_manager()->drop_tablet(tablet_id, schema_hash); ASSERT_EQ(OLAP_SUCCESS, res); } @@ -652,11 +438,9 @@ int main(int argc, char** argv) { int ret = doris::OLAP_SUCCESS; testing::InitGoogleTest(&argc, argv); doris::CpuInfo::init(); - doris::set_up(); ret = RUN_ALL_TESTS(); doris::tear_down(); - google::protobuf::ShutdownProtobufLibrary(); return ret; } diff --git a/be/test/olap/in_list_predicate_test.cpp b/be/test/olap/in_list_predicate_test.cpp index 5b187d4c34aa58..e34ea89c45f828 100644 --- a/be/test/olap/in_list_predicate_test.cpp +++ b/be/test/olap/in_list_predicate_test.cpp @@ -100,25 +100,30 @@ class TestInListPredicate : public testing::Test { } } - void SetFieldInfo(FieldInfo &field_info, std::string name, - FieldType type, FieldAggregationMethod aggregation, - uint32_t length, bool is_allow_null, bool is_key) { - field_info.name = name; - field_info.type = type; - field_info.aggregation = aggregation; - field_info.length = length; - field_info.is_allow_null = is_allow_null; - field_info.is_key = is_key; - field_info.precision = 1000; - field_info.frac = 10000; - field_info.unique_id = 0; - field_info.is_bf_column = false; + void SetTabletSchema(std::string name, + const std::string& type, const std::string& aggregation, + uint32_t length, bool is_allow_null, bool is_key, TabletSchema* tablet_schema) { + TabletSchemaPB tablet_schema_pb; + static int id = 0; + ColumnPB* column = tablet_schema_pb.add_column(); + column->set_unique_id(++id); + column->set_name(name); + column->set_type(type); + column->set_is_key(is_key); + column->set_is_nullable(is_allow_null); + column->set_length(length); + column->set_aggregation(aggregation); + column->set_precision(1000); + column->set_frac(1000); + column->set_is_bf_column(false); + + tablet_schema->init_from_pb(tablet_schema_pb); } - void InitVectorizedBatch(const std::vector& schema, + void InitVectorizedBatch(const TabletSchema* tablet_schema, const std::vector& ids, int size) { - _vectorized_batch = new VectorizedRowBatch(schema, ids, size); + _vectorized_batch = new VectorizedRowBatch(tablet_schema, ids, size); _vectorized_batch->set_size(size); } std::unique_ptr _mem_tracker; @@ -128,17 +133,15 @@ class TestInListPredicate : public testing::Test { #define TEST_IN_LIST_PREDICATE(TYPE, TYPE_NAME, FIELD_TYPE) \ TEST_F(TestInListPredicate, TYPE_NAME##_COLUMN) { \ - std::vector schema; \ - FieldInfo field_info; \ - SetFieldInfo(field_info, std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, \ - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); \ - schema.push_back(field_info); \ + TabletSchema tablet_schema; \ + SetTabletSchema(std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, \ + "REPLACE", 1, false, true, &tablet_schema); \ int size = 10; \ std::vector return_columns; \ - for (int i = 0; i < schema.size(); ++i) { \ + for (int i = 0; i < tablet_schema.num_columns(); ++i) { \ return_columns.push_back(i); \ } \ - InitVectorizedBatch(schema, return_columns, size); \ + InitVectorizedBatch(&tablet_schema, return_columns, size); \ ColumnVector* col_vector = _vectorized_batch->column(0); \ \ /* for no nulls */ \ @@ -181,24 +184,22 @@ TEST_F(TestInListPredicate, TYPE_NAME##_COLUMN) { \ ASSERT_EQ(*(col_data + sel[0]), 5); \ } \ -TEST_IN_LIST_PREDICATE(int8_t, TINYINT, OLAP_FIELD_TYPE_TINYINT) -TEST_IN_LIST_PREDICATE(int16_t, SMALLINT, OLAP_FIELD_TYPE_SMALLINT) -TEST_IN_LIST_PREDICATE(int32_t, INT, OLAP_FIELD_TYPE_INT) -TEST_IN_LIST_PREDICATE(int64_t, BIGINT, OLAP_FIELD_TYPE_BIGINT) -TEST_IN_LIST_PREDICATE(int128_t, LARGEINT, OLAP_FIELD_TYPE_LARGEINT) +TEST_IN_LIST_PREDICATE(int8_t, TINYINT, "TINYINT") +TEST_IN_LIST_PREDICATE(int16_t, SMALLINT, "SMALLINT") +TEST_IN_LIST_PREDICATE(int32_t, INT, "INT") +TEST_IN_LIST_PREDICATE(int64_t, BIGINT, "BIGINT") +TEST_IN_LIST_PREDICATE(int128_t, LARGEINT, "LARGEINT") TEST_F(TestInListPredicate, FLOAT_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("FLOAT_COLUMN"), OLAP_FIELD_TYPE_FLOAT, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("FLOAT_COLUMN"), "FLOAT", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -241,17 +242,15 @@ TEST_F(TestInListPredicate, FLOAT_COLUMN) { } TEST_F(TestInListPredicate, DOUBLE_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DOUBLE_COLUMN"), OLAP_FIELD_TYPE_DOUBLE, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DOUBLE_COLUMN"), "DOUBLE", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -295,17 +294,15 @@ TEST_F(TestInListPredicate, DOUBLE_COLUMN) { } TEST_F(TestInListPredicate, DECIMAL_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DECIMAL_COLUMN"), OLAP_FIELD_TYPE_DECIMAL, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DECIMAL_COLUMN"), "DECIMAL", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -358,17 +355,15 @@ TEST_F(TestInListPredicate, DECIMAL_COLUMN) { } TEST_F(TestInListPredicate, CHAR_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("STRING_COLUMN"), OLAP_FIELD_TYPE_CHAR, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("STRING_COLUMN"), "CHAR", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -443,17 +438,15 @@ TEST_F(TestInListPredicate, CHAR_COLUMN) { } TEST_F(TestInListPredicate, VARCHAR_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("STRING_COLUMN"), OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("STRING_COLUMN"), "VARCHAR", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -526,17 +519,15 @@ TEST_F(TestInListPredicate, VARCHAR_COLUMN) { } TEST_F(TestInListPredicate, DATE_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DATE_COLUMN"), OLAP_FIELD_TYPE_DATE, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DATE_COLUMN"), "DATE", + "REPLACE", 1, false, true, &tablet_schema); int size = 6; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -597,17 +588,15 @@ TEST_F(TestInListPredicate, DATE_COLUMN) { } TEST_F(TestInListPredicate, DATETIME_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DATETIME_COLUMN"), OLAP_FIELD_TYPE_DATETIME, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DATETIME_COLUMN"), "DATETIME", + "REPLACE", 1, false, true, &tablet_schema); int size = 6; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls diff --git a/be/test/olap/null_predicate_test.cpp b/be/test/olap/null_predicate_test.cpp index 52c45975611f20..f59b4a5b27a844 100644 --- a/be/test/olap/null_predicate_test.cpp +++ b/be/test/olap/null_predicate_test.cpp @@ -70,25 +70,29 @@ class TestNullPredicate : public testing::Test { } } - void SetFieldInfo(FieldInfo &field_info, std::string name, - FieldType type, FieldAggregationMethod aggregation, - uint32_t length, bool is_allow_null, bool is_key) { - field_info.name = name; - field_info.type = type; - field_info.aggregation = aggregation; - field_info.length = length; - field_info.is_allow_null = is_allow_null; - field_info.is_key = is_key; - field_info.precision = 1000; - field_info.frac = 10000; - field_info.unique_id = 0; - field_info.is_bf_column = false; + void SetTabletSchema(std::string name, + std::string type, std::string aggregation, + uint32_t length, bool is_allow_null, bool is_key, TabletSchema* tablet_schema) { + TabletSchemaPB tablet_schema_pb; + static int id = 0; + ColumnPB* column = tablet_schema_pb.add_column();; + column->set_unique_id(++id); + column->set_name(name); + column->set_type(type); + column->set_is_key(is_key); + column->set_is_nullable(is_allow_null); + column->set_length(length); + column->set_aggregation(aggregation); + column->set_precision(1000); + column->set_frac(1000); + column->set_is_bf_column(false); + tablet_schema->init_from_pb(tablet_schema_pb); } - void InitVectorizedBatch(const std::vector& schema, + void InitVectorizedBatch(const TabletSchema* tablet_schema, const std::vector&ids, int size) { - _vectorized_batch = new VectorizedRowBatch(schema, ids, size); + _vectorized_batch = new VectorizedRowBatch(tablet_schema, ids, size); _vectorized_batch->set_size(size); } std::unique_ptr _mem_tracker; @@ -98,17 +102,15 @@ class TestNullPredicate : public testing::Test { #define TEST_IN_LIST_PREDICATE(TYPE, TYPE_NAME, FIELD_TYPE) \ TEST_F(TestNullPredicate, TYPE_NAME##_COLUMN) { \ - std::vector schema; \ - FieldInfo field_info; \ - SetFieldInfo(field_info, std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, \ - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); \ - schema.push_back(field_info); \ + TabletSchema tablet_schema; \ + SetTabletSchema(std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, \ + "REPLACE", 1, false, true, &tablet_schema); \ int size = 10; \ std::vector return_columns; \ - for (int i = 0; i < schema.size(); ++i) { \ + for (int i = 0; i < tablet_schema.num_columns(); ++i) { \ return_columns.push_back(i); \ } \ - InitVectorizedBatch(schema, return_columns, size); \ + InitVectorizedBatch(&tablet_schema, return_columns, size); \ ColumnVector* col_vector = _vectorized_batch->column(0); \ \ /* for no nulls */ \ @@ -141,24 +143,22 @@ TEST_F(TestNullPredicate, TYPE_NAME##_COLUMN) { \ ASSERT_EQ(_vectorized_batch->size(), 5); \ } \ -TEST_IN_LIST_PREDICATE(int8_t, TINYINT, OLAP_FIELD_TYPE_TINYINT) -TEST_IN_LIST_PREDICATE(int16_t, SMALLINT, OLAP_FIELD_TYPE_SMALLINT) -TEST_IN_LIST_PREDICATE(int32_t, INT, OLAP_FIELD_TYPE_INT) -TEST_IN_LIST_PREDICATE(int64_t, BIGINT, OLAP_FIELD_TYPE_BIGINT) -TEST_IN_LIST_PREDICATE(int128_t, LARGEINT, OLAP_FIELD_TYPE_LARGEINT) +TEST_IN_LIST_PREDICATE(int8_t, TINYINT, "TINYINT") +TEST_IN_LIST_PREDICATE(int16_t, SMALLINT, "SMALLINT") +TEST_IN_LIST_PREDICATE(int32_t, INT, "INT") +TEST_IN_LIST_PREDICATE(int64_t, BIGINT, "BIGINT") +TEST_IN_LIST_PREDICATE(int128_t, LARGEINT, "LARGEINT") TEST_F(TestNullPredicate, FLOAT_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("FLOAT_COLUMN"), OLAP_FIELD_TYPE_FLOAT, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("FLOAT_COLUMN"), "FLOAT", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -191,17 +191,15 @@ TEST_F(TestNullPredicate, FLOAT_COLUMN) { } TEST_F(TestNullPredicate, DOUBLE_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DOUBLE_COLUMN"), OLAP_FIELD_TYPE_DOUBLE, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DOUBLE_COLUMN"), "DOUBLE", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -235,17 +233,15 @@ TEST_F(TestNullPredicate, DOUBLE_COLUMN) { } TEST_F(TestNullPredicate, DECIMAL_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DECIMAL_COLUMN"), OLAP_FIELD_TYPE_DECIMAL, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DECIMAL_COLUMN"), "DECIMAL", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -282,17 +278,15 @@ TEST_F(TestNullPredicate, DECIMAL_COLUMN) { } TEST_F(TestNullPredicate, STRING_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("STRING_COLUMN"), OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("STRING_COLUMN"), "VARCHAR", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -339,17 +333,15 @@ TEST_F(TestNullPredicate, STRING_COLUMN) { } TEST_F(TestNullPredicate, DATE_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DATE_COLUMN"), OLAP_FIELD_TYPE_DATE, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DATE_COLUMN"), "DATE", + "REPLACE", 1, false, true, &tablet_schema); int size = 6; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -394,17 +386,15 @@ TEST_F(TestNullPredicate, DATE_COLUMN) { } TEST_F(TestNullPredicate, DATETIME_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DATETIME_COLUMN"), OLAP_FIELD_TYPE_DATETIME, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DATETIME_COLUMN"), "DATETIME", + "REPLACE", 1, false, true, &tablet_schema); int size = 6; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls diff --git a/be/test/olap/row_block_test.cpp b/be/test/olap/row_block_test.cpp index 4b108e94522eff..4b52d13cb3fc58 100644 --- a/be/test/olap/row_block_test.cpp +++ b/be/test/olap/row_block_test.cpp @@ -45,46 +45,51 @@ class TestRowBlock : public testing::Test { } }; -TEST_F(TestRowBlock, init) { - std::vector fields; +void init_tablet_schema(TabletSchema* tablet_schema) { + TabletSchemaPB tablet_schema_pb; { // k1: bigint { - FieldInfo info; - info.name = "k1"; - info.type = OLAP_FIELD_TYPE_BIGINT; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 8; - info.is_key = true; - fields.push_back(info); + ColumnPB* column_1 = tablet_schema_pb.add_column(); + column_1->set_unique_id(1); + column_1->set_name("k1"); + column_1->set_type("BIGINT"); + column_1->set_is_key(true); + column_1->set_length(8); + column_1->set_aggregation("NONE"); } // k2: char { - FieldInfo info; - info.name = "k2"; - info.type = OLAP_FIELD_TYPE_CHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 10; - info.is_key = true; - fields.push_back(info); + ColumnPB* column_2 = tablet_schema_pb.add_column(); + column_2->set_unique_id(2); + column_2->set_name("k2"); + column_2->set_type("CHAR"); + column_2->set_is_key(true); + column_2->set_length(10); + column_2->set_aggregation("NONE"); } // k3: varchar { - FieldInfo info; - info.name = "k3"; - info.type = OLAP_FIELD_TYPE_VARCHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 20; - info.is_key = true; - fields.push_back(info); + ColumnPB* column_3 = tablet_schema_pb.add_column(); + column_3->set_unique_id(3); + column_3->set_name("k3"); + column_3->set_type("VARCHAR"); + column_3->set_is_key(true); + column_3->set_length(20); + column_3->set_aggregation("NONE"); } } + tablet_schema->init_from_pb(tablet_schema_pb); +} + +TEST_F(TestRowBlock, init) { + TabletSchema tablet_schema; + init_tablet_schema(&tablet_schema); { // has nullbyte - RowBlock block(fields); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); @@ -93,63 +98,29 @@ TEST_F(TestRowBlock, init) { } { // has nullbyte - RowBlock block(fields); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = false; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); // num_rows * (num_nullbytes + bigint + char + varchar) - ASSERT_EQ(1024 * (3 + 8 + 10 + (4 + 20)), block.buf_len()); + ASSERT_EQ(1024 * (8 + 10 + (4 + 20)), block.buf_len()); } } TEST_F(TestRowBlock, write_and_read) { - std::vector fields; - { - // k1: bigint - { - FieldInfo info; - info.name = "k1"; - info.type = OLAP_FIELD_TYPE_BIGINT; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 8; - info.is_key = true; - fields.push_back(info); - } - // k2: char - { - FieldInfo info; - info.name = "k2"; - info.type = OLAP_FIELD_TYPE_CHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 10; - info.is_key = true; - fields.push_back(info); - } - // k3: varchar - { - FieldInfo info; - info.name = "k3"; - info.type = OLAP_FIELD_TYPE_VARCHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 20; - info.is_key = true; - fields.push_back(info); - } - } - // - RowBlock block(fields); + TabletSchema tablet_schema; + init_tablet_schema(&tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); RowCursor row; - row.init(fields); + row.init(tablet_schema); for (int i = 0; i < 5; ++i) { block.get_row(i, &row); @@ -185,7 +156,7 @@ TEST_F(TestRowBlock, write_and_read) { ASSERT_EQ(OLAP_SUCCESS, res); { - RowBlock resolve_block(fields); + RowBlock resolve_block(&tablet_schema); block_info.checksum = block.row_block_info().checksum; block_info.row_num = 5; res = resolve_block.init(block_info); @@ -222,50 +193,17 @@ TEST_F(TestRowBlock, write_and_read) { } TEST_F(TestRowBlock, write_and_read_without_nullbyte) { - std::vector fields; - { - // k1: bigint - { - FieldInfo info; - info.name = "k1"; - info.type = OLAP_FIELD_TYPE_BIGINT; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 8; - info.is_key = true; - fields.push_back(info); - } - // k2: char - { - FieldInfo info; - info.name = "k2"; - info.type = OLAP_FIELD_TYPE_CHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 10; - info.is_key = true; - fields.push_back(info); - } - // k3: varchar - { - FieldInfo info; - info.name = "k3"; - info.type = OLAP_FIELD_TYPE_VARCHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 20; - info.is_key = true; - fields.push_back(info); - } - } - // - RowBlock block(fields); + TabletSchema tablet_schema; + init_tablet_schema(&tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = false; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); RowCursor row; - row.init(fields); + row.init(tablet_schema); for (int i = 0; i < 5; ++i) { block.get_row(i, &row); @@ -301,7 +239,7 @@ TEST_F(TestRowBlock, write_and_read_without_nullbyte) { ASSERT_EQ(OLAP_SUCCESS, res); { - RowBlock resolve_block(fields); + RowBlock resolve_block(&tablet_schema); block_info.checksum = block.row_block_info().checksum; block_info.row_num = 5; res = resolve_block.init(block_info); @@ -338,50 +276,17 @@ TEST_F(TestRowBlock, write_and_read_without_nullbyte) { } TEST_F(TestRowBlock, compress_failed) { - std::vector fields; - { - // k1: bigint - { - FieldInfo info; - info.name = "k1"; - info.type = OLAP_FIELD_TYPE_BIGINT; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 8; - info.is_key = true; - fields.push_back(info); - } - // k2: char - { - FieldInfo info; - info.name = "k2"; - info.type = OLAP_FIELD_TYPE_CHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 10; - info.is_key = true; - fields.push_back(info); - } - // k3: varchar - { - FieldInfo info; - info.name = "k3"; - info.type = OLAP_FIELD_TYPE_VARCHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 20; - info.is_key = true; - fields.push_back(info); - } - } - // - RowBlock block(fields); + TabletSchema tablet_schema; + init_tablet_schema(&tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); RowCursor row; - row.init(fields); + row.init(tablet_schema); for (int i = 0; i < 5; ++i) { block.get_row(i, &row); @@ -415,50 +320,17 @@ TEST_F(TestRowBlock, compress_failed) { } TEST_F(TestRowBlock, decompress_failed) { - std::vector fields; - { - // k1: bigint - { - FieldInfo info; - info.name = "k1"; - info.type = OLAP_FIELD_TYPE_BIGINT; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 8; - info.is_key = true; - fields.push_back(info); - } - // k2: char - { - FieldInfo info; - info.name = "k2"; - info.type = OLAP_FIELD_TYPE_CHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 10; - info.is_key = true; - fields.push_back(info); - } - // k3: varchar - { - FieldInfo info; - info.name = "k3"; - info.type = OLAP_FIELD_TYPE_VARCHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 20; - info.is_key = true; - fields.push_back(info); - } - } - // - RowBlock block(fields); + TabletSchema tablet_schema; + init_tablet_schema(&tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); RowCursor row; - row.init(fields); + row.init(tablet_schema); for (int i = 0; i < 5; ++i) { block.get_row(i, &row); @@ -492,7 +364,7 @@ TEST_F(TestRowBlock, decompress_failed) { { // checksum failed - RowBlock resolve_block(fields); + RowBlock resolve_block(&tablet_schema); block_info.checksum = 0; block_info.row_num = 5; res = resolve_block.init(block_info); @@ -503,7 +375,7 @@ TEST_F(TestRowBlock, decompress_failed) { } { // buffer is not ok - RowBlock resolve_block(fields); + RowBlock resolve_block(&tablet_schema); block_info.checksum = block.row_block_info().checksum; block_info.row_num = 5; res = resolve_block.init(block_info); @@ -515,50 +387,17 @@ TEST_F(TestRowBlock, decompress_failed) { } TEST_F(TestRowBlock, find_row) { - std::vector fields; - { - // k1: bigint - { - FieldInfo info; - info.name = "k1"; - info.type = OLAP_FIELD_TYPE_BIGINT; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 8; - info.is_key = true; - fields.push_back(info); - } - // k2: char - { - FieldInfo info; - info.name = "k2"; - info.type = OLAP_FIELD_TYPE_CHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 10; - info.is_key = true; - fields.push_back(info); - } - // k3: varchar - { - FieldInfo info; - info.name = "k3"; - info.type = OLAP_FIELD_TYPE_VARCHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 20; - info.is_key = true; - fields.push_back(info); - } - } - // - RowBlock block(fields); + TabletSchema tablet_schema; + init_tablet_schema(&tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); RowCursor row; - row.init(fields); + row.init(tablet_schema); for (int i = 0; i < 5; ++i) { block.get_row(i, &row); @@ -590,7 +429,7 @@ TEST_F(TestRowBlock, find_row) { { RowCursor find_row; - find_row.init(fields); + find_row.init(tablet_schema); for (int i = 0; i < 5; ++i) { // bigint { @@ -677,44 +516,11 @@ TEST_F(TestRowBlock, find_row) { } TEST_F(TestRowBlock, clear) { - std::vector fields; - { - // k1: bigint - { - FieldInfo info; - info.name = "k1"; - info.type = OLAP_FIELD_TYPE_BIGINT; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 8; - info.is_key = true; - fields.push_back(info); - } - // k2: char - { - FieldInfo info; - info.name = "k2"; - info.type = OLAP_FIELD_TYPE_CHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 10; - info.is_key = true; - fields.push_back(info); - } - // k3: varchar - { - FieldInfo info; - info.name = "k3"; - info.type = OLAP_FIELD_TYPE_VARCHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 20; - info.is_key = true; - fields.push_back(info); - } - } - // - RowBlock block(fields); + TabletSchema tablet_schema; + init_tablet_schema(&tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); @@ -727,44 +533,11 @@ TEST_F(TestRowBlock, clear) { } TEST_F(TestRowBlock, pos_limit) { - std::vector fields; - { - // k1: bigint - { - FieldInfo info; - info.name = "k1"; - info.type = OLAP_FIELD_TYPE_BIGINT; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 8; - info.is_key = true; - fields.push_back(info); - } - // k2: char - { - FieldInfo info; - info.name = "k2"; - info.type = OLAP_FIELD_TYPE_CHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 10; - info.is_key = true; - fields.push_back(info); - } - // k3: varchar - { - FieldInfo info; - info.name = "k3"; - info.type = OLAP_FIELD_TYPE_VARCHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 20; - info.is_key = true; - fields.push_back(info); - } - } - // - RowBlock block(fields); + TabletSchema tablet_schema; + init_tablet_schema(&tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); diff --git a/be/test/olap/row_cursor_test.cpp b/be/test/olap/row_cursor_test.cpp index 604126d0f0cd05..1f4fae9659e0d5 100644 --- a/be/test/olap/row_cursor_test.cpp +++ b/be/test/olap/row_cursor_test.cpp @@ -18,220 +18,232 @@ #include #include "olap/row_cursor.h" +#include "olap/tablet_schema.h" #include "runtime/mem_tracker.h" #include "runtime/mem_pool.h" #include "util/logging.h" namespace doris { -void set_tablet_schema_for_init(std::vector* tablet_schema) { - FieldInfo k1; - k1.name = "k1"; - k1.type = OLAP_FIELD_TYPE_TINYINT; - k1.length = 1; - k1.is_key = true; - k1.index_length = 1; - k1.is_allow_null = true; - tablet_schema->push_back(k1); - - FieldInfo k2; - k2.name = "k2"; - k2.type = OLAP_FIELD_TYPE_SMALLINT; - k2.length = 2; - k2.default_value = "0"; - k2.is_key = true; - k2.index_length = 2; - k2.is_allow_null = true; - tablet_schema->push_back(k2); - - FieldInfo k3; - k3.name = "k3"; - k3.type = OLAP_FIELD_TYPE_INT; - k3.length = 4; - k3.is_key = true; - k3.index_length = 4; - k3.is_allow_null = true; - tablet_schema->push_back(k3); - - FieldInfo k4; - k4.name = "k4"; - k4.type = OLAP_FIELD_TYPE_DATE; - k4.length = 3; - k4.is_key = true; - k4.index_length = 3; - k4.is_allow_null = true; - tablet_schema->push_back(k4); - - FieldInfo k5; - k5.name = "k5"; - k5.type = OLAP_FIELD_TYPE_DATETIME; - k5.length = 8; - k5.is_key = true; - k5.index_length = 8; - k5.is_allow_null = true; - tablet_schema->push_back(k5); - - FieldInfo k6; - k6.name = "k6"; - k6.type = OLAP_FIELD_TYPE_DECIMAL; - k6.length = 12; - k6.precision = 6; - k6.frac = 3; - k6.is_key = true; - k6.index_length = 12; - k6.is_allow_null = true; - tablet_schema->push_back(k6); - - FieldInfo k7; - k7.name = "k7"; - k7.type = OLAP_FIELD_TYPE_CHAR; - k7.length = 4; - k7.default_value = "char"; - k7.is_key = true; - k7.index_length = 4; - k7.is_allow_null = true; - tablet_schema->push_back(k7); - - FieldInfo v1; - v1.name = "v1"; - v1.type = OLAP_FIELD_TYPE_BIGINT; - v1.length = 8; - v1.aggregation = OLAP_FIELD_AGGREGATION_SUM; - v1.is_key = false; - v1.is_allow_null = true; - tablet_schema->push_back(v1); - - FieldInfo v2; - v2.name = "v2"; - v2.type = OLAP_FIELD_TYPE_VARCHAR; - v2.length = 16 + OLAP_STRING_MAX_BYTES; - v2.aggregation = OLAP_FIELD_AGGREGATION_REPLACE; - v2.is_key = false; - v2.is_allow_null = true; - tablet_schema->push_back(v2); - - FieldInfo v3; - v3.name = "v3"; - v3.type = OLAP_FIELD_TYPE_LARGEINT; - v3.length = 16; - v3.aggregation = OLAP_FIELD_AGGREGATION_MAX; - v3.is_key = false; - v3.is_allow_null = true; - tablet_schema->push_back(v3); - - FieldInfo v4; - v4.name = "v4"; - v4.type = OLAP_FIELD_TYPE_DECIMAL; - v4.length = 12; - v4.aggregation = OLAP_FIELD_AGGREGATION_MIN; - v4.is_key = false; - v4.is_allow_null = true; - tablet_schema->push_back(v4); - - FieldInfo v5; - v5.name = "v5"; - v5.type = OLAP_FIELD_TYPE_HLL; - v5.length = HLL_COLUMN_DEFAULT_LEN; - v5.aggregation = OLAP_FIELD_AGGREGATION_HLL_UNION; - v5.is_key = false; - v5.is_allow_null = true; - tablet_schema->push_back(v5); +void set_tablet_schema_for_init(TabletSchema* tablet_schema) { + TabletSchemaPB tablet_schema_pb; + ColumnPB* column_1 = tablet_schema_pb.add_column(); + column_1->set_unique_id(1); + column_1->set_name("column_1"); + column_1->set_type("TINYINT"); + column_1->set_is_key(true); + column_1->set_is_nullable(true); + column_1->set_length(1); + column_1->set_index_length(1); + + ColumnPB* column_2 = tablet_schema_pb.add_column(); + column_2->set_unique_id(2); + column_2->set_name("column_2"); + column_2->set_type("SMALLINT"); + column_2->set_is_key(true); + column_2->set_is_nullable(true); + column_2->set_length(2); + column_2->set_index_length(2); + column_2->set_default_value("0"); + + ColumnPB* column_3 = tablet_schema_pb.add_column(); + column_3->set_unique_id(3); + column_3->set_name("column_3"); + column_3->set_type("INT"); + column_3->set_is_key(true); + column_3->set_is_nullable(true); + column_3->set_length(4); + column_3->set_index_length(4); + + ColumnPB* column_4 = tablet_schema_pb.add_column(); + column_4->set_unique_id(4); + column_4->set_name("column_4"); + column_4->set_type("DATE"); + column_4->set_is_key(true); + column_4->set_is_nullable(true); + column_4->set_length(3); + column_4->set_index_length(3); + + ColumnPB* column_5 = tablet_schema_pb.add_column(); + column_5->set_unique_id(5); + column_5->set_name("column_5"); + column_5->set_type("DATETIME"); + column_5->set_is_key(true); + column_5->set_is_nullable(true); + column_5->set_length(8); + column_5->set_index_length(8); + + ColumnPB* column_6 = tablet_schema_pb.add_column(); + column_6->set_unique_id(6); + column_6->set_name("column_6"); + column_6->set_type("DECIMAL"); + column_6->set_is_key(true); + column_6->set_is_nullable(true); + column_6->set_length(12); + column_6->set_index_length(12); + column_6->set_frac(3); + column_6->set_precision(6); + + ColumnPB* column_7 = tablet_schema_pb.add_column(); + column_7->set_unique_id(7); + column_7->set_name("column_7"); + column_7->set_type("CHAR"); + column_7->set_is_key(true); + column_7->set_is_nullable(true); + column_7->set_length(4); + column_7->set_index_length(4); + column_7->set_default_value("char"); + + ColumnPB* column_8 = tablet_schema_pb.add_column(); + column_8->set_unique_id(8); + column_8->set_name("column_8"); + column_8->set_type("BIGINT"); + column_8->set_is_nullable(true); + column_8->set_length(8); + column_8->set_aggregation("SUM"); + column_8->set_is_key(false); + + ColumnPB* column_9 = tablet_schema_pb.add_column(); + column_9->set_unique_id(9); + column_9->set_name("column_9"); + column_9->set_type("VARCHAR"); + column_9->set_is_nullable(true); + column_9->set_length(16 + OLAP_STRING_MAX_BYTES); + column_9->set_aggregation("REPLACE"); + column_9->set_is_key(false); + + ColumnPB* column_10 = tablet_schema_pb.add_column(); + column_10->set_unique_id(10); + column_10->set_name("column_10"); + column_10->set_type("LARGEINT"); + column_10->set_is_nullable(true); + column_10->set_length(16); + column_10->set_aggregation("MAX"); + column_10->set_is_key(false); + + ColumnPB* column_11 = tablet_schema_pb.add_column(); + column_11->set_unique_id(11); + column_11->set_name("column_11"); + column_11->set_type("DECIMAL"); + column_11->set_is_nullable(true); + column_11->set_length(12); + column_11->set_aggregation("MIN"); + column_11->set_is_key(false); + + ColumnPB* column_12 = tablet_schema_pb.add_column(); + column_12->set_unique_id(12); + column_12->set_name("column_12"); + column_12->set_type("HLL"); + column_12->set_is_nullable(true); + column_12->set_length(HLL_COLUMN_DEFAULT_LEN); + column_12->set_aggregation("HLL_UNION"); + column_12->set_is_key(false); + + tablet_schema->init_from_pb(tablet_schema_pb); } -void set_tablet_schema_for_scan_key(std::vector* tablet_schema) { - FieldInfo k1; - k1.name = "k1"; - k1.type = OLAP_FIELD_TYPE_CHAR; - k1.length = 4; - k1.index_length = 4; - k1.default_value = "char"; - k1.is_key = true; - k1.is_allow_null = true; - tablet_schema->push_back(k1); - - FieldInfo k2; - k2.name = "k2"; - k2.type = OLAP_FIELD_TYPE_VARCHAR; - k2.length = 16 + OLAP_STRING_MAX_BYTES; - k2.index_length = 20; - k2.is_key = true; - k2.is_allow_null = true; - tablet_schema->push_back(k2); - - FieldInfo v1; - v1.name = "v1"; - v1.type = OLAP_FIELD_TYPE_LARGEINT; - v1.length = 16; - v1.aggregation = OLAP_FIELD_AGGREGATION_MAX; - v1.is_key = false; - v1.is_allow_null = true; - tablet_schema->push_back(v1); - - FieldInfo v2; - v2.name = "v2"; - v2.type = OLAP_FIELD_TYPE_DECIMAL; - v2.length = 12; - v2.aggregation = OLAP_FIELD_AGGREGATION_MIN; - v2.is_key = false; - v2.is_allow_null = true; - tablet_schema->push_back(v2); +void set_tablet_schema_for_scan_key(TabletSchema* tablet_schema) { + TabletSchemaPB tablet_schema_pb; + + ColumnPB* column_1 = tablet_schema_pb.add_column(); + column_1->set_unique_id(1); + column_1->set_name("column_1"); + column_1->set_type("CHAR"); + column_1->set_is_key(true); + column_1->set_is_nullable(true); + column_1->set_length(4); + column_1->set_index_length(4); + column_1->set_default_value("char"); + + ColumnPB* column_2 = tablet_schema_pb.add_column(); + column_2->set_unique_id(2); + column_2->set_name("column_2"); + column_2->set_type("VARCHAR"); + column_2->set_is_key(true); + column_2->set_is_nullable(true); + column_2->set_length(16 + OLAP_STRING_MAX_BYTES); + column_2->set_index_length(20); + + ColumnPB* column_3 = tablet_schema_pb.add_column(); + column_3->set_unique_id(3); + column_3->set_name("column_3"); + column_3->set_type("LARGEINT"); + column_3->set_is_nullable(true); + column_3->set_length(16); + column_3->set_aggregation("MAX"); + column_3->set_is_key(false); + + ColumnPB* column_4 = tablet_schema_pb.add_column(); + column_4->set_unique_id(9); + column_4->set_name("column_4"); + column_4->set_type("DECIMAL"); + column_4->set_is_nullable(true); + column_4->set_length(12); + column_4->set_aggregation("MIN"); + column_4->set_is_key(false); + + tablet_schema->init_from_pb(tablet_schema_pb); } -void set_tablet_schema_for_cmp_and_aggregate(std::vector* tablet_schema) { - FieldInfo k1; - k1.name = "k1"; - k1.type = OLAP_FIELD_TYPE_CHAR; - k1.length = 4; - k1.default_value = "char"; - k1.is_key = true; - k1.index_length = 4; - k1.is_allow_null = true; - tablet_schema->push_back(k1); - - FieldInfo k2; - k2.name = "k2"; - k2.type = OLAP_FIELD_TYPE_INT; - k2.length = 4; - k2.is_key = true; - k2.index_length = 4; - k2.is_allow_null = true; - tablet_schema->push_back(k2); - - FieldInfo v1; - v1.name = "v1"; - v1.type = OLAP_FIELD_TYPE_LARGEINT; - v1.length = 16; - v1.aggregation = OLAP_FIELD_AGGREGATION_SUM; - v1.is_key = false; - v1.is_allow_null = true; - tablet_schema->push_back(v1); - - FieldInfo v2; - v2.name = "v2"; - v2.type = OLAP_FIELD_TYPE_DOUBLE; - v2.length = 8; - v2.aggregation = OLAP_FIELD_AGGREGATION_MIN; - v2.is_key = false; - v2.is_allow_null = true; - tablet_schema->push_back(v2); - - FieldInfo v3; - v3.name = "v3"; - v3.type = OLAP_FIELD_TYPE_DECIMAL; - v3.length = 12; - v3.aggregation = OLAP_FIELD_AGGREGATION_MAX; - v3.is_key = false; - v3.is_allow_null = true; - tablet_schema->push_back(v3); - - FieldInfo v4; - v4.name = "v4"; - v4.type = OLAP_FIELD_TYPE_VARCHAR; - v4.length = 16 + OLAP_STRING_MAX_BYTES; - v4.aggregation = OLAP_FIELD_AGGREGATION_REPLACE; - v4.is_key = false; - v4.is_allow_null = true; - tablet_schema->push_back(v4); +void set_tablet_schema_for_cmp_and_aggregate(TabletSchema* tablet_schema) { + TabletSchemaPB tablet_schema_pb; + + ColumnPB* column_1 = tablet_schema_pb.add_column(); + column_1->set_unique_id(1); + column_1->set_name("column_1"); + column_1->set_type("CHAR"); + column_1->set_is_key(true); + column_1->set_is_nullable(true); + column_1->set_length(4); + column_1->set_index_length(4); + column_1->set_default_value("char"); + + ColumnPB* column_2 = tablet_schema_pb.add_column(); + column_2->set_unique_id(2); + column_2->set_name("column_2"); + column_2->set_type("INT"); + column_2->set_is_key(true); + column_2->set_is_nullable(true); + column_2->set_length(4); + column_2->set_index_length(4); + + ColumnPB* column_3 = tablet_schema_pb.add_column(); + column_3->set_unique_id(3); + column_3->set_name("column_3"); + column_3->set_type("LARGEINT"); + column_3->set_is_nullable(true); + column_3->set_length(16); + column_3->set_aggregation("SUM"); + column_3->set_is_key(false); + + ColumnPB* column_4 = tablet_schema_pb.add_column(); + column_4->set_unique_id(9); + column_4->set_name("column_4"); + column_4->set_type("DOUBLE"); + column_4->set_is_nullable(true); + column_4->set_length(8); + column_4->set_aggregation("MIN"); + column_4->set_is_key(false); + + ColumnPB* column_5 = tablet_schema_pb.add_column(); + column_5->set_unique_id(3); + column_5->set_name("column_5"); + column_5->set_type("DECIMAL"); + column_5->set_is_nullable(true); + column_5->set_length(12); + column_5->set_aggregation("MAX"); + column_5->set_is_key(false); + + ColumnPB* column_6 = tablet_schema_pb.add_column(); + column_6->set_unique_id(9); + column_6->set_name("column_6"); + column_6->set_type("VARCHAR"); + column_6->set_is_nullable(true); + column_6->set_length(16 + OLAP_STRING_MAX_BYTES); + column_6->set_aggregation("REPLACE"); + column_6->set_is_key(false); + + tablet_schema->init_from_pb(tablet_schema_pb); } class TestRowCursor : public testing::Test { @@ -250,7 +262,7 @@ class TestRowCursor : public testing::Test { }; TEST_F(TestRowCursor, InitRowCursor) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_init(&tablet_schema); RowCursor row; OLAPStatus res = row.init(tablet_schema); @@ -260,7 +272,7 @@ TEST_F(TestRowCursor, InitRowCursor) { } TEST_F(TestRowCursor, InitRowCursorWithColumnCount) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_init(&tablet_schema); RowCursor row; OLAPStatus res = row.init(tablet_schema, 5); @@ -272,11 +284,11 @@ TEST_F(TestRowCursor, InitRowCursorWithColumnCount) { } TEST_F(TestRowCursor, InitRowCursorWithColIds) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_init(&tablet_schema); std::vector col_ids; - for (size_t i = 0; i < tablet_schema.size() / 2; ++i) { + for (size_t i = 0; i < tablet_schema.num_columns() / 2; ++i) { col_ids.push_back(i * 2); } @@ -288,7 +300,7 @@ TEST_F(TestRowCursor, InitRowCursorWithColIds) { } TEST_F(TestRowCursor, InitRowCursorWithScanKey) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_scan_key(&tablet_schema); std::vector scan_keys; @@ -311,7 +323,7 @@ TEST_F(TestRowCursor, InitRowCursorWithScanKey) { } TEST_F(TestRowCursor, SetMinAndMaxKey) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_init(&tablet_schema); RowCursor min_row; @@ -321,7 +333,7 @@ TEST_F(TestRowCursor, SetMinAndMaxKey) { res = min_row.build_min_key(); ASSERT_EQ(res, OLAP_SUCCESS); - for (size_t i = 0; i < tablet_schema.size(); ++i) { + for (size_t i = 0; i < tablet_schema.num_columns(); ++i) { ASSERT_TRUE(min_row.is_min(i)); } @@ -332,7 +344,7 @@ TEST_F(TestRowCursor, SetMinAndMaxKey) { } TEST_F(TestRowCursor, EqualAndCompare) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_cmp_and_aggregate(&tablet_schema); RowCursor left; @@ -373,7 +385,7 @@ TEST_F(TestRowCursor, EqualAndCompare) { } TEST_F(TestRowCursor, IndexCmp) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_cmp_and_aggregate(&tablet_schema); RowCursor left; @@ -414,7 +426,7 @@ TEST_F(TestRowCursor, IndexCmp) { } TEST_F(TestRowCursor, FullKeyCmp) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_cmp_and_aggregate(&tablet_schema); RowCursor left; @@ -454,7 +466,7 @@ TEST_F(TestRowCursor, FullKeyCmp) { } TEST_F(TestRowCursor, AggregateWithoutNull) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_cmp_and_aggregate(&tablet_schema); RowCursor row; @@ -514,7 +526,7 @@ TEST_F(TestRowCursor, AggregateWithoutNull) { } TEST_F(TestRowCursor, AggregateWithNull) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_cmp_and_aggregate(&tablet_schema); RowCursor row; diff --git a/be/test/olap/rowset/rowset_meta_manager_test.cpp b/be/test/olap/rowset/rowset_meta_manager_test.cpp index 0d9d3345d241d4..cce0693cedf74a 100644 --- a/be/test/olap/rowset/rowset_meta_manager_test.cpp +++ b/be/test/olap/rowset/rowset_meta_manager_test.cpp @@ -75,27 +75,27 @@ TEST_F(RowsetMetaManagerTest, TestSaveAndGetAndRemove) { uint64_t rowset_id = 10000; RowsetMeta rowset_meta; rowset_meta.init_from_json(_json_rowset_meta); - ASSERT_EQ(rowset_meta.get_rowset_id(), rowset_id); - NewStatus status = RowsetMetaManager::save(_meta, rowset_id, &rowset_meta); - ASSERT_TRUE(status.ok()); + ASSERT_EQ(rowset_meta.rowset_id(), rowset_id); + OLAPStatus status = RowsetMetaManager::save(_meta, rowset_id, &rowset_meta); + ASSERT_TRUE(status == OLAP_SUCCESS); std::string json_rowset_meta_read; status = RowsetMetaManager::get_json_rowset_meta(_meta, rowset_id, &json_rowset_meta_read); - ASSERT_TRUE(status.ok()); + ASSERT_TRUE(status == OLAP_SUCCESS); ASSERT_EQ(_json_rowset_meta, json_rowset_meta_read); status = RowsetMetaManager::remove(_meta, rowset_id); - ASSERT_TRUE(status.ok()); - RowsetMeta rowset_meta_read; - status = RowsetMetaManager::get_rowset_meta(_meta, rowset_id, &rowset_meta_read); - ASSERT_TRUE(!status.ok()); + ASSERT_TRUE(status == OLAP_SUCCESS); + RowsetMetaSharedPtr rowset_meta_read(new RowsetMeta()); + status = RowsetMetaManager::get_rowset_meta(_meta, rowset_id, rowset_meta_read); + ASSERT_TRUE(status != OLAP_SUCCESS); } TEST_F(RowsetMetaManagerTest, TestLoad) { uint64_t rowset_id = 10000; - NewStatus status = RowsetMetaManager::load_json_rowset_meta(_meta, rowset_meta_path); - ASSERT_TRUE(status.ok()); + OLAPStatus status = RowsetMetaManager::load_json_rowset_meta(_meta, rowset_meta_path); + ASSERT_TRUE(status == OLAP_SUCCESS); std::string json_rowset_meta_read; status = RowsetMetaManager::get_json_rowset_meta(_meta, rowset_id, &json_rowset_meta_read); - ASSERT_TRUE(status.ok()); + ASSERT_TRUE(status == OLAP_SUCCESS); ASSERT_EQ(_json_rowset_meta, json_rowset_meta_read); } diff --git a/be/test/olap/run_length_byte_test.cpp b/be/test/olap/run_length_byte_test.cpp index 65db8a833b3aa9..d6a7535fcfa420 100755 --- a/be/test/olap/run_length_byte_test.cpp +++ b/be/test/olap/run_length_byte_test.cpp @@ -21,9 +21,9 @@ #include "olap/out_stream.h" #include "olap/in_stream.h" #include "olap/file_stream.h" -#include "olap/run_length_byte_writer.h" -#include "olap/run_length_byte_reader.h" -#include "olap/column_reader.h" +#include "olap/rowset/run_length_byte_writer.h" +#include "olap/rowset/run_length_byte_reader.h" +#include "olap/rowset/column_reader.h" #include "olap/stream_index_reader.h" #include "olap/stream_index_writer.h" #include "util/logging.h" diff --git a/be/test/olap/run_length_integer_test.cpp b/be/test/olap/run_length_integer_test.cpp index fafb4a035a74b6..ff27fdac9aef11 100755 --- a/be/test/olap/run_length_integer_test.cpp +++ b/be/test/olap/run_length_integer_test.cpp @@ -20,8 +20,8 @@ #include "olap/byte_buffer.h" #include "olap/out_stream.h" #include "olap/in_stream.h" -#include "olap/run_length_integer_writer.h" -#include "olap/run_length_integer_reader.h" +#include "olap/rowset/run_length_integer_writer.h" +#include "olap/rowset/run_length_integer_reader.h" #include "olap/stream_index_writer.h" #include "olap/stream_index_reader.h" #include "util/logging.h" diff --git a/be/test/olap/tablet_meta_manager_test.cpp b/be/test/olap/tablet_meta_manager_test.cpp index 4dc4d8def74483..28eef409fa18f3 100755 --- a/be/test/olap/tablet_meta_manager_test.cpp +++ b/be/test/olap/tablet_meta_manager_test.cpp @@ -45,9 +45,9 @@ class TabletMetaManagerTest : public testing::Test { virtual void SetUp() { std::string root_path = "./store"; ASSERT_TRUE(boost::filesystem::create_directory(root_path)); - _store = new(std::nothrow) DataDir(root_path); - ASSERT_NE(nullptr, _store); - Status st = _store->init(); + _data_dir = new(std::nothrow) DataDir(root_path); + ASSERT_NE(nullptr, _data_dir); + Status st = _data_dir->init(); ASSERT_TRUE(st.ok()); ASSERT_TRUE(boost::filesystem::exists("./store/meta")); @@ -59,55 +59,47 @@ class TabletMetaManagerTest : public testing::Test { } _json_header = _json_header.substr(0, _json_header.size() - 1); _json_header = _json_header.substr(0, _json_header.size() - 1); - std::cout << "set up finish" << std::endl; } virtual void TearDown() { - delete _store; + delete _data_dir; ASSERT_TRUE(boost::filesystem::remove_all("./store")); } private: - DataDir* _store; + DataDir* _data_dir; std::string _json_header; }; -TEST_F(TabletMetaManagerTest, TestConvertedFlag) { - bool converted_flag; - OLAPStatus s = TabletMetaManager::get_header_converted(_store, converted_flag); - ASSERT_EQ(false, converted_flag); - s = TabletMetaManager::set_converted_flag(_store); - ASSERT_EQ(OLAP_SUCCESS, s); - s = TabletMetaManager::get_header_converted(_store, converted_flag); - ASSERT_EQ(true, converted_flag); -} - TEST_F(TabletMetaManagerTest, TestSaveAndGetAndRemove) { - const TTabletId tablet_id = 20487; - const TSchemaHash schema_hash = 1520686811; - TabletMeta header; - bool ret = json2pb::JsonToProtoMessage(_json_header, &header); + const TTabletId tablet_id = 15672; + const TSchemaHash schema_hash = 567997577; + TabletMetaPB tablet_meta_pb; + bool ret = json2pb::JsonToProtoMessage(_json_header, &tablet_meta_pb); ASSERT_TRUE(ret); - OLAPStatus s = TabletMetaManager::save(_store, tablet_id, schema_hash, &header); + TabletMeta tablet_meta; + OLAPStatus s = tablet_meta.init_from_pb(tablet_meta_pb); + ASSERT_EQ(OLAP_SUCCESS, s); + s = TabletMetaManager::save(_data_dir, tablet_id, schema_hash, &tablet_meta); ASSERT_EQ(OLAP_SUCCESS, s); std::string json_header_read; - s = TabletMetaManager::get_json_header(_store, tablet_id, schema_hash, &json_header_read); + s = TabletMetaManager::get_json_header(_data_dir, tablet_id, schema_hash, &json_header_read); ASSERT_EQ(OLAP_SUCCESS, s); ASSERT_EQ(_json_header, json_header_read); - s = TabletMetaManager::remove(_store, tablet_id, schema_hash); + s = TabletMetaManager::remove(_data_dir, tablet_id, schema_hash); ASSERT_EQ(OLAP_SUCCESS, s); TabletMeta header_read; - s = TabletMetaManager::get_header(_store, tablet_id, schema_hash, &header_read); + s = TabletMetaManager::get_header(_data_dir, tablet_id, schema_hash, &header_read); ASSERT_EQ(OLAP_ERR_META_KEY_NOT_FOUND, s); } TEST_F(TabletMetaManagerTest, TestLoad) { - const TTabletId tablet_id = 20487; - const TSchemaHash schema_hash = 1520686811; - OLAPStatus s = TabletMetaManager::load_json_header(_store, header_path); + const TTabletId tablet_id = 15672; + const TSchemaHash schema_hash = 567997577; + OLAPStatus s = TabletMetaManager::load_json_header(_data_dir, header_path); ASSERT_EQ(OLAP_SUCCESS, s); std::string json_header_read; - s = TabletMetaManager::get_json_header(_store, tablet_id, schema_hash, &json_header_read); + s = TabletMetaManager::get_json_header(_data_dir, tablet_id, schema_hash, &json_header_read); ASSERT_EQ(OLAP_SUCCESS, s); ASSERT_EQ(_json_header, json_header_read); } diff --git a/be/test/olap/test_data/header.txt b/be/test/olap/test_data/header.txt index a657c2c40b12b2..b56aa5e3c0d5c0 100644 --- a/be/test/olap/test_data/header.txt +++ b/be/test/olap/test_data/header.txt @@ -1,152 +1,114 @@ { - "num_rows_per_data_block": 1024, - "file_version": [ + "table_id": 15670, + "partition_id": 15671, + "tablet_id": 15672, + "schema_hash": 567997577, + "shard_id": 34, + "creation_time": 1553765664, + "cumulative_layer_point": 2, + "schema": { + "keys_type": "AGG_KEYS", + "column": [ + { + "unique_id": 0, + "name": "k1", + "type": "BIGINT", + "is_key": true, + "aggregation": "NONE", + "is_nullable": false, + "length": 8, + "index_length": 8 + }, + { + "unique_id": 1, + "name": "v1", + "type": "HLL", + "is_key": false, + "aggregation": "HLL_UNION", + "is_nullable": false, + "default_value": "MA==", + "length": 16387, + "index_length": 16 + }, + { + "unique_id": 2, + "name": "v2", + "type": "INT", + "is_key": false, + "aggregation": "SUM", + "is_nullable": false, + "length": 4, + "index_length": 4 + } + ], + "num_short_key_columns": 1, + "num_rows_per_row_block": 1024, + "compress_kind": "COMPRESS_LZ4", + "next_column_unique_id": 3 + }, + "rs_metas": [ { - "num_segments": 1, + "rowset_id": 540072, + "tablet_id": 15673, + "tablet_schema_hash": 567997577, + "rowset_type": "ALPHA_ROWSET", + "rowset_state": "VISIBLE", "start_version": 0, "end_version": 1, "version_hash": 0, - "max_timestamp": 0, - "index_size": 67, - "data_size": 477, "num_rows": 0, - "creation_time": 1534750461, - "delta_pruning": { - "column_pruning": [ - { - "min": "OTk5OS0xMi0zMQ==", - "max": "MC0wMC0wMA==", - "null_flag": false - }, - { - "min": "MjE0NzQ4MzY0Nw==", - "max": "MA==", - "null_flag": false - }, - { - "min": "MzI3Njc=", - "max": "MA==", - "null_flag": false - }, - { - "min": "/w==", - "max": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "null_flag": false - } - ] - } + "total_disk_size": 0, + "data_disk_size": 0, + "index_disk_size": 0, + "empty": true, + "creation_time": 1553765664, + "extra_properties": "CgwIABAAGAAgACgAOAE=" }, { - "num_segments": 1, + "rowset_id": 540081, + "tablet_id": 15673, + "txn_id": 4042, + "tablet_schema_hash": 567997577, + "rowset_type": "ALPHA_ROWSET", + "rowset_state": "VISIBLE", "start_version": 2, "end_version": 2, - "version_hash": 0, - "max_timestamp": 0, - "index_size": 67, - "data_size": 477, - "num_rows": 0, - "creation_time": 1534750461, - "delta_pruning": { - "column_pruning": [ - { - "min": "OTk5OS0xMi0zMQ==", - "max": "MC0wMC0wMA==", - "null_flag": false - }, - { - "min": "MjE0NzQ4MzY0Nw==", - "max": "MA==", - "null_flag": false - }, - { - "min": "MzI3Njc=", - "max": "MA==", - "null_flag": false - }, - { - "min": "/w==", - "max": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "null_flag": false - } - ] - } + "version_hash": 8391828013814912580, + "num_rows": 3929, + "total_disk_size": 84699, + "data_disk_size": 84464, + "index_disk_size": 235, + "empty": false, + "load_id": { + "hi": -5350970832824939812, + "lo": -6717994719194512122 + }, + "creation_time": 1553765670, + "extra_properties": "Cj8IABABGIkBIMKRBCj3GDItChQtOTIyMTU1MjY3MjkzMDc2NDM3MBITOTIxMzI0NDQwNTY3MzIwODI3MhgAOAAKPggBEAEYYiCuggEo4gUyLQoULTkyMDQ4NjgyNzgzOTU5NTgyOTYSEzkyMjE4NjQ3ODUyMjk2NDE3MDYYADgA" } ], - "cumulative_layer_point": 2, - "num_short_key_fields": 4, - "column": [ + "inc_rs_metas": [ { - "name": "event_day", - "type": "DATE", - "aggregation": "NONE", - "length": 3, - "is_key": true, - "index_length": 3, - "is_allow_null": true, - "unique_id": 0, - "is_root_column": true - }, - { - "name": "siteid", - "type": "INT", - "aggregation": "NONE", - "length": 4, - "is_key": true, - "default_value": "10", - "index_length": 4, - "is_allow_null": true, - "unique_id": 1, - "is_root_column": true - }, - { - "name": "citycode", - "type": "SMALLINT", - "aggregation": "NONE", - "length": 2, - "is_key": true, - "index_length": 2, - "is_allow_null": true, - "unique_id": 2, - "is_root_column": true - }, - { - "name": "username", - "type": "VARCHAR", - "aggregation": "NONE", - "length": 34, - "is_key": true, - "default_value": "", - "index_length": 20, - "is_allow_null": true, - "unique_id": 3, - "is_root_column": true - }, - { - "name": "pv", - "type": "BIGINT", - "aggregation": "SUM", - "length": 8, - "is_key": false, - "default_value": "0", - "index_length": 8, - "is_allow_null": true, - "unique_id": 4, - "is_root_column": true + "rowset_id": 540081, + "tablet_id": 15673, + "txn_id": 4042, + "tablet_schema_hash": 567997577, + "rowset_type": "ALPHA_ROWSET", + "rowset_state": "VISIBLE", + "start_version": 2, + "end_version": 2, + "version_hash": 8391828013814912580, + "num_rows": 3929, + "total_disk_size": 84699, + "data_disk_size": 84464, + "index_disk_size": 235, + "empty": false, + "load_id": { + "hi": -5350970832824939812, + "lo": -6717994719194512122 + }, + "creation_time": 1553765670, + "extra_properties": "Cj8IABABGIkBIMKRBCj3GDItChQtOTIyMTU1MjY3MjkzMDc2NDM3MBITOTIxMzI0NDQwNTY3MzIwODI3MhgAOAAKPggBEAEYYiCuggEo4gUyLQoULTkyMDQ4NjgyNzgzOTU5NTgyOTYSEzkyMjE4NjQ3ODUyMjk2NDE3MDYYADgA" } - ], - "creation_time": 1534750461, - "selectivity": [ - 1, - 1, - 1, - 1 - ], - "data_file_type": "COLUMN_ORIENTED_FILE", - "next_column_unique_id": 5, - "compress_kind": "COMPRESS_LZ4", - "segment_size": 268435456, - "keys_type": "AGG_KEYS", - "tablet_id": 20487, - "schema_hash": 1520686811, - "shard": 0 + ] } diff --git a/be/test/olap/test_data/rowset_meta.json b/be/test/olap/test_data/rowset_meta.json index db962f2323cab7..866b4c8596e683 100644 --- a/be/test/olap/test_data/rowset_meta.json +++ b/be/test/olap/test_data/rowset_meta.json @@ -1,12 +1,17 @@ { "rowset_id": 10000, - "tablet_id": 20487, - "tablet_schema_hash": 1520686811, + "tablet_id": 12046, + "tablet_schema_hash": 365187263, "rowset_type": "ALPHA_ROWSET", "rowset_state": "VISIBLE", "start_version": 0, "end_version": 1, - "row_number": 123456, - "total_disk_size": 100000, - "data_disk_size": 95000 + "version_hash": 0, + "num_rows": 0, + "total_disk_size": 0, + "data_disk_size": 0, + "index_disk_size": 0, + "empty": true, + "creation_time": 1552911435, + "extra_properties": "CgwIABAAGAAgACgAOAE=" } diff --git a/be/test/util/doris_metrics_test.cpp b/be/test/util/doris_metrics_test.cpp index 1765ffaafa9b12..08631c818f8c20 100644 --- a/be/test/util/doris_metrics_test.cpp +++ b/be/test/util/doris_metrics_test.cpp @@ -227,14 +227,6 @@ TEST_F(DorisMetricsTest, Normal) { ASSERT_TRUE(metric != nullptr); ASSERT_STREQ("22", ((SimpleMetric*)metric)->to_string().c_str()); } - { - DorisMetrics::cancel_delete_requests_total.increment(23); - auto metric = metrics->get_metric("engine_requests_total", - MetricLabels().add("type", "cancel_delete") - .add("status", "total")); - ASSERT_TRUE(metric != nullptr); - ASSERT_STREQ("23", ((SimpleMetric*)metric)->to_string().c_str()); - } // comapction { DorisMetrics::base_compaction_deltas_total.increment(30); diff --git a/run-ut.sh b/run-ut.sh index cc594a22e59a6a..62f6106b097abd 100755 --- a/run-ut.sh +++ b/run-ut.sh @@ -151,10 +151,10 @@ ${DORIS_TEST_BINARY_DIR}/util/uid_util_test ${DORIS_TEST_BINARY_DIR}/util/aes_util_test ${DORIS_TEST_BINARY_DIR}/util/string_util_test -## Running common Unittest +# Running common Unittest ${DORIS_TEST_BINARY_DIR}/common/resource_tls_test -## Running exec unit test +# Running exec unit test ${DORIS_TEST_BINARY_DIR}/exec/plain_text_line_reader_uncompressed_test ${DORIS_TEST_BINARY_DIR}/exec/plain_text_line_reader_gzip_test ${DORIS_TEST_BINARY_DIR}/exec/plain_text_line_reader_bzip_test @@ -165,7 +165,7 @@ ${DORIS_TEST_BINARY_DIR}/exec/broker_scan_node_test ${DORIS_TEST_BINARY_DIR}/exec/tablet_info_test ${DORIS_TEST_BINARY_DIR}/exec/tablet_sink_test -## Running runtime Unittest +# Running runtime Unittest ${DORIS_TEST_BINARY_DIR}/runtime/fragment_mgr_test ${DORIS_TEST_BINARY_DIR}/runtime/decimal_value_test ${DORIS_TEST_BINARY_DIR}/runtime/datetime_value_test @@ -177,7 +177,6 @@ ${DORIS_TEST_BINARY_DIR}/runtime/stream_load_pipe_test ${DORIS_TEST_BINARY_DIR}/runtime/tablet_writer_mgr_test ${DORIS_TEST_BINARY_DIR}/runtime/snapshot_loader_test ${DORIS_TEST_BINARY_DIR}/runtime/user_function_cache_test -## Running expr Unittest # Running http ${DORIS_TEST_BINARY_DIR}/http/metrics_action_test @@ -200,7 +199,7 @@ ${DORIS_TEST_BINARY_DIR}/olap/in_list_predicate_test ${DORIS_TEST_BINARY_DIR}/olap/null_predicate_test ${DORIS_TEST_BINARY_DIR}/olap/file_helper_test ${DORIS_TEST_BINARY_DIR}/olap/file_utils_test -#${DORIS_TEST_BINARY_DIR}/olap/delete_handler_test +${DORIS_TEST_BINARY_DIR}/olap/delete_handler_test ${DORIS_TEST_BINARY_DIR}/olap/column_reader_test ${DORIS_TEST_BINARY_DIR}/olap/row_cursor_test ${DORIS_TEST_BINARY_DIR}/olap/skiplist_test