From 257de53ac7dd630ebc08443993488c188a098475 Mon Sep 17 00:00:00 2001 From: huangkangping Date: Mon, 8 Apr 2019 14:20:29 +0800 Subject: [PATCH 1/2] Fix BE unit tests --- be/src/olap/olap_meta.cpp | 6 +- be/src/olap/olap_server.cpp | 16 +- be/src/olap/row_block.cpp | 29 +- be/src/olap/row_block.h | 6 +- be/src/olap/row_cursor.cpp | 57 +- be/src/olap/row_cursor.h | 11 +- be/src/olap/rowset/column_data.cpp | 8 +- be/src/olap/rowset/column_reader.cpp | 16 +- be/src/olap/rowset/column_reader.h | 6 + be/src/olap/rowset/column_writer.cpp | 6 +- be/src/olap/rowset/column_writer.h | 3 +- be/src/olap/rowset/rowset_meta_manager.cpp | 10 +- be/src/olap/rowset/rowset_meta_manager.h | 2 +- be/src/olap/rowset/segment_writer.cpp | 2 +- be/src/olap/tablet.cpp | 5 +- be/src/olap/tablet.h | 3 +- be/src/olap/tablet_manager.cpp | 8 +- be/src/olap/tablet_meta.cpp | 4 +- be/src/olap/tablet_meta_manager.cpp | 13 +- be/src/olap/tablet_meta_manager.h | 8 +- be/src/olap/tablet_schema.cpp | 2 +- be/src/olap/txn_manager.cpp | 6 +- be/src/olap/txn_manager.h | 2 +- be/src/runtime/vectorized_row_batch.cpp | 6 +- be/src/runtime/vectorized_row_batch.h | 4 +- be/test/olap/bit_field_test.cpp | 4 +- be/test/olap/column_reader_test.cpp | 945 ++++++++---------- be/test/olap/comparison_predicate_test.cpp | 219 ++-- be/test/olap/delete_handler_test.cpp | 464 +++------ be/test/olap/delta_writer_test.cpp | 304 +----- be/test/olap/in_list_predicate_test.cpp | 137 +-- be/test/olap/null_predicate_test.cpp | 125 +-- be/test/olap/row_block_test.cpp | 353 ++----- be/test/olap/row_cursor_test.cpp | 413 ++++---- .../olap/rowset/rowset_meta_manager_test.cpp | 22 +- be/test/olap/run_length_byte_test.cpp | 6 +- be/test/olap/run_length_integer_test.cpp | 4 +- be/test/olap/tablet_meta_manager_test.cpp | 48 +- be/test/olap/test_data/header.txt | 236 ++--- be/test/olap/test_data/rowset_meta.json | 15 +- be/test/util/doris_metrics_test.cpp | 8 - run-ut.sh | 9 +- 42 files changed, 1473 insertions(+), 2078 deletions(-) diff --git a/be/src/olap/olap_meta.cpp b/be/src/olap/olap_meta.cpp index 4d6b84e634ecaf..28ad378de8b053 100755 --- a/be/src/olap/olap_meta.cpp +++ b/be/src/olap/olap_meta.cpp @@ -55,10 +55,10 @@ OlapMeta::~OlapMeta() { for (auto handle : _handles) { delete handle; } - if (_db != NULL) { + if (_db != nullptr) { _db->Close(); delete _db; - _db= NULL; + _db= nullptr; } } @@ -79,7 +79,7 @@ OLAPStatus OlapMeta::init() { meta_column_family.prefix_extractor.reset(NewFixedPrefixTransform(PREFIX_LENGTH)); column_families.emplace_back(META_COLUMN_FAMILY, meta_column_family); Status s = DB::Open(options, db_path, column_families, &_handles, &_db); - if (!s.ok() || _db == NULL) { + if (!s.ok() || _db == nullptr) { LOG(WARNING) << "rocks db open failed, reason:" << s.ToString(); return OLAP_ERR_META_OPEN_DB; } diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 48d530e36f1b2a..602c7bef8b1fd8 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -45,12 +45,13 @@ OLAPStatus StorageEngine::_start_bg_worker() { [this] { _garbage_sweeper_thread_callback(nullptr); }); - + _garbage_sweeper_thread.detach(); // start thread for monitoring the tablet with io error _disk_stat_monitor_thread = std::thread( [this] { _disk_stat_monitor_thread_callback(nullptr); }); + _disk_stat_monitor_thread.detach(); // start be and ce threads for merge data int32_t base_compaction_num_threads = config::base_compaction_num_threads; @@ -61,6 +62,9 @@ OLAPStatus StorageEngine::_start_bg_worker() { _base_compaction_thread_callback(nullptr); }); } + for (auto& thread : _base_compaction_threads) { + thread.detach(); + } int32_t cumulative_compaction_num_threads = config::cumulative_compaction_num_threads; _cumulative_compaction_threads.reserve(cumulative_compaction_num_threads); @@ -70,11 +74,15 @@ OLAPStatus StorageEngine::_start_bg_worker() { _cumulative_compaction_thread_callback(nullptr); }); } + for (auto& thread : _cumulative_compaction_threads) { + thread.detach(); + } _fd_cache_clean_thread = std::thread( [this] { _fd_cache_clean_callback(nullptr); }); + _fd_cache_clean_thread.detach(); // path scan and gc thread if (config::path_gc_check) { @@ -89,6 +97,12 @@ OLAPStatus StorageEngine::_start_bg_worker() { _path_gc_thread_callback((void*)data_dir); }); } + for (auto& thread : _path_scan_threads) { + thread.detach(); + } + for (auto& thread : _path_gc_threads) { + thread.detach(); + } } VLOG(10) << "init finished."; diff --git a/be/src/olap/row_block.cpp b/be/src/olap/row_block.cpp index f0c381fa80c456..0a4755bac9aed6 100644 --- a/be/src/olap/row_block.cpp +++ b/be/src/olap/row_block.cpp @@ -38,18 +38,33 @@ namespace doris { RowBlock::RowBlock(const TabletSchema* schema) : _capacity(0), - _schema(schema) { + _schema(schema), + _columns(nullptr) { + if (_schema != nullptr) { + _columns = &_schema->columns(); + } _tracker.reset(new MemTracker(-1)); _mem_pool.reset(new MemPool(_tracker.get())); } +RowBlock::RowBlock(const std::vector& columns) : + _capacity(0), + _schema(nullptr), + _columns(&columns) { + _tracker.reset(new MemTracker(-1)); + _mem_pool.reset(new MemPool(_tracker.get())); +} + RowBlock::~RowBlock() { delete[] _mem_buf; delete[] _storage_buf; } OLAPStatus RowBlock::init(const RowBlockInfo& block_info) { - _field_count = _schema->num_columns(); + if (_columns == nullptr) { + return OLAP_ERR_INIT_FAILED; + } + _field_count = _columns->size(); _info = block_info; _null_supported = block_info.null_supported; _capacity = _info.row_num; @@ -114,7 +129,7 @@ void RowBlock::_convert_storage_to_memory() { // some data file in history not suppored null size_t null_byte = has_nullbyte() ? 1 : 0; for (int col = 0; col < _field_count; ++col) { - const TabletColumn& column = _schema->column(col); + const TabletColumn& column = _columns->at(col); char* memory_ptr = _mem_buf + _field_offset_in_memory[col]; if (column.type() == OLAP_FIELD_TYPE_VARCHAR || column.type() == OLAP_FIELD_TYPE_HLL) { @@ -192,7 +207,7 @@ void RowBlock::_convert_memory_to_storage(uint32_t num_rows) { char* storage_variable_ptr = _storage_buf + num_rows * _storage_row_fixed_bytes; size_t null_byte = has_nullbyte() ? 1 : 0; for (int col = 0; col < _field_count; ++col) { - const TabletColumn& column = _schema->column(col); + const TabletColumn& column = _columns->at(col);; char* memory_ptr = _mem_buf + _field_offset_in_memory[col]; if (column.type() == OLAP_FIELD_TYPE_VARCHAR || column.type() == OLAP_FIELD_TYPE_HLL) { @@ -281,7 +296,7 @@ OLAPStatus RowBlock::find_row(const RowCursor& key, OLAPStatus res = OLAP_SUCCESS; RowCursor helper_cursor; - if ((res = helper_cursor.init(*_schema)) != OLAP_SUCCESS) { + if ((res = helper_cursor.init(*_columns)) != OLAP_SUCCESS) { OLAP_LOG_WARNING("Init helper cursor fail. [res=%d]", res); return OLAP_ERR_INIT_FAILED; } @@ -320,8 +335,8 @@ void RowBlock::_compute_layout() { size_t memory_size = 0; size_t storage_fixed_bytes = 0; size_t storage_variable_bytes = 0; - for (size_t col_id = 0; col_id < _schema->num_columns(); ++col_id) { - const TabletColumn& column = _schema->column(col_id); + for (size_t col_id = 0; col_id < _columns->size(); ++col_id) { + const TabletColumn& column = _columns->at(col_id); _field_offset_in_memory.push_back(memory_size); // All column has a nullbyte in memory diff --git a/be/src/olap/row_block.h b/be/src/olap/row_block.h index 19f86f8c5437ca..21f4f6ee33f140 100644 --- a/be/src/olap/row_block.h +++ b/be/src/olap/row_block.h @@ -61,6 +61,9 @@ class RowBlock { public: RowBlock(const TabletSchema* schema); + // this constructor is just for unit test + RowBlock(const std::vector& columns); + // 注意回收内部buffer ~RowBlock(); @@ -186,7 +189,8 @@ class RowBlock { uint32_t _capacity; RowBlockInfo _info; const TabletSchema* _schema; // 内部保存的schema句柄 - + const std::vector* _columns; + bool _null_supported; size_t _field_count = 0; diff --git a/be/src/olap/row_cursor.cpp b/be/src/olap/row_cursor.cpp index 4410c1a541bc86..94f72887c2125d 100644 --- a/be/src/olap/row_cursor.cpp +++ b/be/src/olap/row_cursor.cpp @@ -81,8 +81,8 @@ OLAPStatus RowCursor::_init(const std::vector& schema, for (auto cid : _columns) { const TabletColumn& column = schema[cid]; _field_array[cid] = Field::create(column); - if (_field_array[cid] == NULL) { - OLAP_LOG_WARNING("Fail to create field."); + if (_field_array[cid] == nullptr) { + LOG(WARNING) << "Fail to create field."; return OLAP_ERR_INIT_FAILED; } _fixed_len += field_buf_lens[cid] + 1; //1 for null byte @@ -99,7 +99,7 @@ OLAPStatus RowCursor::_init(const std::vector& schema, _fixed_buf = new (nothrow) char[_fixed_len]; if (_fixed_buf == nullptr) { - OLAP_LOG_WARNING("Fail to malloc _fixed_buf."); + LOG(WARNING) << "Fail to malloc _fixed_buf."; return OLAP_ERR_MALLOC_ERROR; } _owned_fixed_buf = _fixed_buf; @@ -162,13 +162,24 @@ OLAPStatus RowCursor::init(const TabletSchema& schema, return OLAP_SUCCESS; } +OLAPStatus RowCursor::init(const std::vector& schema, + const vector& columns) { + RETURN_NOT_OK(_init(schema, columns)); + return OLAP_SUCCESS; +} + OLAPStatus RowCursor::init_scan_key(const TabletSchema& schema, const std::vector& scan_keys) { + return init_scan_key(schema.columns(), scan_keys); +} + +OLAPStatus RowCursor::init_scan_key(const std::vector& schema, + const std::vector& scan_keys) { size_t scan_key_size = scan_keys.size(); - if (scan_key_size > schema.num_columns()) { + if (scan_key_size > schema.size()) { LOG(WARNING) << "Input param are invalid. Column count is bigger than num_columns of schema. " << "column_count=" << scan_key_size - << ", schema.num_columns=" << schema.num_columns(); + << ", schema.num_columns=" << schema.size(); return OLAP_ERR_INPUT_PARAMETER_ERROR; } @@ -177,13 +188,13 @@ OLAPStatus RowCursor::init_scan_key(const TabletSchema& schema, columns.push_back(i); } - RETURN_NOT_OK(_init(schema.columns(), columns)); + RETURN_NOT_OK(_init(schema, columns)); // NOTE: cid equal with column index // Hyperloglog cannot be key, no need to handle it _variable_len = 0; for (auto cid : _columns) { - const TabletColumn& column = schema.column(cid); + const TabletColumn& column = schema[cid]; FieldType type = column.type(); if (type == OLAP_FIELD_TYPE_VARCHAR) { _variable_len += scan_keys[cid].length(); @@ -195,7 +206,7 @@ OLAPStatus RowCursor::init_scan_key(const TabletSchema& schema, // variable_len for null bytes _variable_buf = new (nothrow) char[_variable_len]; - if (_variable_buf == NULL) { + if (_variable_buf == nullptr) { OLAP_LOG_WARNING("Fail to malloc _variable_buf."); return OLAP_ERR_MALLOC_ERROR; } @@ -203,7 +214,7 @@ OLAPStatus RowCursor::init_scan_key(const TabletSchema& schema, char* fixed_ptr = _fixed_buf; char* variable_ptr = _variable_buf; for (auto cid : _columns) { - const TabletColumn& column = schema.column(cid); + const TabletColumn& column = schema[cid]; fixed_ptr = _fixed_buf + _field_array[cid]->get_offset(); FieldType type = column.type(); if (type == OLAP_FIELD_TYPE_VARCHAR) { @@ -223,7 +234,13 @@ OLAPStatus RowCursor::init_scan_key(const TabletSchema& schema, } OLAPStatus RowCursor::allocate_memory_for_string_type( - const TabletSchema& schema, + const TabletSchema& schema, + MemPool* mem_pool) { + return allocate_memory_for_string_type(schema.columns(), mem_pool); +} + +OLAPStatus RowCursor::allocate_memory_for_string_type( + const std::vector& schema, MemPool* mem_pool) { // allocate memory for string type(char, varchar, hll) // The memory allocated in this function is used in aggregate and copy function @@ -247,7 +264,7 @@ OLAPStatus RowCursor::allocate_memory_for_string_type( char* fixed_ptr = _fixed_buf; char* variable_ptr = _variable_buf; for (auto cid : _columns) { - const TabletColumn& column = schema.column(cid); + const TabletColumn& column = schema[cid]; fixed_ptr = _fixed_buf + _field_array[cid]->get_offset(); FieldType type = column.type(); if (type == OLAP_FIELD_TYPE_VARCHAR) { @@ -304,7 +321,7 @@ int RowCursor::cmp(const RowCursor& other) const { size_t common_prefix_count = min(_key_column_num, other._key_column_num); // 只有key column才会参与比较 for (size_t i = 0; i < common_prefix_count; ++i) { - if (_field_array[i] == NULL || other._field_array[i] == NULL) { + if (_field_array[i] == nullptr || other._field_array[i] == nullptr) { continue; } @@ -325,7 +342,7 @@ int RowCursor::index_cmp(const RowCursor& other) const { size_t common_prefix_count = min(_columns.size(), other._key_column_num); // 只有key column才会参与比较 for (size_t i = 0; i < common_prefix_count; ++i) { - if (_field_array[i] == NULL || other._field_array[i] == NULL) { + if (_field_array[i] == nullptr || other._field_array[i] == nullptr) { continue; } char* left = _field_array[i]->get_field_ptr(_fixed_buf); @@ -343,7 +360,7 @@ bool RowCursor::equal(const RowCursor& other) const { // 按field顺序从后往前比较，有利于尽快发现不同，提升比较性能 size_t common_prefix_count = min(_key_column_num, other._key_column_num); for (int i = common_prefix_count - 1; i >= 0; --i) { - if (_field_array[i] == NULL || other._field_array[i] == NULL) { + if (_field_array[i] == nullptr || other._field_array[i] == nullptr) { continue; } char* left = _field_array[i]->get_field_ptr(_fixed_buf); @@ -357,7 +374,7 @@ bool RowCursor::equal(const RowCursor& other) const { void RowCursor::finalize_one_merge() { for (size_t i = _key_column_num; i < _field_array.size(); ++i) { - if (_field_array[i] == NULL) { + if (_field_array[i] == nullptr) { continue; } char* dest = _field_array[i]->get_ptr(_fixed_buf); @@ -368,7 +385,7 @@ void RowCursor::finalize_one_merge() { void RowCursor::aggregate(const RowCursor& other) { // 只有value column才会参与aggregate for (size_t i = _key_column_num; i < _field_array.size(); ++i) { - if (_field_array[i] == NULL || other._field_array[i] == NULL) { + if (_field_array[i] == nullptr || other._field_array[i] == nullptr) { continue; } @@ -429,7 +446,7 @@ OlapTuple RowCursor::to_tuple() const { OlapTuple tuple; for (auto cid : _columns) { - if (_field_array[cid] != NULL) { + if (_field_array[cid] != nullptr) { Field* field = _field_array[cid]; char* src = field->get_ptr(_fixed_buf); if (field->is_null(_fixed_buf)) { @@ -476,7 +493,7 @@ string RowCursor::to_string(string sep) const { } Field* field = _field_array[cid]; - if (field != NULL) { + if (field != nullptr) { char* src = field->get_ptr(_fixed_buf); result.append(field->to_string(src)); } else { @@ -489,7 +506,7 @@ string RowCursor::to_string(string sep) const { OLAPStatus RowCursor::get_first_different_column_id(const RowCursor& other, size_t* first_diff_id) const { - if (first_diff_id == NULL) { + if (first_diff_id == nullptr) { OLAP_LOG_WARNING("input parameter 'first_diff_id' is NULL."); return OLAP_ERR_INPUT_PARAMETER_ERROR; } @@ -501,7 +518,7 @@ OLAPStatus RowCursor::get_first_different_column_id(const RowCursor& other, size_t i = 0; for (; i < _field_array.size(); ++i) { - if (_field_array[i] == NULL || other._field_array[i] == NULL) { + if (_field_array[i] == nullptr || other._field_array[i] == nullptr) { continue; } diff --git a/be/src/olap/row_cursor.h b/be/src/olap/row_cursor.h index 3101addf2c23e6..cf58a513a96b21 100644 --- a/be/src/olap/row_cursor.h +++ b/be/src/olap/row_cursor.h @@ -73,18 +73,27 @@ class RowCursor { OLAPStatus init(const TabletSchema& schema, const std::vector& columns); + OLAPStatus init(const std::vector& schema, + const std::vector& columns); + // 用传入的key的size来初始化 // 目前仅用在拆分key区间的时候 OLAPStatus init_scan_key(const TabletSchema& schema, const std::vector& keys); + OLAPStatus init_scan_key(const std::vector& schema, + const std::vector& keys); + OLAPStatus init_scan_key(const TabletSchema& schema, const std::vector& field_lengths); //allocate memory for string type, which include char, varchar, hyperloglog OLAPStatus allocate_memory_for_string_type(const TabletSchema& schema, MemPool* mem_pool = nullptr); - + + OLAPStatus allocate_memory_for_string_type(const std::vector& schema, + MemPool* mem_pool = nullptr); + // 两个RowCurosr做比较，返回-1，0，1 int cmp(const RowCursor& other) const; diff --git a/be/src/olap/rowset/column_data.cpp b/be/src/olap/rowset/column_data.cpp index 08e7151e187aed..c71c1383a152b5 100644 --- a/be/src/olap/rowset/column_data.cpp +++ b/be/src/olap/rowset/column_data.cpp @@ -449,14 +449,14 @@ void ColumnData::set_read_params( auto res = _cursor.init(_segment_group->get_tablet_schema()); if (res != OLAP_SUCCESS) { - OLAP_LOG_WARNING("fail to init row_cursor"); + LOG(WARNING) << "fail to init row_cursor"; } _read_vector_batch.reset(new VectorizedRowBatch( - &(_segment_group->get_tablet_schema()), _return_columns, _num_rows_per_block)); + &(_segment_group->get_tablet_schema().columns()), _return_columns, _num_rows_per_block)); _seek_vector_batch.reset(new VectorizedRowBatch( - &(_segment_group->get_tablet_schema()), _seek_columns, _num_rows_per_block)); + &(_segment_group->get_tablet_schema().columns()), _seek_columns, _num_rows_per_block)); _read_block.reset(new RowBlock(&(_segment_group->get_tablet_schema()))); RowBlockInfo block_info; @@ -602,7 +602,7 @@ OLAPStatus ColumnData::schema_change_init() { } _read_vector_batch.reset(new VectorizedRowBatch( - &(_segment_group->get_tablet_schema()), _return_columns, _num_rows_per_block)); + &(_segment_group->get_tablet_schema().columns()), _return_columns, _num_rows_per_block)); _read_block.reset(new RowBlock(&(_segment_group->get_tablet_schema()))); diff --git a/be/src/olap/rowset/column_reader.cpp b/be/src/olap/rowset/column_reader.cpp index 5ecaef444477d2..cba9c2fa6f8709 100644 --- a/be/src/olap/rowset/column_reader.cpp +++ b/be/src/olap/rowset/column_reader.cpp @@ -485,18 +485,26 @@ ColumnReader::ColumnReader(uint32_t column_id, uint32_t column_unique_id) : _present_reader(NULL) { } -ColumnReader* ColumnReader::create(uint32_t column_id, + ColumnReader* ColumnReader::create(uint32_t column_id, const TabletSchema& schema, const UniqueIdToColumnIdMap& included, UniqueIdToColumnIdMap& segment_included, const UniqueIdEncodingMap& encodings) { - if (column_id >= schema.num_columns()) { + return create(column_id, schema.columns(), included, segment_included, encodings); +} + +ColumnReader* ColumnReader::create(uint32_t column_id, + const std::vector& schema, + const UniqueIdToColumnIdMap& included, + UniqueIdToColumnIdMap& segment_included, + const UniqueIdEncodingMap& encodings) { + if (column_id >= schema.size()) { LOG(WARNING) << "invalid column_id, column_id=" << column_id - << ", columns_size=" << schema.num_columns(); + << ", columns_size=" << schema.size(); return NULL; } - const TabletColumn& column = schema.column(column_id); + const TabletColumn& column = schema[column_id]; ColumnReader* reader = NULL; int32_t column_unique_id = column.unique_id(); diff --git a/be/src/olap/rowset/column_reader.h b/be/src/olap/rowset/column_reader.h index e5ad43a761f20c..8d928526e2233e 100644 --- a/be/src/olap/rowset/column_reader.h +++ b/be/src/olap/rowset/column_reader.h @@ -187,6 +187,12 @@ class ColumnReader { const UniqueIdToColumnIdMap& included, UniqueIdToColumnIdMap& segment_included, const UniqueIdEncodingMap& encodings); + + static ColumnReader* create(uint32_t column_id, + const std::vector& schema, + const UniqueIdToColumnIdMap& included, + UniqueIdToColumnIdMap& segment_included, + const UniqueIdEncodingMap& encodings); ColumnReader(uint32_t column_id, uint32_t column_unique_id); virtual ~ColumnReader(); diff --git a/be/src/olap/rowset/column_writer.cpp b/be/src/olap/rowset/column_writer.cpp index 110bc6f2d969ee..0f19901b6cc93a 100755 --- a/be/src/olap/rowset/column_writer.cpp +++ b/be/src/olap/rowset/column_writer.cpp @@ -23,12 +23,12 @@ namespace doris { ColumnWriter* ColumnWriter::create(uint32_t column_id, - const TabletSchema& schema, + const std::vector& schema, OutStreamFactory* stream_factory, size_t num_rows_per_row_block, double bf_fpp) { - ColumnWriter* column_writer = NULL; - const TabletColumn& column = schema.column(column_id); + ColumnWriter* column_writer = nullptr; + const TabletColumn& column = schema[column_id]; switch (column.type()) { case OLAP_FIELD_TYPE_TINYINT: diff --git a/be/src/olap/rowset/column_writer.h b/be/src/olap/rowset/column_writer.h index 6613770c4ce737..3725011fbbc55c 100644 --- a/be/src/olap/rowset/column_writer.h +++ b/be/src/olap/rowset/column_writer.h @@ -53,10 +53,11 @@ class ColumnWriter { // stream_factory: 用于创建输出流的工厂对象, 该对象的生命期由调用者所有 static ColumnWriter* create( uint32_t column_id, - const TabletSchema& schema, + const std::vector& schema, OutStreamFactory* stream_factory, size_t num_rows_per_row_block, double bf_fpp); + virtual ~ColumnWriter(); virtual OLAPStatus init(); diff --git a/be/src/olap/rowset/rowset_meta_manager.cpp b/be/src/olap/rowset/rowset_meta_manager.cpp index af63847dd4fc19..74a00594ddc60f 100644 --- a/be/src/olap/rowset/rowset_meta_manager.cpp +++ b/be/src/olap/rowset/rowset_meta_manager.cpp @@ -55,13 +55,12 @@ OLAPStatus RowsetMetaManager::get_rowset_meta(OlapMeta* meta, int64_t rowset_id, } OLAPStatus RowsetMetaManager::get_json_rowset_meta(OlapMeta* meta, int64_t rowset_id, std::string* json_rowset_meta) { - RowsetMeta rowset_meta; - RowsetMetaSharedPtr rowset_meta_ptr(&rowset_meta); + RowsetMetaSharedPtr rowset_meta_ptr(new(std::nothrow) RowsetMeta()); OLAPStatus status = get_rowset_meta(meta, rowset_id, rowset_meta_ptr); if (status != OLAP_SUCCESS) { return status; } - bool ret = rowset_meta.json_rowset_meta(json_rowset_meta); + bool ret = rowset_meta_ptr->json_rowset_meta(json_rowset_meta); if (!ret) { std::string error_msg = "get json rowset meta failed. rowset id:" + std::to_string(rowset_id); return OLAP_ERR_SERIALIZE_PROTOBUF_ERROR; @@ -69,7 +68,7 @@ OLAPStatus RowsetMetaManager::get_json_rowset_meta(OlapMeta* meta, int64_t rowse return OLAP_SUCCESS; } -OLAPStatus RowsetMetaManager::save(OlapMeta* meta, int64_t rowset_id, RowsetMetaSharedPtr rowset_meta) { +OLAPStatus RowsetMetaManager::save(OlapMeta* meta, int64_t rowset_id, RowsetMeta* rowset_meta) { std::string key = ROWSET_PREFIX + std::to_string(rowset_id); std::string value; bool ret = rowset_meta->serialize(&value); @@ -130,8 +129,7 @@ OLAPStatus RowsetMetaManager::load_json_rowset_meta(OlapMeta* meta, const std::s return OLAP_ERR_SERIALIZE_PROTOBUF_ERROR; } uint64_t rowset_id = rowset_meta.rowset_id(); - RowsetMetaSharedPtr rowset_meta_ptr(&rowset_meta); - OLAPStatus status = save(meta, rowset_id, rowset_meta_ptr); + OLAPStatus status = save(meta, rowset_id, &rowset_meta); return status; } diff --git a/be/src/olap/rowset/rowset_meta_manager.h b/be/src/olap/rowset/rowset_meta_manager.h index 14c207d89b0af1..67d52bfa7b69f5 100644 --- a/be/src/olap/rowset/rowset_meta_manager.h +++ b/be/src/olap/rowset/rowset_meta_manager.h @@ -35,7 +35,7 @@ class RowsetMetaManager { static OLAPStatus get_json_rowset_meta(OlapMeta* meta, int64_t rowset_id, std::string* json_rowset_meta); - static OLAPStatus save(OlapMeta* meta, int64_t rowset_id, RowsetMetaSharedPtr rowset_meta); + static OLAPStatus save(OlapMeta* meta, int64_t rowset_id, RowsetMeta* rowset_meta); static OLAPStatus save(OlapMeta* meta, int64_t rowset_id, const string& meta_binary); diff --git a/be/src/olap/rowset/segment_writer.cpp b/be/src/olap/rowset/segment_writer.cpp index 09a77ecd367c19..7ef23546444a22 100644 --- a/be/src/olap/rowset/segment_writer.cpp +++ b/be/src/olap/rowset/segment_writer.cpp @@ -63,7 +63,7 @@ OLAPStatus SegmentWriter::init(uint32_t write_mbytes_per_sec) { // 创建writer for (uint32_t i = 0; i < _segment_group->get_tablet_schema().num_columns(); i++) { - ColumnWriter* writer = ColumnWriter::create(i, _segment_group->get_tablet_schema(), + ColumnWriter* writer = ColumnWriter::create(i, _segment_group->get_tablet_schema().columns(), _stream_factory, _segment_group->get_num_rows_per_row_block(), _bloom_filter_fpp); diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index b7e61a41430bc3..9004a6755182f0 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -96,7 +96,6 @@ Tablet::Tablet(TabletMeta* tablet_meta, DataDir* data_dir) _tablet_meta(tablet_meta), _schema(tablet_meta->tablet_schema()), _data_dir(data_dir) { - _is_dropped = false; _tablet_meta->set_data_dir(_data_dir); _tablet_path.append(_data_dir->path()); @@ -455,6 +454,10 @@ OLAPStatus Tablet::capture_rs_readers(const vector& version_path, return OLAP_SUCCESS; } +OLAPStatus Tablet::add_delete_predicate(const DeletePredicatePB& delete_predicate, int64_t version) { + return _tablet_meta->add_delete_predicate(delete_predicate, version); +} + bool Tablet::version_for_delete_predicate(const Version& version) { return _tablet_meta->version_for_delete_predicate(version); } diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index c1d2899ece6e69..aee6a446a3bbeb 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -69,7 +69,6 @@ class Tablet : public std::enable_shared_from_this { // operation for TabletState TabletState tablet_state() const { return _state; } inline OLAPStatus set_tablet_state(TabletState state); - void mark_dropped() { _is_dropped = true; } // Property encapsulated in TabletMeta inline const TabletMeta& tablet_meta(); @@ -142,6 +141,7 @@ class Tablet : public std::enable_shared_from_this { vector* rs_readers) const; DelPredicateArray delete_predicates() { return _tablet_meta->delete_predicates(); } + OLAPStatus add_delete_predicate(const DeletePredicatePB& delete_predicate, int64_t version); bool version_for_delete_predicate(const Version& version); bool version_for_load_deletion(const Version& version); @@ -222,7 +222,6 @@ class Tablet : public std::enable_shared_from_this { std::string _tablet_path; RowsetGraph _rs_graph; - bool _is_dropped; DorisInitOnce _init_once; RWMutex _meta_lock; Mutex _ingest_lock; diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index 6d7eb493cc4a88..ff5c176d7adfb4 100755 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -138,7 +138,6 @@ OLAPStatus TabletManager::add_tablet(TTabletId tablet_id, SchemaHash schema_hash _tablet_map[tablet_id].table_arr.sort(_sort_tablet_by_creation_time); _tablet_map_lock.unlock(); } else { - tablet->mark_dropped(); res = OLAP_ERR_ENGINE_INSERT_EXISTS_TABLE; } LOG(WARNING) << "add duplicated tablet. force=" << force << ", res=" << res @@ -707,7 +706,6 @@ OLAPStatus TabletManager::load_tablet_from_meta(DataDir* data_dir, TTabletId tab LOG(WARNING) << "tablet not in schema change state without delta is invalid." << "tablet=" << tablet->full_name(); // tablet state is invalid, drop tablet - tablet->mark_dropped(); return OLAP_ERR_TABLE_INDEX_VALIDATE_ERROR; } @@ -1096,10 +1094,12 @@ OLAPStatus TabletManager::_drop_tablet_directly_unlocked( for (list::iterator it = _tablet_map[tablet_id].table_arr.begin(); it != _tablet_map[tablet_id].table_arr.end();) { if ((*it)->equal(tablet_id, schema_hash)) { + TabletSharedPtr tablet = *it; + it = _tablet_map[tablet_id].table_arr.erase(it); if (!keep_files) { - (*it)->mark_dropped(); + LOG(INFO) << "remove tablet:" << tablet_id << " path:" << tablet->tablet_path(); + boost::filesystem::remove_all(tablet->tablet_path()); } - it = _tablet_map[tablet_id].table_arr.erase(it); } else { ++it; } diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 4d1da4d02913e6..a322ebf2263b8c 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -236,6 +236,7 @@ OLAPStatus TabletMeta::deserialize(const string& meta_binary) { OLAPStatus TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) { // init _tablet_state + _tablet_meta_pb = tablet_meta_pb; switch (tablet_meta_pb.tablet_state()) { case PB_NOTREADY: _tablet_state = TabletState::TABLET_NOTREADY; @@ -264,9 +265,6 @@ OLAPStatus TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) { for (auto& it : tablet_meta_pb.rs_metas()) { RowsetMetaSharedPtr rs_meta(new AlphaRowsetMeta()); rs_meta->init_from_pb(it); - if (rs_meta->has_delete_predicate()) { - add_delete_predicate(rs_meta->delete_predicate(), rs_meta->version().first); - } _rs_metas.push_back(std::move(rs_meta)); } for (auto& it : tablet_meta_pb.inc_rs_metas()) { diff --git a/be/src/olap/tablet_meta_manager.cpp b/be/src/olap/tablet_meta_manager.cpp index 9ac3945c4b8e18..3e80a87b4691be 100755 --- a/be/src/olap/tablet_meta_manager.cpp +++ b/be/src/olap/tablet_meta_manager.cpp @@ -76,7 +76,7 @@ OLAPStatus TabletMetaManager::get_json_header(DataDir* store, } OLAPStatus TabletMetaManager::save(DataDir* store, - TTabletId tablet_id, TSchemaHash schema_hash, const TabletMeta* tablet_meta, string header_prefix) { + TTabletId tablet_id, TSchemaHash schema_hash, const TabletMeta* tablet_meta, const string& header_prefix) { std::stringstream key_stream; key_stream << header_prefix << tablet_id << "_" << schema_hash; std::string key = key_stream.str(); @@ -87,7 +87,7 @@ OLAPStatus TabletMetaManager::save(DataDir* store, } OLAPStatus TabletMetaManager::save(DataDir* store, - TTabletId tablet_id, TSchemaHash schema_hash, const std::string& meta_binary, string header_prefix) { + TTabletId tablet_id, TSchemaHash schema_hash, const std::string& meta_binary, const string& header_prefix) { std::stringstream key_stream; key_stream << header_prefix << tablet_id << "_" << schema_hash; std::string key = key_stream.str(); @@ -96,7 +96,8 @@ OLAPStatus TabletMetaManager::save(DataDir* store, return meta->put(META_COLUMN_FAMILY_INDEX, key, meta_binary); } -OLAPStatus TabletMetaManager::remove(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash, string header_prefix) { +OLAPStatus TabletMetaManager::remove(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash, + const string& header_prefix) { std::stringstream key_stream; key_stream << header_prefix << tablet_id << "_" << schema_hash; std::string key = key_stream.str(); @@ -108,7 +109,7 @@ OLAPStatus TabletMetaManager::remove(DataDir* store, TTabletId tablet_id, TSchem } OLAPStatus TabletMetaManager::traverse_headers(OlapMeta* meta, - std::function const& func, string header_prefix) { + std::function const& func, const string& header_prefix) { auto traverse_header_func = [&func](const std::string& key, const std::string& value) -> bool { std::vector parts; // key format: "hdr_" + tablet_id + "_" + schema_hash @@ -117,8 +118,8 @@ OLAPStatus TabletMetaManager::traverse_headers(OlapMeta* meta, LOG(WARNING) << "invalid tablet_meta key:" << key << ", splitted size:" << parts.size(); return true; } - TTabletId tablet_id = std::stol(parts[1].c_str(), NULL, 10); - TSchemaHash schema_hash = std::stol(parts[2].c_str(), NULL, 10); + TTabletId tablet_id = std::stol(parts[1].c_str(), nullptr, 10); + TSchemaHash schema_hash = std::stol(parts[2].c_str(), nullptr, 10); return func(tablet_id, schema_hash, value); }; OLAPStatus status = meta->iterate(META_COLUMN_FAMILY_INDEX, header_prefix, traverse_header_func); diff --git a/be/src/olap/tablet_meta_manager.h b/be/src/olap/tablet_meta_manager.h index 2533a1361d5a90..a7d037d0dda466 100644 --- a/be/src/olap/tablet_meta_manager.h +++ b/be/src/olap/tablet_meta_manager.h @@ -39,15 +39,15 @@ class TabletMetaManager { TSchemaHash schema_hash, std::string* json_header); static OLAPStatus save(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash, - const TabletMeta* tablet_meta, string header_prefix = "tabletmeta_"); + const TabletMeta* tablet_meta, const string& header_prefix = "tabletmeta_"); static OLAPStatus save(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash, - const std::string& meta_binary, string header_prefix = "tabletmeta_"); + const std::string& meta_binary, const string& header_prefix = "tabletmeta_"); static OLAPStatus remove(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash, - string header_prefix = "tabletmeta_"); + const string& header_prefix = "tabletmeta_"); static OLAPStatus traverse_headers(OlapMeta* meta, - std::function const& func, string header_prefix = "tabletmeta_"); + std::function const& func, const string& header_prefix = "tabletmeta_"); static OLAPStatus load_json_header(DataDir* store, const std::string& header_path); diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index 49294f75790de2..d2cf7021d4c1f7 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -46,7 +46,7 @@ OLAPStatus TabletColumn::init_from_pb(const ColumnPB& column) { _precision = column.precision(); } if (column.has_frac()) { - _precision = column.frac(); + _frac = column.frac(); } _length = column.length(); _index_length = column.length(); diff --git a/be/src/olap/txn_manager.cpp b/be/src/olap/txn_manager.cpp index d2348bc3ba0bca..03d71e82b6b53a 100755 --- a/be/src/olap/txn_manager.cpp +++ b/be/src/olap/txn_manager.cpp @@ -175,7 +175,7 @@ OLAPStatus TxnManager::commit_txn( // it is under a single txn lock if (!is_recovery) { OLAPStatus save_status = RowsetMetaManager::save(meta, rowset_ptr->rowset_id(), - rowset_ptr->rowset_meta()); + rowset_ptr->rowset_meta().get()); if (save_status != OLAP_SUCCESS) { LOG(WARNING) << "save committed rowset failed. when commit txn rowset_id:" << rowset_ptr->rowset_id() @@ -226,7 +226,7 @@ OLAPStatus TxnManager::publish_txn(OlapMeta* meta, TPartitionId partition_id, TT rowset_ptr->set_version_and_version_hash(version, version_hash); OLAPStatus save_status = RowsetMetaManager::save(meta, rowset_ptr->rowset_id(), - rowset_ptr->rowset_meta()); + rowset_ptr->rowset_meta().get()); if (save_status != OLAP_SUCCESS) { LOG(WARNING) << "save committed rowset failed. when publish txn rowset_id:" << rowset_ptr->rowset_id() @@ -378,7 +378,7 @@ void TxnManager::get_txn_related_tablets(const TTransactionId transaction_id, // each tablet for (auto& load_info : load_info_map) { const TabletInfo& tablet_info = load_info.first; - tablet_infos->emplace(tablet_info,load_info.second.rowset); + tablet_infos->emplace(tablet_info, load_info.second.rowset); } } diff --git a/be/src/olap/txn_manager.h b/be/src/olap/txn_manager.h index dec6305ce7f353..54975786b601ac 100755 --- a/be/src/olap/txn_manager.h +++ b/be/src/olap/txn_manager.h @@ -86,7 +86,7 @@ class TxnManager { // remove a txn from txn manager // not persist rowset meta because OLAPStatus publish_txn(OlapMeta* meta, TPartitionId partition_id, TTransactionId transaction_id, - TTabletId tablet_id, SchemaHash schema_hash, + TTabletId tablet_id, SchemaHash schema_hash, Version& version, VersionHash& version_hash); // delete the txn from manager if it is not committed(not have a valid rowset) diff --git a/be/src/runtime/vectorized_row_batch.cpp b/be/src/runtime/vectorized_row_batch.cpp index d082c083e0579a..07789da476bc51 100644 --- a/be/src/runtime/vectorized_row_batch.cpp +++ b/be/src/runtime/vectorized_row_batch.cpp @@ -23,7 +23,7 @@ namespace doris { VectorizedRowBatch::VectorizedRowBatch( - const TabletSchema* schema, + const std::vector* schema, const std::vector& cols, int capacity) : _schema(schema), _cols(cols), _capacity(capacity), _limit(capacity) { @@ -49,7 +49,7 @@ void VectorizedRowBatch::dump_to_row_block(RowBlock* row_block) { // pointer of this field in row block char* row_field_ptr = row_block->_mem_buf + row_block->_field_offset_in_memory[column_id]; - const TabletColumn& column = _schema->column(column_id); + const TabletColumn& column = _schema->at(column_id); size_t field_size = 0; if (column.type() == OLAP_FIELD_TYPE_CHAR || column.type() == OLAP_FIELD_TYPE_VARCHAR || @@ -93,7 +93,7 @@ void VectorizedRowBatch::dump_to_row_block(RowBlock* row_block) { char* row_field_ptr = row_block->_mem_buf + row_block->_field_offset_in_memory[column_id]; - const TabletColumn& column = _schema->column(column_id); + const TabletColumn& column = _schema->at(column_id); size_t field_size = 0; if (column.type() == OLAP_FIELD_TYPE_CHAR || column.type() == OLAP_FIELD_TYPE_VARCHAR || diff --git a/be/src/runtime/vectorized_row_batch.h b/be/src/runtime/vectorized_row_batch.h index 4960d35b0e0a94..a2c29100984cc6 100644 --- a/be/src/runtime/vectorized_row_batch.h +++ b/be/src/runtime/vectorized_row_batch.h @@ -73,7 +73,7 @@ class ColumnVector { class VectorizedRowBatch { public: VectorizedRowBatch( - const TabletSchema* schema, + const std::vector* schema, const std::vector& cols, int capacity); @@ -139,7 +139,7 @@ class VectorizedRowBatch { void dump_to_row_block(RowBlock* row_block); private: - const TabletSchema* _schema; + const std::vector* _schema; const std::vector& _cols; const uint16_t _capacity; uint16_t _size = 0; diff --git a/be/test/olap/bit_field_test.cpp b/be/test/olap/bit_field_test.cpp index 44859bfc1c6aed..b691845e6815f4 100755 --- a/be/test/olap/bit_field_test.cpp +++ b/be/test/olap/bit_field_test.cpp @@ -20,8 +20,8 @@ #include "olap/byte_buffer.h" #include "olap/out_stream.h" #include "olap/in_stream.h" -#include "olap/bit_field_reader.h" -#include "olap/bit_field_writer.h" +#include "olap/rowset/bit_field_reader.h" +#include "olap/rowset/bit_field_writer.h" #include "util/logging.h" namespace doris { diff --git a/be/test/olap/column_reader_test.cpp b/be/test/olap/column_reader_test.cpp index 6ff2f4c65ae9e9..78a9530f2a16d6 100644 --- a/be/test/olap/column_reader_test.cpp +++ b/be/test/olap/column_reader_test.cpp @@ -19,12 +19,13 @@ #include "olap/byte_buffer.h" #include "olap/stream_name.h" -#include "olap/column_reader.h" -#include "olap/column_writer.h" +#include "olap/rowset/column_reader.h" +#include "olap/rowset/column_writer.h" #include "olap/field.h" #include "olap/olap_define.h" #include "olap/olap_common.h" #include "olap/row_cursor.h" +#include "olap/row_block.h" #include "runtime/mem_pool.h" #include "runtime/string_value.hpp" #include "runtime/vectorized_row_batch.h" @@ -68,7 +69,7 @@ class TestColumn : public testing::Test { _stream_factory = new(std::nothrow) OutStreamFactory(COMPRESS_LZO, - OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE); + OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE); ASSERT_TRUE(_stream_factory != NULL); config::column_dictionary_key_ration_threshold = 30; config::column_dictionary_key_size_threshold = 1000; @@ -98,7 +99,7 @@ class TestColumn : public testing::Test { _length_buffers.clear(); } - void CreateColumnWriter(const std::vector &tablet_schema) { + void CreateColumnWriter(const std::vector& tablet_schema) { _column_writer = ColumnWriter::create( 0, tablet_schema, _stream_factory, 1024, BLOOM_FILTER_DEFAULT_FPP); @@ -106,7 +107,7 @@ class TestColumn : public testing::Test { ASSERT_EQ(_column_writer->init(), OLAP_SUCCESS); } - void CreateColumnReader(const std::vector &tablet_schema) { + void CreateColumnReader(const std::vector &tablet_schema) { UniqueIdEncodingMap encodings; encodings[0] = ColumnEncodingMessage(); encodings[0].set_kind(ColumnEncodingMessage::DIRECT); @@ -115,7 +116,7 @@ class TestColumn : public testing::Test { } void CreateColumnReader( - const std::vector &tablet_schema, + const std::vector &tablet_schema, UniqueIdEncodingMap &encodings) { UniqueIdToColumnIdMap included; included[0] = 0; @@ -123,19 +124,19 @@ class TestColumn : public testing::Test { segment_included[0] = 0; _column_reader = ColumnReader::create(0, - tablet_schema, - included, - segment_included, - encodings); + tablet_schema, + included, + segment_included, + encodings); ASSERT_TRUE(_column_reader != NULL); system("rm ./tmp_file"); ASSERT_EQ(OLAP_SUCCESS, - helper.open_with_mode("tmp_file", - O_CREAT | O_EXCL | O_WRONLY, - S_IRUSR | S_IWUSR)); + helper.open_with_mode("tmp_file", + O_CREAT | O_EXCL | O_WRONLY, + S_IRUSR | S_IWUSR)); std::vector off; std::vector length; std::vector buffer_size; @@ -186,42 +187,45 @@ class TestColumn : public testing::Test { for (int i = 0; i < off.size(); ++i) { ReadOnlyFileStream* in_stream = new (std::nothrow) ReadOnlyFileStream( - &helper, - &_shared_buffer, - off[i], - length[i], - lzo_decompress, - buffer_size[i], - &_stats); + &helper, + &_shared_buffer, + off[i], + length[i], + lzo_decompress, + buffer_size[i], + &_stats); ASSERT_EQ(OLAP_SUCCESS, in_stream->init()); _map_in_streams[name[i]] = in_stream; } ASSERT_EQ(_column_reader->init( - &_map_in_streams, - 1024, - _mem_pool.get(), - &_stats), OLAP_SUCCESS); + &_map_in_streams, + 1024, + _mem_pool.get(), + &_stats), OLAP_SUCCESS); } - void SetFieldInfo(FieldInfo &field_info, - std::string name, - FieldType type, - FieldAggregationMethod aggregation, - uint32_t length, - bool is_allow_null, - bool is_key) { - field_info.name = name; - field_info.type = type; - field_info.aggregation = aggregation; - field_info.length = length; - field_info.is_allow_null = is_allow_null; - field_info.is_key = is_key; - field_info.precision = 1000; - field_info.frac = 10000; - field_info.unique_id = 0; - field_info.is_bf_column = false; + void SetTabletSchemaWithOneColumn(std::string name, + std::string type, + std::string aggregation, + uint32_t length, + bool is_allow_null, + bool is_key, std::vector* tablet_schema) { + ColumnPB column; + column.set_unique_id(0); + column.set_name(name); + column.set_type(type); + column.set_is_key(is_key); + column.set_is_nullable(is_allow_null); + column.set_length(length); + column.set_aggregation(aggregation); + column.set_precision(1000); + column.set_frac(1000); + column.set_is_bf_column(false); + TabletColumn tablet_column; + tablet_column.init_from_pb(column); + tablet_schema->push_back(tablet_column); } void create_and_save_last_position() { @@ -260,17 +264,14 @@ class TestColumn : public testing::Test { TEST_F(TestColumn, VectorizedTinyColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("TinyColumn"), - OLAP_FIELD_TYPE_TINYINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 1, - false, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "TinyColumn", + "TINYINT", + "REPLACE", + 1, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -316,16 +317,15 @@ TEST_F(TestColumn, VectorizedTinyColumnWithoutPresent) { TEST_F(TestColumn, SeekTinyColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("TinyColumn"), - OLAP_FIELD_TYPE_TINYINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 1, - false, - true); - tablet_schema.push_back(field_info); + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "TinyColumn", + "TINYINT", + "REPLACE", + 1, + false, + true, + &tablet_schema); CreateColumnWriter(tablet_schema); @@ -403,16 +403,15 @@ TEST_F(TestColumn, SeekTinyColumnWithoutPresent) { TEST_F(TestColumn, SkipTinyColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("TinyColumn"), - OLAP_FIELD_TYPE_TINYINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 1, - false, - true); - tablet_schema.push_back(field_info); + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "TinyColumn", + "TINYINT", + "REPLACE", + 1, + false, + true, + &tablet_schema); CreateColumnWriter(tablet_schema); @@ -459,17 +458,15 @@ TEST_F(TestColumn, SkipTinyColumnWithoutPresent) { TEST_F(TestColumn, VectorizedTinyColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("TinyColumn"), - OLAP_FIELD_TYPE_TINYINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 1, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "TinyColumn", + "TINYINT", + "REPLACE", + 1, + true, + true, + &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -513,17 +510,15 @@ TEST_F(TestColumn, VectorizedTinyColumnWithPresent) { TEST_F(TestColumn, TinyColumnIndex) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("TinyColumn"), - OLAP_FIELD_TYPE_TINYINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 1, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "TinyColumn", + "TINYINT", + "REPLACE", + 1, + true, + true, + &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -567,17 +562,15 @@ TEST_F(TestColumn, TinyColumnIndex) { TEST_F(TestColumn, SeekTinyColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("TinyColumn"), - OLAP_FIELD_TYPE_TINYINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 1, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "TinyColumn", + "TINYINT", + "REPLACE", + 1, + true, + true, + &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -653,17 +646,14 @@ TEST_F(TestColumn, SeekTinyColumnWithPresent) { TEST_F(TestColumn, SkipTinyColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("TinyColumn"), - OLAP_FIELD_TYPE_TINYINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 1, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "TinyColumn", + "TINYINT", + "REPLACE", + 1, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -709,17 +699,14 @@ TEST_F(TestColumn, SkipTinyColumnWithPresent) { TEST_F(TestColumn, VectorizedShortColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("ShortColumn"), - OLAP_FIELD_TYPE_SMALLINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 2, - false, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "ShortColumn", + "SMALLINT", + "REPLACE", + 2, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -763,17 +750,14 @@ TEST_F(TestColumn, VectorizedShortColumnWithoutPresent) { TEST_F(TestColumn, SeekShortColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("ShortColumn"), - OLAP_FIELD_TYPE_SMALLINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 2, - false, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "ShortColumn", + "SMALLINT", + "REPLACE", + 2, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -848,17 +832,14 @@ TEST_F(TestColumn, SeekShortColumnWithoutPresent) { TEST_F(TestColumn, SkipShortColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("ShortColumn"), - OLAP_FIELD_TYPE_SMALLINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 2, - false, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "ShortColumn", + "SMALLINT", + "REPLACE", + 2, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -904,17 +885,14 @@ TEST_F(TestColumn, SkipShortColumnWithoutPresent) { TEST_F(TestColumn, SeekShortColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("ShortColumn"), - OLAP_FIELD_TYPE_SMALLINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 2, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "ShortColumn", + "SMALLINT", + "REPLACE", + 2, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -986,17 +964,15 @@ TEST_F(TestColumn, SeekShortColumnWithPresent) { TEST_F(TestColumn, VectorizedShortColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("ShortColumn"), - OLAP_FIELD_TYPE_SMALLINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 2, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "ShortColumn", + "SMALLINT", + "REPLACE", + 2, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -1042,17 +1018,14 @@ TEST_F(TestColumn, VectorizedShortColumnWithPresent) { TEST_F(TestColumn, SkipShortColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("ShortColumn"), - OLAP_FIELD_TYPE_SMALLINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 2, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "ShortColumn", + "SMALLINT", + "REPLACE", + 2, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -1097,17 +1070,14 @@ TEST_F(TestColumn, SkipShortColumnWithPresent) { TEST_F(TestColumn, VectorizedIntColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("IntColumn"), - OLAP_FIELD_TYPE_INT, - OLAP_FIELD_AGGREGATION_REPLACE, - 4, - false, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "IntColumn", + "INT", + "REPLACE", + 4, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -1151,17 +1121,14 @@ TEST_F(TestColumn, VectorizedIntColumnWithoutPresent) { TEST_F(TestColumn, VectorizedIntColumnMassWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("IntColumn"), - OLAP_FIELD_TYPE_INT, - OLAP_FIELD_AGGREGATION_REPLACE, - 4, - false, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "IntColumn", + "INT", + "REPLACE", + 4, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -1207,17 +1174,14 @@ TEST_F(TestColumn, VectorizedIntColumnMassWithoutPresent) { TEST_F(TestColumn, VectorizedIntColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("IntColumn"), - OLAP_FIELD_TYPE_INT, - OLAP_FIELD_AGGREGATION_REPLACE, - 4, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "IntColumn", + "INT", + "REPLACE", + 4, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -1264,17 +1228,14 @@ TEST_F(TestColumn, VectorizedIntColumnWithPresent) { TEST_F(TestColumn, VectorizedLongColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("LongColumnWithoutPresent"), - OLAP_FIELD_TYPE_BIGINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 8, - false, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "LongColumnWithoutPresent", + "BIGINT", + "REPLACE", + 8, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -1319,17 +1280,14 @@ TEST_F(TestColumn, VectorizedLongColumnWithoutPresent) { TEST_F(TestColumn, VectorizedLongColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("LongColumnWithPresent"), - OLAP_FIELD_TYPE_BIGINT, - OLAP_FIELD_AGGREGATION_REPLACE, - 8, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "LongColumnWithPresent", + "BIGINT", + "REPLACE", + 8, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -1376,17 +1334,14 @@ TEST_F(TestColumn, VectorizedLongColumnWithPresent) { TEST_F(TestColumn, VectorizedFloatColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("FloatColumnWithoutPresent"), - OLAP_FIELD_TYPE_FLOAT, - OLAP_FIELD_AGGREGATION_REPLACE, - 4, - false, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "FloatColumnWithoutPresent", + "FLOAT", + "REPLACE", + 4, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -1432,17 +1387,15 @@ TEST_F(TestColumn, VectorizedFloatColumnWithoutPresent) { TEST_F(TestColumn, VectorizedFloatColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("FloatColumnWithPresent"), - OLAP_FIELD_TYPE_FLOAT, - OLAP_FIELD_AGGREGATION_REPLACE, - 4, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "FloatColumnWithPresent", + "FLOAT", + "REPLACE", + 4, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -1489,17 +1442,15 @@ TEST_F(TestColumn, VectorizedFloatColumnWithPresent) { TEST_F(TestColumn, SeekFloatColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("FloatColumnWithPresent"), - OLAP_FIELD_TYPE_FLOAT, - OLAP_FIELD_AGGREGATION_REPLACE, - 4, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "FloatColumnWithPresent", + "FLOAT", + "REPLACE", + 4, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -1562,17 +1513,15 @@ TEST_F(TestColumn, SeekFloatColumnWithPresent) { TEST_F(TestColumn, SkipFloatColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("FloatColumnWithPresent"), - OLAP_FIELD_TYPE_FLOAT, - OLAP_FIELD_AGGREGATION_REPLACE, - 4, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "FloatColumnWithPresent", + "FLOAT", + "REPLACE", + 4, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -1615,17 +1564,15 @@ TEST_F(TestColumn, SkipFloatColumnWithPresent) { TEST_F(TestColumn, VectorizedDoubleColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DoubleColumnWithoutPresent"), - OLAP_FIELD_TYPE_DOUBLE, - OLAP_FIELD_AGGREGATION_REPLACE, - 8, - false, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "DoubleColumnWithoutPresent", + "DOUBLE", + "REPLACE", + 8, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -1671,17 +1618,15 @@ TEST_F(TestColumn, VectorizedDoubleColumnWithoutPresent) { TEST_F(TestColumn, VectorizedDoubleColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DoubleColumnWithPresent"), - OLAP_FIELD_TYPE_DOUBLE, - OLAP_FIELD_AGGREGATION_REPLACE, - 8, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "DoubleColumnWithPresent", + "DOUBLE", + "REPLACE", + 8, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -1729,17 +1674,15 @@ TEST_F(TestColumn, VectorizedDoubleColumnWithPresent) { TEST_F(TestColumn, VectorizedDatetimeColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DatetimeColumnWithoutPresent"), - OLAP_FIELD_TYPE_DATETIME, - OLAP_FIELD_AGGREGATION_REPLACE, - 8, - false, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "DatetimeColumnWithoutPresent", + "DATETIME", + "REPLACE", + 8, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -1778,17 +1721,15 @@ TEST_F(TestColumn, VectorizedDatetimeColumnWithoutPresent) { TEST_F(TestColumn, VectorizedDatetimeColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DatetimeColumnWithoutPresent"), - OLAP_FIELD_TYPE_DATETIME, - OLAP_FIELD_AGGREGATION_REPLACE, - 8, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "DatetimeColumnWithoutPresent", + "DATETIME", + "REPLACE", + 8, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -1842,17 +1783,15 @@ TEST_F(TestColumn, VectorizedDatetimeColumnWithPresent) { TEST_F(TestColumn, VectorizedDateColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DateColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_DATE, - OLAP_FIELD_AGGREGATION_REPLACE, - 3, - false, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "DateColumnWithoutoutPresent", + "DATE", + "REPLACE", + 3, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -1890,17 +1829,15 @@ TEST_F(TestColumn, VectorizedDateColumnWithoutPresent) { TEST_F(TestColumn, VectorizedDateColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DateColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_DATE, - OLAP_FIELD_AGGREGATION_REPLACE, - 3, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "DateColumnWithoutoutPresent", + "DATE", + "REPLACE", + 3, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -1954,17 +1891,15 @@ TEST_F(TestColumn, VectorizedDateColumnWithPresent) { TEST_F(TestColumn, VectorizedDecimalColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DecimalColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_DECIMAL, - OLAP_FIELD_AGGREGATION_REPLACE, - 12, - false, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "DecimalColumnWithoutoutPresent", + "DECIMAL", + "REPLACE", + 12, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -2015,17 +1950,15 @@ TEST_F(TestColumn, VectorizedDecimalColumnWithoutPresent) { TEST_F(TestColumn, VectorizedDecimalColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DecimalColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_DECIMAL, - OLAP_FIELD_AGGREGATION_REPLACE, - 12, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "DecimalColumnWithoutoutPresent", + "DECIMAL", + "REPLACE", + 12, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -2076,17 +2009,15 @@ TEST_F(TestColumn, VectorizedDecimalColumnWithPresent) { TEST_F(TestColumn, SkipDecimalColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DecimalColumnWithPresent"), - OLAP_FIELD_TYPE_DECIMAL, - OLAP_FIELD_AGGREGATION_REPLACE, - 12, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "DecimalColumnWithPresent", + "DECIMAL", + "REPLACE", + 12, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -2133,17 +2064,14 @@ TEST_F(TestColumn, SkipDecimalColumnWithPresent) { TEST_F(TestColumn, SeekDecimalColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DecimalColumnWithPresent"), - OLAP_FIELD_TYPE_DECIMAL, - OLAP_FIELD_AGGREGATION_REPLACE, - 12, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "DecimalColumnWithPresent", + "DECIMAL", + "REPLACE", + 12, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -2217,17 +2145,15 @@ TEST_F(TestColumn, SeekDecimalColumnWithPresent) { TEST_F(TestColumn, VectorizedLargeIntColumnWithoutPresent) { // init tablet schema - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("LargeIntColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_LARGEINT, - OLAP_FIELD_AGGREGATION_SUM, - 16, - false, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "LargeIntColumnWithoutoutPresent", + "LARGEINT", + "SUM", + 16, + false, + true, &tablet_schema); // test data string value1 = "100000000000000000000000000000000000000"; string value2 = "-170141183460469231731687303715884105728"; @@ -2281,16 +2207,16 @@ TEST_F(TestColumn, VectorizedLargeIntColumnWithoutPresent) { TEST_F(TestColumn, VectorizedLargeIntColumnWithPresent) { // init tablet schema - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("LargeIntColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_LARGEINT, - OLAP_FIELD_AGGREGATION_SUM, - 16, - true, - true); - tablet_schema.push_back(field_info); + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "LargeIntColumnWithoutoutPresent", + "LARGEINT", + "SUM", + 16, + true, + true, &tablet_schema); + // test data string value1 = "100000000000000000000000000000000000000"; @@ -2360,17 +2286,15 @@ TEST_F(TestColumn, VectorizedLargeIntColumnWithPresent) { TEST_F(TestColumn, SkipLargeIntColumnWithPresent) { // init tablet schema - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("LargeIntColumnWithPresent"), - OLAP_FIELD_TYPE_LARGEINT, - OLAP_FIELD_AGGREGATION_SUM, - 16, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "LargeIntColumnWithPresent", + "LARGEINT", + "SUM", + 16, + true, + true, &tablet_schema); // test data string value1 = "100000000000000000000000000000000000000"; string value2 = "-170141183460469231731687303715884105728"; @@ -2421,16 +2345,15 @@ TEST_F(TestColumn, SkipLargeIntColumnWithPresent) { TEST_F(TestColumn, VectorizedDirectVarcharColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DirectVarcharColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, - 10, - false, - true); - tablet_schema.push_back(field_info); + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "DirectVarcharColumnWithoutoutPresent", + "VARCHAR", + "REPLACE", + 10, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); @@ -2495,17 +2418,14 @@ TEST_F(TestColumn, VectorizedDirectVarcharColumnWithoutPresent) { TEST_F(TestColumn, VectorizedDirectVarcharColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DirectVarcharColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, - 10, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "DirectVarcharColumnWithoutoutPresent", + "VARCHAR", + "REPLACE", + 10, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -2555,17 +2475,15 @@ TEST_F(TestColumn, VectorizedDirectVarcharColumnWithPresent) { TEST_F(TestColumn, SkipDirectVarcharColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DirectVarcharColumnWithPresent"), - OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, - 10, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "DirectVarcharColumnWithPresent", + "VARCHAR", + "REPLACE", + 10, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -2615,17 +2533,15 @@ TEST_F(TestColumn, SkipDirectVarcharColumnWithPresent) { TEST_F(TestColumn, SeekDirectVarcharColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DirectVarcharColumnWithPresent"), - OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, - 10, - false, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "DirectVarcharColumnWithPresent", + "VARCHAR", + "REPLACE", + 10, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -2697,17 +2613,15 @@ TEST_F(TestColumn, SeekDirectVarcharColumnWithoutPresent) { TEST_F(TestColumn, SeekDirectVarcharColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DirectVarcharColumnWithPresent"), - OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, - 10, - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + + SetTabletSchemaWithOneColumn( + "DirectVarcharColumnWithPresent", + "VARCHAR", + "REPLACE", + 10, + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -2779,17 +2693,14 @@ TEST_F(TestColumn, SeekDirectVarcharColumnWithPresent) { TEST_F(TestColumn, VectorizedStringColumnWithoutPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("VarcharColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_CHAR, - OLAP_FIELD_AGGREGATION_REPLACE, - strlen("abcde"), - false, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "VarcharColumnWithoutoutPresent", + "CHAR", + "REPLACE", + strlen("abcde"), + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -2853,17 +2764,14 @@ TEST_F(TestColumn, VectorizedStringColumnWithoutPresent) { TEST_F(TestColumn, VectorizedStringColumnWithPresent) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("VarcharColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_CHAR, - OLAP_FIELD_AGGREGATION_REPLACE, - strlen("abcde"), - true, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "VarcharColumnWithoutoutPresent", + "CHAR", + "REPLACE", + strlen("abcde"), + true, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -2911,17 +2819,14 @@ TEST_F(TestColumn, VectorizedStringColumnWithPresent) { TEST_F(TestColumn, VectorizedStringColumnWithoutoutPresent2) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("VarcharColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_CHAR, - OLAP_FIELD_AGGREGATION_REPLACE, - 20, - false, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "VarcharColumnWithoutoutPresent", + "CHAR", + "REPLACE", + 20, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -3005,17 +2910,14 @@ TEST_F(TestColumn, VectorizedStringColumnWithoutoutPresent2) { TEST_F(TestColumn, VectorizedDirectVarcharColumnWith65533) { // write data - std::vector tablet_schema; - FieldInfo field_info; - SetFieldInfo(field_info, - std::string("DirectVarcharColumnWithoutoutPresent"), - OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, - 65535, - false, - true); - tablet_schema.push_back(field_info); - + std::vector tablet_schema; + SetTabletSchemaWithOneColumn( + "DirectVarcharColumnWithoutoutPresent", + "VARCHAR", + "REPLACE", + 65535, + false, + true, &tablet_schema); CreateColumnWriter(tablet_schema); RowCursor write_row; @@ -3084,4 +2986,3 @@ int main(int argc, char** argv) { ret = RUN_ALL_TESTS(); return ret; } - diff --git a/be/test/olap/comparison_predicate_test.cpp b/be/test/olap/comparison_predicate_test.cpp index 3494b64ae8b2ad..c9e9db3d259e82 100644 --- a/be/test/olap/comparison_predicate_test.cpp +++ b/be/test/olap/comparison_predicate_test.cpp @@ -99,21 +99,24 @@ public: \ delete _vectorized_batch; \ } \ } \ - void SetFieldInfo(FieldInfo &field_info, std::string name, \ - FieldType type, FieldAggregationMethod aggregation, \ - uint32_t length, bool is_allow_null, bool is_key) { \ - field_info.name = name; \ - field_info.type = type; \ - field_info.aggregation = aggregation; \ - field_info.length = length; \ - field_info.is_allow_null = is_allow_null; \ - field_info.is_key = is_key; \ - field_info.precision = 1000; \ - field_info.frac = 10000; \ - field_info.unique_id = 0; \ - field_info.is_bf_column = false; \ + void SetTabletColumn(std::string name, \ + std::string type, std::string aggregation, \ + uint32_t length, bool is_allow_null, bool is_key, TabletColumn* tablet_column) { \ + static int id = 0; \ + ColumnPB column; \ + column.set_unique_id(++id); \ + column.set_name(name); \ + column.set_type(type); \ + column.set_is_key(is_key); \ + column.set_is_nullable(is_allow_null); \ + column.set_length(length); \ + column.set_aggregation(aggregation); \ + column.set_precision(1000); \ + column.set_frac(1000); \ + column.set_is_bf_column(false); \ + tablet_column->init_from_pb(column); \ } \ - void InitVectorizedBatch(const std::vector& schema, \ + void InitVectorizedBatch(const std::vector* schema, \ const std::vector&ids, \ int size) { \ _vectorized_batch = new VectorizedRowBatch(schema, ids, size); \ @@ -129,17 +132,17 @@ TEST_PREDICATE_DEFINITION(TestLessPredicate) #define TEST_EQUAL_PREDICATE(TYPE, TYPE_NAME, FIELD_TYPE) \ TEST_F(TestEqualPredicate, TYPE_NAME##_COLUMN) { \ - std::vector schema; \ - FieldInfo field_info; \ - SetFieldInfo(field_info, std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, \ - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); \ - schema.push_back(field_info); \ + std::vector schema; \ + TabletColumn tablet_column; \ + SetTabletColumn(std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, \ + "REPLACE", 1, false, true, &tablet_column); \ + schema.push_back(tablet_column); \ int size = 10; \ std::vector return_columns; \ for (int i = 0; i < schema.size(); ++i) { \ return_columns.push_back(i); \ } \ - InitVectorizedBatch(schema, return_columns, size); \ + InitVectorizedBatch(&schema, return_columns, size); \ ColumnVector* col_vector = _vectorized_batch->column(0); \ \ /* for no nulls */ \ @@ -176,24 +179,24 @@ TEST_F(TestEqualPredicate, TYPE_NAME##_COLUMN) { \ ASSERT_EQ(*(col_data + sel[0]), 5); \ } \ -TEST_EQUAL_PREDICATE(int8_t, TINYINT, OLAP_FIELD_TYPE_TINYINT) -TEST_EQUAL_PREDICATE(int16_t, SMALLINT, OLAP_FIELD_TYPE_SMALLINT) -TEST_EQUAL_PREDICATE(int32_t, INT, OLAP_FIELD_TYPE_INT) -TEST_EQUAL_PREDICATE(int64_t, BIGINT, OLAP_FIELD_TYPE_BIGINT) -TEST_EQUAL_PREDICATE(int128_t, LARGEINT, OLAP_FIELD_TYPE_LARGEINT) +TEST_EQUAL_PREDICATE(int8_t, TINYINT, "TINYINT") +TEST_EQUAL_PREDICATE(int16_t, SMALLINT, "SMALLINT") +TEST_EQUAL_PREDICATE(int32_t, INT, "INT") +TEST_EQUAL_PREDICATE(int64_t, BIGINT, "BIGINT") +TEST_EQUAL_PREDICATE(int128_t, LARGEINT, "LARGEINT") TEST_F(TestEqualPredicate, FLOAT_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("FLOAT_COLUMN"), OLAP_FIELD_TYPE_FLOAT, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_column; + SetTabletColumn(std::string("FLOAT_COLUMN"), "FLOAT", + "REPLACE", 1, false, true, &tablet_column); + schema.push_back(tablet_column); int size = 10; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -231,17 +234,17 @@ TEST_F(TestEqualPredicate, FLOAT_COLUMN) { } TEST_F(TestEqualPredicate, DOUBLE_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DOUBLE_COLUMN"), OLAP_FIELD_TYPE_DOUBLE, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_column; + SetTabletColumn(std::string("DOUBLE_COLUMN"), "DOUBLE", + "REPLACE", 1, false, true, &tablet_column); + schema.push_back(tablet_column); int size = 10; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -279,17 +282,17 @@ TEST_F(TestEqualPredicate, DOUBLE_COLUMN) { } TEST_F(TestEqualPredicate, DECIMAL_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DECIMAL_COLUMN"), OLAP_FIELD_TYPE_DECIMAL, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_column; + SetTabletColumn(std::string("DECIMAL_COLUMN"), "DECIMAL", + "REPLACE", 1, false, true, &tablet_column); + schema.push_back(tablet_column); int size = 10; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -330,17 +333,17 @@ TEST_F(TestEqualPredicate, DECIMAL_COLUMN) { } TEST_F(TestEqualPredicate, STRING_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("STRING_COLUMN"), OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_column; + SetTabletColumn(std::string("STRING_COLUMN"), "VARCHAR", + "REPLACE", 1, false, true, &tablet_column); + schema.push_back(tablet_column); int size = 10; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -398,17 +401,17 @@ TEST_F(TestEqualPredicate, STRING_COLUMN) { } TEST_F(TestEqualPredicate, DATE_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DATE_COLUMN"), OLAP_FIELD_TYPE_DATE, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_column; + SetTabletColumn(std::string("DATE_COLUMN"), "DATA", + "REPLACE", 1, false, true, &tablet_column); + schema.push_back(tablet_column); int size = 6; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -461,17 +464,17 @@ TEST_F(TestEqualPredicate, DATE_COLUMN) { } TEST_F(TestEqualPredicate, DATETIME_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DATETIME_COLUMN"), OLAP_FIELD_TYPE_DATETIME, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_column; + SetTabletColumn(std::string("DATETIME_COLUMN"), "DATETIME", + "REPLACE", 1, false, true, &tablet_column); + schema.push_back(tablet_column); int size = 6; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -525,17 +528,17 @@ TEST_F(TestEqualPredicate, DATETIME_COLUMN) { #define TEST_LESS_PREDICATE(TYPE, TYPE_NAME, FIELD_TYPE) \ TEST_F(TestLessPredicate, TYPE_NAME##_COLUMN) { \ - std::vector schema; \ - FieldInfo field_info; \ - SetFieldInfo(field_info, std::string("TYPE_NAME_COLUMN"), FIELD_TYPE, \ - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); \ - schema.push_back(field_info); \ + std::vector schema; \ + TabletColumn tablet_column; \ + SetTabletColumn(std::string("TYPE_NAME_COLUMN"), FIELD_TYPE, \ + "REPLACE", 1, false, true, &tablet_column); \ + schema.push_back(tablet_column); \ int size = 10; \ std::vector return_columns; \ for (int i = 0; i < schema.size(); ++i) { \ return_columns.push_back(i); \ } \ - InitVectorizedBatch(schema, return_columns, size); \ + InitVectorizedBatch(&schema, return_columns, size); \ ColumnVector* col_vector = _vectorized_batch->column(0); \ \ /* for no nulls */ \ @@ -580,24 +583,24 @@ TEST_F(TestLessPredicate, TYPE_NAME##_COLUMN) { \ ASSERT_EQ(sum, 4); \ } \ -TEST_LESS_PREDICATE(int8_t, TINYINT, OLAP_FIELD_TYPE_TINYINT) -TEST_LESS_PREDICATE(int16_t, SMALLINT, OLAP_FIELD_TYPE_SMALLINT) -TEST_LESS_PREDICATE(int32_t, INT, OLAP_FIELD_TYPE_INT) -TEST_LESS_PREDICATE(int64_t, BIGINT, OLAP_FIELD_TYPE_BIGINT) -TEST_LESS_PREDICATE(int128_t, LARGEINT, OLAP_FIELD_TYPE_LARGEINT) +TEST_LESS_PREDICATE(int8_t, TINYINT, "TINYINT") +TEST_LESS_PREDICATE(int16_t, SMALLINT, "SMALLINT") +TEST_LESS_PREDICATE(int32_t, INT, "INT") +TEST_LESS_PREDICATE(int64_t, BIGINT, "BIGINT") +TEST_LESS_PREDICATE(int128_t, LARGEINT, "LARGEINT") TEST_F(TestLessPredicate, FLOAT_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("FLOAT_COLUMN"), OLAP_FIELD_TYPE_FLOAT, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_column; + SetTabletColumn(std::string("FLOAT_COLUMN"), "FLOAT", + "REPLACE", 1, false, true, &tablet_column); + schema.push_back(tablet_column); int size = 10; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -644,17 +647,17 @@ TEST_F(TestLessPredicate, FLOAT_COLUMN) { } TEST_F(TestLessPredicate, DOUBLE_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DOUBLE_COLUMN"), OLAP_FIELD_TYPE_DOUBLE, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_column; + SetTabletColumn(std::string("DOUBLE_COLUMN"), "DOUBLE", + "REPLACE", 1, false, true, &tablet_column); + schema.push_back(tablet_column); int size = 10; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -701,17 +704,17 @@ TEST_F(TestLessPredicate, DOUBLE_COLUMN) { } TEST_F(TestLessPredicate, DECIMAL_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DECIMAL_COLUMN"), OLAP_FIELD_TYPE_DECIMAL, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_column; + SetTabletColumn(std::string("DECIMAL_COLUMN"), "DECIMAL", + "REPLACE", 1, false, true, &tablet_column); + schema.push_back(tablet_column); int size = 10; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -762,17 +765,17 @@ TEST_F(TestLessPredicate, DECIMAL_COLUMN) { } TEST_F(TestLessPredicate, STRING_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("STRING_COLUMN"), OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_column; + SetTabletColumn(std::string("STRING_COLUMN"), "VARCHAR", + "REPLACE", 1, false, true, &tablet_column); + schema.push_back(tablet_column); int size = 10; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -828,17 +831,17 @@ TEST_F(TestLessPredicate, STRING_COLUMN) { } TEST_F(TestLessPredicate, DATE_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DATE_COLUMN"), OLAP_FIELD_TYPE_DATE, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_column; + SetTabletColumn(std::string("DATE_COLUMN"), "DATE", + "REPLACE", 1, false, true, &tablet_column); + schema.push_back(tablet_column); int size = 6; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -888,17 +891,17 @@ TEST_F(TestLessPredicate, DATE_COLUMN) { } TEST_F(TestLessPredicate, DATETIME_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DATETIME_COLUMN"), OLAP_FIELD_TYPE_DATETIME, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_column; + SetTabletColumn(std::string("DATETIME_COLUMN"), "DATETIME", + "REPLACE", 1, false, true, &tablet_column); + schema.push_back(tablet_column); int size = 6; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls diff --git a/be/test/olap/delete_handler_test.cpp b/be/test/olap/delete_handler_test.cpp index 818ae03dc91155..ef840bd32ccef5 100644 --- a/be/test/olap/delete_handler_test.cpp +++ b/be/test/olap/delete_handler_test.cpp @@ -151,8 +151,6 @@ void set_default_create_tablet_request(TCreateTabletReq* request) { void set_default_push_request(TPushReq* request) { request->tablet_id = 10003; request->schema_hash = 270068375; - request->__set_version(2); - request->__set_version_hash(1); request->timeout = 86400; request->push_type = TPushType::LOAD; } @@ -173,26 +171,16 @@ class TestDeleteConditionHandler : public testing::Test { set_default_create_tablet_request(&_create_tablet); res = k_engine->create_tablet(_create_tablet); ASSERT_EQ(OLAP_SUCCESS, res); - tablet = k_engine->get_tablet( + tablet = k_engine->tablet_manager()->get_tablet( _create_tablet.tablet_id, _create_tablet.tablet_schema.schema_hash); ASSERT_TRUE(tablet.get() != NULL); _tablet_path = tablet->tablet_path(); } - OLAPStatus push_empty_delta(int32_t version) { - // push data - TPushReq push_req; - set_default_push_request(&push_req); - push_req.version = version; - push_req.version_hash = version; - std::vector tablets_info; - return k_engine->push(push_req, &tablets_info); - } - void TearDown() { // Remove all dir. tablet.reset(); - StorageEngine::get_instance()->drop_tablet( + StorageEngine::instance()->tablet_manager()->drop_tablet( _create_tablet.tablet_id, _create_tablet.tablet_schema.schema_hash); while (0 == access(_tablet_path.c_str(), F_OK)) { sleep(1); @@ -229,82 +217,23 @@ TEST_F(TestDeleteConditionHandler, StoreCondSucceed) { condition.condition_values.push_back("5"); conditions.push_back(condition); - success_res = _delete_condition_handler.store_cond(tablet, 3, conditions); + DeletePredicatePB del_pred; + success_res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); ASSERT_EQ(OLAP_SUCCESS, success_res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(3)); // 验证存储在header中的过滤条件正确 - const DelPredicateArray& delete_conditions = tablet->delete_data_conditions(); - ASSERT_EQ(size_t(1), delete_conditions.size()); - EXPECT_EQ(3, delete_conditions.Get(0).version()); - ASSERT_EQ(size_t(3), delete_conditions.Get(0).sub_conditions_size()); - EXPECT_STREQ("k1=1", delete_conditions.Get(0).sub_conditions(0).c_str()); - EXPECT_STREQ("k2>>3", delete_conditions.Get(0).sub_conditions(1).c_str()); - EXPECT_STREQ("k2<=5", delete_conditions.Get(0).sub_conditions(2).c_str()); - - // 再次存储相同版本号(版本号为3)的过滤条件 - conditions.clear(); - condition.column_name = "k1"; - condition.condition_op = "!="; - condition.condition_values.clear(); - condition.condition_values.push_back("1"); - conditions.push_back(condition); - - success_res = _delete_condition_handler.store_cond(tablet, 3, conditions); - ASSERT_EQ(OLAP_SUCCESS, success_res); - - // 验证存储相同版本号的过滤条件情况下，新的过滤条件替换掉旧的过滤条件 - const DelPredicateArray& new_delete_conditions = tablet->delete_data_conditions(); - ASSERT_EQ(size_t(1), new_delete_conditions.size()); - EXPECT_EQ(3, new_delete_conditions.Get(0).version()); - ASSERT_EQ(size_t(1), new_delete_conditions.Get(0).sub_conditions_size()); - EXPECT_STREQ("k1!=1", new_delete_conditions.Get(0).sub_conditions(0).c_str()); - - // 第三次存储不同版本号(版本号为4)的过滤条件 - conditions.clear(); - condition.column_name = "k1"; - condition.condition_op = "!="; - condition.condition_values.clear(); - condition.condition_values.push_back("1"); - conditions.push_back(condition); - - condition.column_name = "k1"; - condition.condition_op = "!="; - condition.condition_values.clear(); - condition.condition_values.push_back("2"); - conditions.push_back(condition); - - success_res = _delete_condition_handler.store_cond(tablet, 4, conditions); - ASSERT_EQ(OLAP_SUCCESS, success_res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(4)); - - const DelPredicateArray& all_delete_conditions = tablet->delete_data_conditions(); - ASSERT_EQ(size_t(2), all_delete_conditions.size()); - EXPECT_EQ(3, all_delete_conditions.Get(0).version()); - ASSERT_EQ(size_t(1), all_delete_conditions.Get(0).sub_conditions_size()); - EXPECT_STREQ("k1!=1", all_delete_conditions.Get(0).sub_conditions(0).c_str()); - EXPECT_EQ(4, all_delete_conditions.Get(1).version()); - ASSERT_EQ(size_t(2), all_delete_conditions.Get(1).sub_conditions_size()); - EXPECT_STREQ("k1!=1", all_delete_conditions.Get(1).sub_conditions(0).c_str()); - EXPECT_STREQ("k1!=2", all_delete_conditions.Get(1).sub_conditions(1).c_str()); + ASSERT_EQ(size_t(3), del_pred.sub_predicates_size()); + EXPECT_STREQ("k1=1", del_pred.sub_predicates(0).c_str()); + EXPECT_STREQ("k2>>3", del_pred.sub_predicates(1).c_str()); + EXPECT_STREQ("k2<=5", del_pred.sub_predicates(2).c_str()); } -// 检测参数不正确的情况，包括：空的过滤条件字符串，以及负的版本号 +// 检测参数不正确的情况，包括：空的过滤条件字符串 TEST_F(TestDeleteConditionHandler, StoreCondInvalidParameters) { // 空的过滤条件 std::vector conditions; - OLAPStatus failed_res = _delete_condition_handler.store_cond(tablet, 3, conditions); - ASSERT_EQ(OLAP_ERR_DELETE_INVALID_PARAMETERS, failed_res); - - // 负的版本号: -10 - TCondition condition; - condition.column_name = "k1"; - condition.condition_op = "="; - condition.condition_values.clear(); - condition.condition_values.push_back("2"); - conditions.push_back(condition); - - failed_res = _delete_condition_handler.store_cond(tablet, -10, conditions); + DeletePredicatePB del_pred; + OLAPStatus failed_res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred);; ASSERT_EQ(OLAP_ERR_DELETE_INVALID_PARAMETERS, failed_res); } @@ -318,8 +247,8 @@ TEST_F(TestDeleteConditionHandler, StoreCondNonexistentColumn) { condition.condition_values.clear(); condition.condition_values.push_back("2"); conditions.push_back(condition); - - OLAPStatus failed_res = _delete_condition_handler.store_cond(tablet, 3, conditions); + DeletePredicatePB del_pred; + OLAPStatus failed_res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred);; ASSERT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, failed_res); // 'v'是value列 @@ -330,138 +259,10 @@ TEST_F(TestDeleteConditionHandler, StoreCondNonexistentColumn) { condition.condition_values.push_back("5"); conditions.push_back(condition); - failed_res = _delete_condition_handler.store_cond(tablet, 3, conditions); + failed_res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred);; ASSERT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, failed_res); } -// 只删除特定版本的过滤条件 -TEST_F(TestDeleteConditionHandler, DeleteCondRemoveOneCondition) { - OLAPStatus res; - std::vector conditions; - TCondition condition; - condition.column_name = "k1"; - condition.condition_op = "="; - condition.condition_values.clear(); - condition.condition_values.push_back("1"); - conditions.push_back(condition); - - condition.column_name = "k2"; - condition.condition_op = ">"; - condition.condition_values.clear(); - condition.condition_values.push_back("3"); - conditions.push_back(condition); - - condition.column_name = "k2"; - condition.condition_op = "<="; - condition.condition_values.clear(); - condition.condition_values.push_back("5"); - conditions.push_back(condition); - - res = _delete_condition_handler.store_cond(tablet, 3, conditions); - ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(3)); - - conditions.clear(); - condition.column_name = "k1"; - condition.condition_op = "!="; - condition.condition_values.clear(); - condition.condition_values.push_back("1"); - conditions.push_back(condition); - - res = _delete_condition_handler.store_cond(tablet, 4, conditions); - ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(4)); - - conditions.clear(); - condition.column_name = "k2"; - condition.condition_op = ">="; - condition.condition_values.clear(); - condition.condition_values.push_back("1"); - conditions.push_back(condition); - - res = _delete_condition_handler.store_cond(tablet, 5, conditions); - ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(5)); - - // 删除版本号为8的过滤条件 - res = _delete_condition_handler.delete_cond(tablet, 5, false); - ASSERT_EQ(OLAP_SUCCESS, res); - - const DelPredicateArray& all_delete_conditions = tablet->delete_data_conditions(); - ASSERT_EQ(size_t(2), all_delete_conditions.size()); - - EXPECT_EQ(3, all_delete_conditions.Get(0).version()); - ASSERT_EQ(size_t(3), all_delete_conditions.Get(0).sub_conditions_size()); - EXPECT_STREQ("k1=1", all_delete_conditions.Get(0).sub_conditions(0).c_str()); - EXPECT_STREQ("k2>>3", all_delete_conditions.Get(0).sub_conditions(1).c_str()); - EXPECT_STREQ("k2<=5", all_delete_conditions.Get(0).sub_conditions(2).c_str()); - - EXPECT_EQ(4, all_delete_conditions.Get(1).version()); - ASSERT_EQ(size_t(1), all_delete_conditions.Get(1).sub_conditions_size()); - EXPECT_STREQ("k1!=1", all_delete_conditions.Get(1).sub_conditions(0).c_str()); -} - -// 删除特定版本以及版本比它小的过滤条件 -TEST_F(TestDeleteConditionHandler, DeleteCondRemovBelowCondition) { - OLAPStatus res; - std::vector conditions; - TCondition condition; - condition.column_name = "k1"; - condition.condition_op = "="; - condition.condition_values.clear(); - condition.condition_values.push_back("1"); - conditions.push_back(condition); - - condition.column_name = "k2"; - condition.condition_op = ">"; - condition.condition_values.clear(); - condition.condition_values.push_back("3"); - conditions.push_back(condition); - - condition.column_name = "k2"; - condition.condition_op = "<="; - condition.condition_values.clear(); - condition.condition_values.push_back("5"); - conditions.push_back(condition); - - res = _delete_condition_handler.store_cond(tablet, 3, conditions); - ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(3)); - - conditions.clear(); - condition.column_name = "k1"; - condition.condition_op = "!="; - condition.condition_values.clear(); - condition.condition_values.push_back("1"); - conditions.push_back(condition); - - res = _delete_condition_handler.store_cond(tablet, 4, conditions); - ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(4)); - - conditions.clear(); - condition.column_name = "k2"; - condition.condition_op = ">="; - condition.condition_values.clear(); - condition.condition_values.push_back("1"); - conditions.push_back(condition); - - res = _delete_condition_handler.store_cond(tablet, 5, conditions); - ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(5)); - - // 删除版本号为7以及版本号小于7的过滤条件 - res = _delete_condition_handler.delete_cond(tablet, 4, true); - ASSERT_EQ(OLAP_SUCCESS, res); - - const DelPredicateArray& all_delete_conditions = tablet->delete_data_conditions(); - ASSERT_EQ(size_t(1), all_delete_conditions.size()); - - EXPECT_EQ(5, all_delete_conditions.Get(0).version()); - ASSERT_EQ(size_t(1), all_delete_conditions.Get(0).sub_conditions_size()); - EXPECT_STREQ("k2>=1", all_delete_conditions.Get(0).sub_conditions(0).c_str()); -} - // 测试删除条件值不符合类型要求 class TestDeleteConditionHandler2 : public testing::Test { protected: @@ -479,7 +280,7 @@ class TestDeleteConditionHandler2 : public testing::Test { set_default_create_tablet_request(&_create_tablet); res = k_engine->create_tablet(_create_tablet); ASSERT_EQ(OLAP_SUCCESS, res); - tablet = k_engine->get_tablet( + tablet = k_engine->tablet_manager()->get_tablet( _create_tablet.tablet_id, _create_tablet.tablet_schema.schema_hash); ASSERT_TRUE(tablet.get() != NULL); _tablet_path = tablet->tablet_path(); @@ -488,7 +289,7 @@ class TestDeleteConditionHandler2 : public testing::Test { void TearDown() { // Remove all dir. tablet.reset(); - StorageEngine::get_instance()->drop_tablet( + StorageEngine::instance()->tablet_manager()->drop_tablet( _create_tablet.tablet_id, _create_tablet.tablet_schema.schema_hash); while (0 == access(_tablet_path.c_str(), F_OK)) { sleep(1); @@ -499,6 +300,7 @@ class TestDeleteConditionHandler2 : public testing::Test { std::string _tablet_path; TabletSharedPtr tablet; TCreateTabletReq _create_tablet; + DeleteConditionHandler _delete_condition_handler; }; TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { @@ -532,8 +334,9 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { condition.condition_values.push_back("-1"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 2, conditions); - EXPECT_EQ(OLAP_SUCCESS, res); + DeletePredicatePB del_pred; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); + ASSERT_EQ(OLAP_SUCCESS, res); // k5类型为int128 conditions.clear(); @@ -543,8 +346,9 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { condition.condition_values.push_back("1"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 2, conditions); - EXPECT_EQ(OLAP_SUCCESS, res); + DeletePredicatePB del_pred_2; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_2); + ASSERT_EQ(OLAP_SUCCESS, res); // k9类型为decimal, precision=6, frac=3 conditions.clear(); @@ -554,23 +358,27 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { condition.condition_values.push_back("2.3"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 2, conditions); - EXPECT_EQ(OLAP_SUCCESS, res); + DeletePredicatePB del_pred_3; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_3); + ASSERT_EQ(OLAP_SUCCESS, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2"); - res = cond_handler.store_cond(tablet, 2, conditions); - EXPECT_EQ(OLAP_SUCCESS, res); + DeletePredicatePB del_pred_4; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_4); + ASSERT_EQ(OLAP_SUCCESS, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-2"); - res = cond_handler.store_cond(tablet, 2, conditions); - EXPECT_EQ(OLAP_SUCCESS, res); + DeletePredicatePB del_pred_5; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_5); + ASSERT_EQ(OLAP_SUCCESS, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-2.3"); - res = cond_handler.store_cond(tablet, 2, conditions); - EXPECT_EQ(OLAP_SUCCESS, res); + DeletePredicatePB del_pred_6; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_6); + ASSERT_EQ(OLAP_SUCCESS, res); // k10,k11类型分别为date, datetime conditions.clear(); @@ -586,8 +394,9 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { condition.condition_values.push_back("2014-01-01 00:00:00"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 2, conditions); - EXPECT_EQ(OLAP_SUCCESS, res); + DeletePredicatePB del_pred_7; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_7); + ASSERT_EQ(OLAP_SUCCESS, res); // k12,k13类型分别为string(64), varchar(64) conditions.clear(); @@ -603,8 +412,9 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { condition.condition_values.push_back("YWFhYQ=="); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 2, conditions); - EXPECT_EQ(OLAP_SUCCESS, res); + DeletePredicatePB del_pred_8; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_8); + ASSERT_EQ(OLAP_SUCCESS, res); } TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { @@ -620,133 +430,155 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { condition.condition_values.push_back("1000"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_1; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_1); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k1的值越下界，k1类型为int8 conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-1000"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_2; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_2); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k2的值越上界，k2类型为int16 conditions[0].condition_values.clear(); conditions[0].column_name = "k2"; conditions[0].condition_values.push_back("32768"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_3; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_3); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k2的值越下界，k2类型为int16 conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-32769"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_4; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_4); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k3的值越上界，k3类型为int32 conditions[0].condition_values.clear(); conditions[0].column_name = "k3"; conditions[0].condition_values.push_back("2147483648"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_5; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_5); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k3的值越下界，k3类型为int32 conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-2147483649"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_6; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_6); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k4的值越上界，k2类型为int64 conditions[0].condition_values.clear(); conditions[0].column_name = "k4"; conditions[0].condition_values.push_back("9223372036854775808"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_7; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_7); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k4的值越下界，k1类型为int64 conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-9223372036854775809"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_8; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_8); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k5的值越上界，k5类型为int128 conditions[0].condition_values.clear(); conditions[0].column_name = "k5"; conditions[0].condition_values.push_back("170141183460469231731687303715884105728"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_9; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_9); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k5的值越下界，k5类型为int128 conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-170141183460469231731687303715884105729"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_10; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_10); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k9整数部分长度过长，k9类型为decimal, precision=6, frac=3 conditions[0].condition_values.clear(); conditions[0].column_name = "k9"; - conditions[0].condition_values.push_back("1234.5"); - res = cond_handler.store_cond(tablet, 2, conditions); + conditions[0].condition_values.push_back("12347876.5"); + DeletePredicatePB del_pred_11; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_11); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k9小数部分长度过长，k9类型为decimal, precision=6, frac=3 conditions[0].condition_values.clear(); - conditions[0].condition_values.push_back("1.2345"); - res = cond_handler.store_cond(tablet, 2, conditions); + conditions[0].condition_values.push_back("1.2345678"); + DeletePredicatePB del_pred_12; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_12); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k9没有小数部分，但包含小数点 conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("1."); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_13; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_13); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k10类型的过滤值不符合对应格式，k10为date conditions[0].condition_values.clear(); conditions[0].column_name = "k10"; conditions[0].condition_values.push_back("20130101"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_14; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_14); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-64-01"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_15; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_15); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-01-40"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_16; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_16); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k11类型的过滤值不符合对应格式，k11为datetime conditions[0].condition_values.clear(); conditions[0].column_name = "k11"; conditions[0].condition_values.push_back("20130101 00:00:00"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_17; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_17); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-64-01 00:00:00"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_18; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_18); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-01-40 00:00:00"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_19; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_19); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-01-01 24:00:00"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_20; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_20); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-01-01 00:60:00"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_21; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_21); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-01-01 00:00:60"); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_22; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_22); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); // 测试k12和k13类型的过滤值过长，k12,k13类型分别为string(64), varchar(64) @@ -755,7 +587,8 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].condition_values.push_back("YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYW" "FhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYW" "FhYWFhYWFhYWFhYWFhYWFhYWFhYWE=;k13=YWFhYQ=="); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_23; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_23); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); conditions[0].condition_values.clear(); @@ -763,7 +596,8 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].condition_values.push_back("YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYW" "FhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYW" "FhYWFhYWFhYWFhYWFhYWFhYWFhYWE=;k13=YWFhYQ=="); - res = cond_handler.store_cond(tablet, 2, conditions); + DeletePredicatePB del_pred_24; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_24); EXPECT_EQ(OLAP_ERR_DELETE_INVALID_CONDITION, res); } @@ -783,30 +617,20 @@ class TestDeleteHandler : public testing::Test { set_default_create_tablet_request(&_create_tablet); res = k_engine->create_tablet(_create_tablet); ASSERT_EQ(OLAP_SUCCESS, res); - tablet = k_engine->get_tablet( + tablet = k_engine->tablet_manager()->get_tablet( _create_tablet.tablet_id, _create_tablet.tablet_schema.schema_hash); - ASSERT_TRUE(tablet.get() != NULL); + ASSERT_TRUE(tablet != nullptr); _tablet_path = tablet->tablet_path(); _data_row_cursor.init(tablet->tablet_schema()); _data_row_cursor.allocate_memory_for_string_type(tablet->tablet_schema()); } - OLAPStatus push_empty_delta(int32_t version) { - // push data - TPushReq push_req; - set_default_push_request(&push_req); - push_req.version = version; - push_req.version_hash = version; - std::vector tablets_info; - return k_engine->push(push_req, &tablets_info); - } - void TearDown() { // Remove all dir. tablet.reset(); _delete_handler.finalize(); - StorageEngine::get_instance()->drop_tablet( + StorageEngine::instance()->tablet_manager()->drop_tablet( _create_tablet.tablet_id, _create_tablet.tablet_schema.schema_hash); while (0 == access(_tablet_path.c_str(), F_OK)) { sleep(1); @@ -819,6 +643,7 @@ class TestDeleteHandler : public testing::Test { TabletSharedPtr tablet; TCreateTabletReq _create_tablet; DeleteHandler _delete_handler; + DeleteConditionHandler _delete_condition_handler; }; TEST_F(TestDeleteHandler, InitSuccess) { @@ -826,10 +651,6 @@ TEST_F(TestDeleteHandler, InitSuccess) { std::vector conditions; DeleteConditionHandler delete_condition_handler; - // Header中还没有删除条件 - res = _delete_handler.init(tablet, 2); - ASSERT_EQ(OLAP_SUCCESS, res); - // 往头文件中添加过滤条件 TCondition condition; condition.column_name = "k1"; @@ -850,9 +671,11 @@ TEST_F(TestDeleteHandler, InitSuccess) { condition.condition_values.push_back("5"); conditions.push_back(condition); - res = delete_condition_handler.store_cond(tablet, 3, conditions); + DeletePredicatePB del_pred; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); + ASSERT_EQ(OLAP_SUCCESS, res); + res = tablet->add_delete_predicate(del_pred, 1); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(3)); conditions.clear(); condition.column_name = "k1"; @@ -861,9 +684,11 @@ TEST_F(TestDeleteHandler, InitSuccess) { condition.condition_values.push_back("3"); conditions.push_back(condition); - res = delete_condition_handler.store_cond(tablet, 4, conditions); + DeletePredicatePB del_pred_2; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_2); + ASSERT_EQ(OLAP_SUCCESS, res); + res = tablet->add_delete_predicate(del_pred_2, 2); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(4)); conditions.clear(); condition.column_name = "k2"; @@ -872,9 +697,11 @@ TEST_F(TestDeleteHandler, InitSuccess) { condition.condition_values.push_back("1"); conditions.push_back(condition); - res = delete_condition_handler.store_cond(tablet, 5, conditions); + DeletePredicatePB del_pred_3; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_3); + ASSERT_EQ(OLAP_SUCCESS, res); + res = tablet->add_delete_predicate(del_pred_3, 3); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(5)); conditions.clear(); condition.column_name = "k2"; @@ -883,19 +710,23 @@ TEST_F(TestDeleteHandler, InitSuccess) { condition.condition_values.push_back("3"); conditions.push_back(condition); - res = delete_condition_handler.store_cond(tablet, 6, conditions); + DeletePredicatePB del_pred_4; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_4); + ASSERT_EQ(OLAP_SUCCESS, res); + res = tablet->add_delete_predicate(del_pred_4, 4); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(6)); // 从header文件中取出版本号小于等于7的过滤条件 - _delete_handler.finalize(); - res = _delete_handler.init(tablet, 4); + res = _delete_handler.init(tablet->tablet_schema(), tablet->delete_predicates(), 4); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(2, _delete_handler.conditions_num()); + ASSERT_EQ(4, _delete_handler.conditions_num()); vector conds_version = _delete_handler.get_conds_version(); + EXPECT_EQ(4, conds_version.size()); sort(conds_version.begin(), conds_version.end()); - EXPECT_EQ(3, conds_version[0]); - EXPECT_EQ(4, conds_version[1]); + EXPECT_EQ(1, conds_version[0]); + EXPECT_EQ(2, conds_version[1]); + EXPECT_EQ(3, conds_version[2]); + EXPECT_EQ(4, conds_version[3]); _delete_handler.finalize(); } @@ -922,12 +753,15 @@ TEST_F(TestDeleteHandler, FilterDataSubconditions) { condition.condition_values.push_back("4"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 3, conditions); + DeletePredicatePB del_pred; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(3)); + res = tablet->add_delete_predicate(del_pred, 1); // 指定版本号为10以载入Header中的所有过滤条件(在这个case中，只有过滤条件1) - _delete_handler.init(tablet, 10); + res = _delete_handler.init(tablet->tablet_schema(), tablet->delete_predicates(), 4); + ASSERT_EQ(OLAP_SUCCESS, res); + ASSERT_EQ(1, _delete_handler.conditions_num()); // 构造一行测试数据 vector data_str; @@ -980,9 +814,10 @@ TEST_F(TestDeleteHandler, FilterDataConditions) { condition.condition_values.push_back("4"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 3, conditions); + DeletePredicatePB del_pred; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(3)); + res = tablet->add_delete_predicate(del_pred, 1); // 过滤条件2 conditions.clear(); @@ -992,9 +827,10 @@ TEST_F(TestDeleteHandler, FilterDataConditions) { condition.condition_values.push_back("3"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 4, conditions); + DeletePredicatePB del_pred_2; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_2); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(4)); + res = tablet->add_delete_predicate(del_pred_2, 2); // 过滤条件3 conditions.clear(); @@ -1004,12 +840,15 @@ TEST_F(TestDeleteHandler, FilterDataConditions) { condition.condition_values.push_back("5"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 5, conditions); + DeletePredicatePB del_pred_3; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_3); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(5)); + res = tablet->add_delete_predicate(del_pred_3, 3); - // 指定版本号为10以载入Header中的所有三条过滤条件 - _delete_handler.init(tablet, 10); + // 指定版本号为4以载入meta中的所有过滤条件(在这个case中，只有过滤条件1) + res = _delete_handler.init(tablet->tablet_schema(), tablet->delete_predicates(), 4); + ASSERT_EQ(OLAP_SUCCESS, res); + ASSERT_EQ(3, _delete_handler.conditions_num()); vector data_str; data_str.push_back("4"); @@ -1027,7 +866,7 @@ TEST_F(TestDeleteHandler, FilterDataConditions) { res = _data_row_cursor.from_tuple(tuple); ASSERT_EQ(OLAP_SUCCESS, res); // 这行数据会因为过滤条件3而被过滤 - ASSERT_TRUE(_delete_handler.is_filter_data(1, _data_row_cursor)); + ASSERT_TRUE(_delete_handler.is_filter_data(3, _data_row_cursor)); _delete_handler.finalize(); } @@ -1053,9 +892,10 @@ TEST_F(TestDeleteHandler, FilterDataVersion) { condition.condition_values.push_back("4"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 3, conditions); + DeletePredicatePB del_pred; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(3)); + res = tablet->add_delete_predicate(del_pred, 3); // 过滤条件2 conditions.clear(); @@ -1065,12 +905,15 @@ TEST_F(TestDeleteHandler, FilterDataVersion) { condition.condition_values.push_back("3"); conditions.push_back(condition); - res = cond_handler.store_cond(tablet, 4, conditions); + DeletePredicatePB del_pred_2; + res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_2); ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(OLAP_SUCCESS, push_empty_delta(4)); + res = tablet->add_delete_predicate(del_pred_2, 4); - // 指定版本号为10以载入Header中的所有过滤条件(过滤条件1，过滤条件2) - _delete_handler.init(tablet, 10); + // 指定版本号为4以载入meta中的所有过滤条件(过滤条件1，过滤条件2) + res = _delete_handler.init(tablet->tablet_schema(), tablet->delete_predicates(), 4); + ASSERT_EQ(OLAP_SUCCESS, res); + ASSERT_EQ(2, _delete_handler.conditions_num()); // 构造一行测试数据 vector data_str; @@ -1088,9 +931,9 @@ TEST_F(TestDeleteHandler, FilterDataVersion) { OlapTuple tuple(data_str); res = _data_row_cursor.from_tuple(tuple); ASSERT_EQ(OLAP_SUCCESS, res); - // 如果数据版本小于6，则过滤条件1生效，这条数据被过滤 - ASSERT_TRUE(_delete_handler.is_filter_data(1, _data_row_cursor)); - // 如果数据版本大于6，则过滤条件1会被跳过 + // 如果数据版本小于3，则过滤条件1生效，这条数据被过滤 + ASSERT_TRUE(_delete_handler.is_filter_data(2, _data_row_cursor)); + // 如果数据版本大于3，则过滤条件1会被跳过 ASSERT_FALSE(_delete_handler.is_filter_data(4, _data_row_cursor)); _delete_handler.finalize(); @@ -1099,11 +942,6 @@ TEST_F(TestDeleteHandler, FilterDataVersion) { } // namespace doris int main(int argc, char** argv) { - std::string conffile = std::string(getenv("DORIS_HOME")) + "/conf/be.conf"; - if (!doris::config::init(conffile.c_str(), false)) { - fprintf(stderr, "error read config file. \n"); - return -1; - } doris::init_glog("be-test"); int ret = doris::OLAP_SUCCESS; testing::InitGoogleTest(&argc, argv); diff --git a/be/test/olap/delta_writer_test.cpp b/be/test/olap/delta_writer_test.cpp index 4e954570c54c5b..9759382374707b 100644 --- a/be/test/olap/delta_writer_test.cpp +++ b/be/test/olap/delta_writer_test.cpp @@ -59,15 +59,17 @@ void set_up() { } void tear_down() { + delete k_engine; + k_engine = nullptr; system("rm -rf ./data_test"); remove_all_dir(std::string(getenv("DORIS_HOME")) + UNUSED_PREFIX); } -void create_tablet_request(TCreateTabletReq* request) { - request->tablet_id = 10003; +void create_tablet_request(int64_t tablet_id, int32_t schema_hash, TCreateTabletReq* request) { + request->tablet_id = tablet_id; request->__set_version(1); request->__set_version_hash(0); - request->tablet_schema.schema_hash = 270068375; + request->tablet_schema.schema_hash = schema_hash; request->tablet_schema.short_key_column_count = 6; request->tablet_schema.keys_type = TKeysType::AGG_KEYS; request->tablet_schema.storage_type = TStorageType::COLUMN; @@ -268,22 +270,20 @@ class TestDeltaWriter : public ::testing::Test { void SetUp() { // Create local data dir for StorageEngine. - char buffer[MAX_PATH_LEN]; - getcwd(buffer, MAX_PATH_LEN); - config::storage_root_path = std::string(buffer) + "/data_push"; - remove_all_dir(config::storage_root_path); - ASSERT_EQ(create_dir(config::storage_root_path), OLAP_SUCCESS); + std::cout << "setup" << std::endl; } void TearDown(){ // Remove all dir. - ASSERT_EQ(OLAP_SUCCESS, remove_all_dir(config::storage_root_path)); + std::cout << "tear down" << std::endl; + //doris::tear_down(); + //ASSERT_EQ(OLAP_SUCCESS, remove_all_dir(config::storage_root_path)); } }; - +/* TEST_F(TestDeltaWriter, open) { TCreateTabletReq request; - create_tablet_request(&request); + create_tablet_request(10003, 270068375, &request); OLAPStatus res = k_engine->create_tablet(request); ASSERT_EQ(OLAP_SUCCESS, res); @@ -308,13 +308,13 @@ TEST_F(TestDeltaWriter, open) { TDropTabletReq drop_request; auto tablet_id = 10003; auto schema_hash = 270068375; - res = k_engine->drop_tablet(tablet_id, schema_hash); + res = k_engine->tablet_manager()->drop_tablet(tablet_id, schema_hash); ASSERT_EQ(OLAP_SUCCESS, res); } - +*/ TEST_F(TestDeltaWriter, write) { TCreateTabletReq request; - create_tablet_request(&request); + create_tablet_request(10004, 270068376, &request); OLAPStatus res = k_engine->create_tablet(request); ASSERT_EQ(OLAP_SUCCESS, res); @@ -327,8 +327,8 @@ TEST_F(TestDeltaWriter, write) { PUniqueId load_id; load_id.set_hi(0); load_id.set_lo(0); - WriteRequest write_req = {10003, 270068375, WriteType::LOAD, - 20001, 30001, load_id, false, tuple_desc}; + WriteRequest write_req = {10004, 270068376, WriteType::LOAD, + 20002, 30002, load_id, false, tuple_desc}; DeltaWriter* delta_writer = nullptr; DeltaWriter::open(&write_req, &delta_writer); ASSERT_NE(delta_writer, nullptr); @@ -381,7 +381,7 @@ TEST_F(TestDeltaWriter, write) { var_ptr = (StringValue*)(tuple->get_slot(slots[18]->tuple_offset())); var_ptr->ptr = arena.Allocate(5); memcpy(var_ptr->ptr, "abcde", 5); - var_ptr->len = 5; + var_ptr->len = 5; DecimalValue val_decimal(1.1); *(DecimalValue*)(tuple->get_slot(slots[19]->tuple_offset())) = val_decimal; @@ -391,252 +391,38 @@ TEST_F(TestDeltaWriter, write) { } res = delta_writer->close(nullptr); - ASSERT_EQ(res, OLAP_SUCCESS); - - // publish version success - TabletSharedPtr tablet = TabletManager::instance()->get_tablet(write_req.tablet_id, write_req.schema_hash); - TPublishVersionRequest publish_req; - publish_req.transaction_id = write_req.transaction_id; - TPartitionVersionInfo info; - info.partition_id = write_req.partition_id; - info.version = tablet->rowset_with_max_version()->end_version() + 1; - info.version_hash = tablet->rowset_with_max_version()->version_hash() + 1; - std::vector partition_version_infos; - partition_version_infos.push_back(info); - publish_req.partition_version_infos = partition_version_infos; - std::vector error_tablet_ids; - res = k_engine->publish_version(publish_req, &error_tablet_ids); - - ASSERT_EQ(1, tablet->get_num_rows()); - - auto tablet_id = 10003; - auto schema_hash = 270068375; - res = k_engine->drop_tablet(tablet_id, schema_hash); - ASSERT_EQ(OLAP_SUCCESS, res); -} - -// ######################### ALTER TABLE TEST BEGIN ######################### - -void schema_change_request(const TCreateTabletReq& base_request, TCreateTabletReq* request) { - //linked schema change, add a value column - request->tablet_id = base_request.tablet_id + 1; - request->__set_version(base_request.version); - request->__set_version_hash(base_request.version_hash); - request->tablet_schema.schema_hash = base_request.tablet_schema.schema_hash + 1; - request->tablet_schema.short_key_column_count = 3; - request->tablet_schema.storage_type = TStorageType::COLUMN; - - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[2]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[3]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[4]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[5]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[6]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[7]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[8]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[9]); - - TColumn v0; - v0.column_name = "v0"; - v0.column_type.type = TPrimitiveType::BIGINT; - v0.__set_is_key(false); - v0.__set_default_value("0"); - v0.__set_aggregation_type(TAggregationType::SUM); - request->tablet_schema.columns.push_back(v0); - - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[10]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[11]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[12]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[13]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[14]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[15]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[16]); - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[17]); - - TColumn v9; - v9.column_name = "v9"; - v9.__set_is_key(false); - v9.column_type.type = TPrimitiveType::VARCHAR; - v9.column_type.__set_len(130); - v9.__set_aggregation_type(TAggregationType::REPLACE); - request->tablet_schema.columns.push_back(v9); - - request->tablet_schema.columns.push_back(base_request.tablet_schema.columns[19]); -} - -AlterTableStatus show_alter_tablet_status(const TAlterTabletReq& request) { - AlterTableStatus status = ALTER_TABLE_RUNNING; - uint32_t max_retry = MAX_RETRY_TIMES; - while (max_retry > 0) { - status = k_engine->show_alter_tablet_status( - request.base_tablet_id, request.base_schema_hash); - if (status != ALTER_TABLE_RUNNING) { break; } - LOG(INFO) << "doing alter tablet......"; - --max_retry; - sleep(1); - } - return status; -} - -class TestSchemaChange : public ::testing::Test { -public: - TestSchemaChange() { } - ~TestSchemaChange() { } - - void SetUp() { - // Create local data dir for StorageEngine. - char buffer[MAX_PATH_LEN]; - getcwd(buffer, MAX_PATH_LEN); - config::storage_root_path = std::string(buffer) + "/data_schema_change"; - remove_all_dir(config::storage_root_path); - ASSERT_EQ(create_dir(config::storage_root_path), OLAP_SUCCESS); - } - - void TearDown(){ - // Remove all dir. - ASSERT_EQ(OLAP_SUCCESS, remove_all_dir(config::storage_root_path)); - } -}; - -TEST_F(TestSchemaChange, schema_change) { - OLAPStatus res = OLAP_SUCCESS; - AlterTableStatus status = ALTER_TABLE_WAITING; - - // 1. Prepare for schema change. - // create base tablet - TCreateTabletReq create_base_tablet; - create_tablet_request(&create_base_tablet); - res = k_engine->create_tablet(create_base_tablet); ASSERT_EQ(OLAP_SUCCESS, res); - TDescriptorTable tdesc_tbl = create_descriptor_tablet(); - ObjectPool obj_pool; - DescriptorTbl* desc_tbl = nullptr; - DescriptorTbl::create(&obj_pool, tdesc_tbl, &desc_tbl); - TupleDescriptor* tuple_desc = desc_tbl->get_tuple_descriptor(0); - - PUniqueId load_id; - load_id.set_hi(0); - load_id.set_lo(0); - WriteRequest write_req = {10003, 270068375, WriteType::LOAD, - 20001, 30001, load_id, false, tuple_desc}; - DeltaWriter* delta_writer = nullptr; - DeltaWriter::open(&write_req, &delta_writer); - ASSERT_NE(delta_writer, nullptr); - - const std::vector& slots = tuple_desc->slots(); - Arena arena; - // streaming load data - { - Tuple* tuple = reinterpret_cast(arena.Allocate(tuple_desc->byte_size())); - memset(tuple, 0, tuple_desc->byte_size()); - *(int8_t*)(tuple->get_slot(slots[0]->tuple_offset())) = -127; - *(int16_t*)(tuple->get_slot(slots[1]->tuple_offset())) = -32767; - *(int32_t*)(tuple->get_slot(slots[2]->tuple_offset())) = -2147483647; - *(int64_t*)(tuple->get_slot(slots[3]->tuple_offset())) = -9223372036854775807L; - - int128_t large_int_value = -90000; - memcpy(tuple->get_slot(slots[4]->tuple_offset()), &large_int_value, sizeof(int128_t)); - - ((DateTimeValue*)(tuple->get_slot(slots[5]->tuple_offset())))->from_date_str("2048-11-10", 10); - ((DateTimeValue*)(tuple->get_slot(slots[6]->tuple_offset())))->from_date_str("2636-08-16 19:39:43", 19); - - StringValue* char_ptr = (StringValue*)(tuple->get_slot(slots[7]->tuple_offset())); - char_ptr->ptr = arena.Allocate(4); - memcpy(char_ptr->ptr, "abcd", 4); - char_ptr->len = 4; - - StringValue* var_ptr = (StringValue*)(tuple->get_slot(slots[8]->tuple_offset())); - var_ptr->ptr = arena.Allocate(5); - memcpy(var_ptr->ptr, "abcde", 5); - var_ptr->len = 5; - - DecimalValue decimal_value(1.1); - *(DecimalValue*)(tuple->get_slot(slots[9]->tuple_offset())) = decimal_value; - - *(int8_t*)(tuple->get_slot(slots[10]->tuple_offset())) = -127; - *(int16_t*)(tuple->get_slot(slots[11]->tuple_offset())) = -32767; - *(int32_t*)(tuple->get_slot(slots[12]->tuple_offset())) = -2147483647; - *(int64_t*)(tuple->get_slot(slots[13]->tuple_offset())) = -9223372036854775807L; - - memcpy(tuple->get_slot(slots[14]->tuple_offset()), &large_int_value, sizeof(int128_t)); - - ((DateTimeValue*)(tuple->get_slot(slots[15]->tuple_offset())))->from_date_str("2048-11-10", 10); - ((DateTimeValue*)(tuple->get_slot(slots[16]->tuple_offset())))->from_date_str("2636-08-16 19:39:43", 19); - - char_ptr = (StringValue*)(tuple->get_slot(slots[17]->tuple_offset())); - char_ptr->ptr = arena.Allocate(4); - memcpy(char_ptr->ptr, "abcd", 4); - char_ptr->len = 4; - - var_ptr = (StringValue*)(tuple->get_slot(slots[18]->tuple_offset())); - var_ptr->ptr = arena.Allocate(5); - memcpy(var_ptr->ptr, "abcde", 5); - var_ptr->len = 5; - - DecimalValue val_decimal(1.1); - *(DecimalValue*)(tuple->get_slot(slots[19]->tuple_offset())) = val_decimal; - - res = delta_writer->write(tuple); + // publish version success + TabletSharedPtr tablet = k_engine->tablet_manager()->get_tablet(write_req.tablet_id, write_req.schema_hash); + std::cout << "before publish, tablet row nums:" << tablet->num_rows() << std::endl; + OlapMeta* meta = tablet->data_dir()->get_meta(); + Version version; + version.first = tablet->rowset_with_max_version()->end_version() + 1; + version.second = tablet->rowset_with_max_version()->end_version() + 1; + std::cout << "start to add rowset version:" << version.first << "-" << version.second << std::endl; + VersionHash version_hash = 2; + std::map tablet_related_rs; + StorageEngine::instance()->txn_manager()->get_txn_related_tablets(write_req.txn_id, write_req.partition_id, &tablet_related_rs); + for (auto& tablet_rs : tablet_related_rs) { + std::cout << "start to publish txn" << std::endl; + RowsetSharedPtr rowset = tablet_rs.second; + res = k_engine->txn_manager()->publish_txn(meta, write_req.partition_id, write_req.txn_id, + write_req.tablet_id, write_req.schema_hash, + version, version_hash); + ASSERT_EQ(OLAP_SUCCESS, res); + std::cout << "start to add inc rowset:" << rowset->rowset_id() << ", num rows:" << rowset->num_rows() + << ", version:" << rowset->version().first << "-" << rowset->version().second + << ", version_hash:" << rowset->version_hash() + << std::endl; + res = tablet->add_inc_rowset(rowset); ASSERT_EQ(OLAP_SUCCESS, res); } + ASSERT_EQ(1, tablet->num_rows()); - // publish version - res = delta_writer->close(nullptr); - ASSERT_EQ(res, OLAP_SUCCESS); - - // publish version success - TabletSharedPtr tablet = TabletManager::instance()->get_tablet(write_req.tablet_id, write_req.schema_hash); - TPublishVersionRequest publish_req; - publish_req.transaction_id = write_req.transaction_id; - TPartitionVersionInfo info; - info.partition_id = write_req.partition_id; - info.version = tablet->rowset_with_max_version()->end_version() + 1; - info.version_hash = tablet->rowset_with_max_version()->version_hash() + 1; - std::vector partition_version_infos; - partition_version_infos.push_back(info); - publish_req.partition_version_infos = partition_version_infos; - std::vector error_tablet_ids; - res = k_engine->publish_version(publish_req, &error_tablet_ids); - ASSERT_EQ(res, OLAP_SUCCESS); - - // 1. set add column request - TCreateTabletReq create_new_tablet; - schema_change_request(create_base_tablet, &create_new_tablet); - TAlterTabletReq request; - request.__set_base_tablet_id(create_base_tablet.tablet_id); - request.__set_base_schema_hash(create_base_tablet.tablet_schema.schema_hash); - request.__set_new_tablet_req(create_new_tablet); - - // 2. Submit schema change - request.base_schema_hash = create_base_tablet.tablet_schema.schema_hash; - res = k_engine->schema_change(request); - ASSERT_EQ(OLAP_SUCCESS, res); - - // 3. Verify schema change result. - // show schema change status - status = show_alter_tablet_status(request); - ASSERT_EQ(ALTER_TABLE_FINISHED, status); - - // check new tablet information - TTabletInfo tablet_info; - tablet_info.tablet_id = create_new_tablet.tablet_id; - tablet_info.schema_hash = create_new_tablet.tablet_schema.schema_hash; - res = k_engine->report_tablet_info(&tablet_info); - ASSERT_EQ(OLAP_SUCCESS, res); - ASSERT_EQ(info.version, tablet_info.version); - ASSERT_EQ(info.version_hash, tablet_info.version_hash); - ASSERT_EQ(1, tablet_info.row_count); - - // 4. Retry the same schema change request. - res = k_engine->schema_change(request); - ASSERT_EQ(OLAP_SUCCESS, res); - status = k_engine->show_alter_tablet_status( - request.base_tablet_id, request.base_schema_hash); - ASSERT_EQ(ALTER_TABLE_FINISHED, status); - - auto tablet_id = create_new_tablet.tablet_id; - auto schema_hash = create_new_tablet.tablet_schema.schema_hash; - res = k_engine->drop_tablet(tablet_id, schema_hash); + auto tablet_id = 10003; + auto schema_hash = 270068375; + res = k_engine->tablet_manager()->drop_tablet(tablet_id, schema_hash); ASSERT_EQ(OLAP_SUCCESS, res); } @@ -652,11 +438,9 @@ int main(int argc, char** argv) { int ret = doris::OLAP_SUCCESS; testing::InitGoogleTest(&argc, argv); doris::CpuInfo::init(); - doris::set_up(); ret = RUN_ALL_TESTS(); doris::tear_down(); - google::protobuf::ShutdownProtobufLibrary(); return ret; } diff --git a/be/test/olap/in_list_predicate_test.cpp b/be/test/olap/in_list_predicate_test.cpp index 5b187d4c34aa58..de2acb52a5301a 100644 --- a/be/test/olap/in_list_predicate_test.cpp +++ b/be/test/olap/in_list_predicate_test.cpp @@ -100,22 +100,25 @@ class TestInListPredicate : public testing::Test { } } - void SetFieldInfo(FieldInfo &field_info, std::string name, - FieldType type, FieldAggregationMethod aggregation, - uint32_t length, bool is_allow_null, bool is_key) { - field_info.name = name; - field_info.type = type; - field_info.aggregation = aggregation; - field_info.length = length; - field_info.is_allow_null = is_allow_null; - field_info.is_key = is_key; - field_info.precision = 1000; - field_info.frac = 10000; - field_info.unique_id = 0; - field_info.is_bf_column = false; + void SetTabletColumn(std::string name, + std::string type, std::string aggregation, + uint32_t length, bool is_allow_null, bool is_key, TabletColumn* tablet_column) { + static int id = 0; + ColumnPB column; + column.set_unique_id(++id); + column.set_name(name); + column.set_type(type); + column.set_is_key(is_key); + column.set_is_nullable(is_allow_null); + column.set_length(length); + column.set_aggregation(aggregation); + column.set_precision(1000); + column.set_frac(1000); + column.set_is_bf_column(false); + tablet_column->init_from_pb(column); } - void InitVectorizedBatch(const std::vector& schema, + void InitVectorizedBatch(const std::vector* schema, const std::vector& ids, int size) { _vectorized_batch = new VectorizedRowBatch(schema, ids, size); @@ -128,17 +131,17 @@ class TestInListPredicate : public testing::Test { #define TEST_IN_LIST_PREDICATE(TYPE, TYPE_NAME, FIELD_TYPE) \ TEST_F(TestInListPredicate, TYPE_NAME##_COLUMN) { \ - std::vector schema; \ - FieldInfo field_info; \ - SetFieldInfo(field_info, std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, \ - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); \ - schema.push_back(field_info); \ + std::vector schema; \ + TabletColumn tablet_schema; \ + SetTabletColumn(std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, \ + "REPLACE", 1, false, true, &tablet_schema); \ + schema.push_back(tablet_schema); \ int size = 10; \ std::vector return_columns; \ for (int i = 0; i < schema.size(); ++i) { \ return_columns.push_back(i); \ } \ - InitVectorizedBatch(schema, return_columns, size); \ + InitVectorizedBatch(&schema, return_columns, size); \ ColumnVector* col_vector = _vectorized_batch->column(0); \ \ /* for no nulls */ \ @@ -181,24 +184,24 @@ TEST_F(TestInListPredicate, TYPE_NAME##_COLUMN) { \ ASSERT_EQ(*(col_data + sel[0]), 5); \ } \ -TEST_IN_LIST_PREDICATE(int8_t, TINYINT, OLAP_FIELD_TYPE_TINYINT) -TEST_IN_LIST_PREDICATE(int16_t, SMALLINT, OLAP_FIELD_TYPE_SMALLINT) -TEST_IN_LIST_PREDICATE(int32_t, INT, OLAP_FIELD_TYPE_INT) -TEST_IN_LIST_PREDICATE(int64_t, BIGINT, OLAP_FIELD_TYPE_BIGINT) -TEST_IN_LIST_PREDICATE(int128_t, LARGEINT, OLAP_FIELD_TYPE_LARGEINT) +TEST_IN_LIST_PREDICATE(int8_t, TINYINT, "TINYINT") +TEST_IN_LIST_PREDICATE(int16_t, SMALLINT, "SMALLINT") +TEST_IN_LIST_PREDICATE(int32_t, INT, "INT") +TEST_IN_LIST_PREDICATE(int64_t, BIGINT, "BIGINT") +TEST_IN_LIST_PREDICATE(int128_t, LARGEINT, "LARGEINT") TEST_F(TestInListPredicate, FLOAT_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("FLOAT_COLUMN"), OLAP_FIELD_TYPE_FLOAT, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_schema; + SetTabletColumn(std::string("FLOAT_COLUMN"), "FLOAT", + "REPLACE", 1, false, true, &tablet_schema); + schema.push_back(tablet_schema); int size = 10; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -241,17 +244,17 @@ TEST_F(TestInListPredicate, FLOAT_COLUMN) { } TEST_F(TestInListPredicate, DOUBLE_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DOUBLE_COLUMN"), OLAP_FIELD_TYPE_DOUBLE, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_schema; + SetTabletColumn(std::string("DOUBLE_COLUMN"), "DOUBLE", + "REPLACE", 1, false, true, &tablet_schema); + schema.push_back(tablet_schema); int size = 10; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -295,17 +298,17 @@ TEST_F(TestInListPredicate, DOUBLE_COLUMN) { } TEST_F(TestInListPredicate, DECIMAL_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DECIMAL_COLUMN"), OLAP_FIELD_TYPE_DECIMAL, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_schema; + SetTabletColumn(std::string("DECIMAL_COLUMN"), "DECIMAL", + "REPLACE", 1, false, true, &tablet_schema); + schema.push_back(tablet_schema); int size = 10; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -358,17 +361,17 @@ TEST_F(TestInListPredicate, DECIMAL_COLUMN) { } TEST_F(TestInListPredicate, CHAR_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("STRING_COLUMN"), OLAP_FIELD_TYPE_CHAR, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_schema; + SetTabletColumn(std::string("STRING_COLUMN"), "CHAR", + "REPLACE", 1, false, true, &tablet_schema); + schema.push_back(tablet_schema); int size = 10; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -443,17 +446,17 @@ TEST_F(TestInListPredicate, CHAR_COLUMN) { } TEST_F(TestInListPredicate, VARCHAR_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("STRING_COLUMN"), OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_schema; + SetTabletColumn(std::string("STRING_COLUMN"), "VARCHAR", + "REPLACE", 1, false, true, &tablet_schema); + schema.push_back(tablet_schema); int size = 10; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -526,17 +529,17 @@ TEST_F(TestInListPredicate, VARCHAR_COLUMN) { } TEST_F(TestInListPredicate, DATE_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DATE_COLUMN"), OLAP_FIELD_TYPE_DATE, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_schema; + SetTabletColumn(std::string("DATE_COLUMN"), "DATE", + "REPLACE", 1, false, true, &tablet_schema); + schema.push_back(tablet_schema); int size = 6; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -597,17 +600,17 @@ TEST_F(TestInListPredicate, DATE_COLUMN) { } TEST_F(TestInListPredicate, DATETIME_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DATETIME_COLUMN"), OLAP_FIELD_TYPE_DATETIME, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_schema; + SetTabletColumn(std::string("DATETIME_COLUMN"), "DATETIME", + "REPLACE", 1, false, true, &tablet_schema); + schema.push_back(tablet_schema); int size = 6; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls diff --git a/be/test/olap/null_predicate_test.cpp b/be/test/olap/null_predicate_test.cpp index 52c45975611f20..7a8fa448db20ab 100644 --- a/be/test/olap/null_predicate_test.cpp +++ b/be/test/olap/null_predicate_test.cpp @@ -70,22 +70,25 @@ class TestNullPredicate : public testing::Test { } } - void SetFieldInfo(FieldInfo &field_info, std::string name, - FieldType type, FieldAggregationMethod aggregation, - uint32_t length, bool is_allow_null, bool is_key) { - field_info.name = name; - field_info.type = type; - field_info.aggregation = aggregation; - field_info.length = length; - field_info.is_allow_null = is_allow_null; - field_info.is_key = is_key; - field_info.precision = 1000; - field_info.frac = 10000; - field_info.unique_id = 0; - field_info.is_bf_column = false; + void SetTabletColumn(std::string name, + std::string type, std::string aggregation, + uint32_t length, bool is_allow_null, bool is_key, TabletColumn* tablet_column) { + static int id = 0; + ColumnPB column; + column.set_unique_id(++id); + column.set_name(name); + column.set_type(type); + column.set_is_key(is_key); + column.set_is_nullable(is_allow_null); + column.set_length(length); + column.set_aggregation(aggregation); + column.set_precision(1000); + column.set_frac(1000); + column.set_is_bf_column(false); + tablet_column->init_from_pb(column); } - void InitVectorizedBatch(const std::vector& schema, + void InitVectorizedBatch(const std::vector* schema, const std::vector&ids, int size) { _vectorized_batch = new VectorizedRowBatch(schema, ids, size); @@ -98,17 +101,17 @@ class TestNullPredicate : public testing::Test { #define TEST_IN_LIST_PREDICATE(TYPE, TYPE_NAME, FIELD_TYPE) \ TEST_F(TestNullPredicate, TYPE_NAME##_COLUMN) { \ - std::vector schema; \ - FieldInfo field_info; \ - SetFieldInfo(field_info, std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, \ - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); \ - schema.push_back(field_info); \ + std::vector schema; \ + TabletColumn tablet_schema; \ + SetTabletColumn(std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, \ + "REPLACE", 1, false, true, &tablet_schema); \ + schema.push_back(tablet_schema); \ int size = 10; \ std::vector return_columns; \ for (int i = 0; i < schema.size(); ++i) { \ return_columns.push_back(i); \ } \ - InitVectorizedBatch(schema, return_columns, size); \ + InitVectorizedBatch(&schema, return_columns, size); \ ColumnVector* col_vector = _vectorized_batch->column(0); \ \ /* for no nulls */ \ @@ -141,24 +144,24 @@ TEST_F(TestNullPredicate, TYPE_NAME##_COLUMN) { \ ASSERT_EQ(_vectorized_batch->size(), 5); \ } \ -TEST_IN_LIST_PREDICATE(int8_t, TINYINT, OLAP_FIELD_TYPE_TINYINT) -TEST_IN_LIST_PREDICATE(int16_t, SMALLINT, OLAP_FIELD_TYPE_SMALLINT) -TEST_IN_LIST_PREDICATE(int32_t, INT, OLAP_FIELD_TYPE_INT) -TEST_IN_LIST_PREDICATE(int64_t, BIGINT, OLAP_FIELD_TYPE_BIGINT) -TEST_IN_LIST_PREDICATE(int128_t, LARGEINT, OLAP_FIELD_TYPE_LARGEINT) +TEST_IN_LIST_PREDICATE(int8_t, TINYINT, "TINYINT") +TEST_IN_LIST_PREDICATE(int16_t, SMALLINT, "SMALLINT") +TEST_IN_LIST_PREDICATE(int32_t, INT, "INT") +TEST_IN_LIST_PREDICATE(int64_t, BIGINT, "BIGINT") +TEST_IN_LIST_PREDICATE(int128_t, LARGEINT, "LARGEINT") TEST_F(TestNullPredicate, FLOAT_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("FLOAT_COLUMN"), OLAP_FIELD_TYPE_FLOAT, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_schema; + SetTabletColumn(std::string("FLOAT_COLUMN"), "FLOAT", + "REPLACE", 1, false, true, &tablet_schema); + schema.push_back(tablet_schema); int size = 10; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -191,17 +194,17 @@ TEST_F(TestNullPredicate, FLOAT_COLUMN) { } TEST_F(TestNullPredicate, DOUBLE_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DOUBLE_COLUMN"), OLAP_FIELD_TYPE_DOUBLE, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_schema; + SetTabletColumn(std::string("DOUBLE_COLUMN"), "DOUBLE", + "REPLACE", 1, false, true, &tablet_schema); + schema.push_back(tablet_schema); int size = 10; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -235,17 +238,17 @@ TEST_F(TestNullPredicate, DOUBLE_COLUMN) { } TEST_F(TestNullPredicate, DECIMAL_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DECIMAL_COLUMN"), OLAP_FIELD_TYPE_DECIMAL, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_schema; + SetTabletColumn(std::string("DECIMAL_COLUMN"), "DECIMAL", + "REPLACE", 1, false, true, &tablet_schema); + schema.push_back(tablet_schema); int size = 10; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -282,17 +285,17 @@ TEST_F(TestNullPredicate, DECIMAL_COLUMN) { } TEST_F(TestNullPredicate, STRING_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("STRING_COLUMN"), OLAP_FIELD_TYPE_VARCHAR, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_schema; + SetTabletColumn(std::string("STRING_COLUMN"), "VARCHAR", + "REPLACE", 1, false, true, &tablet_schema); + schema.push_back(tablet_schema); int size = 10; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -339,17 +342,17 @@ TEST_F(TestNullPredicate, STRING_COLUMN) { } TEST_F(TestNullPredicate, DATE_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DATE_COLUMN"), OLAP_FIELD_TYPE_DATE, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_schema; + SetTabletColumn(std::string("DATE_COLUMN"), "DATE", + "REPLACE", 1, false, true, &tablet_schema); + schema.push_back(tablet_schema); int size = 6; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -394,17 +397,17 @@ TEST_F(TestNullPredicate, DATE_COLUMN) { } TEST_F(TestNullPredicate, DATETIME_COLUMN) { - std::vector schema; - FieldInfo field_info; - SetFieldInfo(field_info, std::string("DATETIME_COLUMN"), OLAP_FIELD_TYPE_DATETIME, - OLAP_FIELD_AGGREGATION_REPLACE, 1, false, true); - schema.push_back(field_info); + std::vector schema; + TabletColumn tablet_schema; + SetTabletColumn(std::string("DATETIME_COLUMN"), "DATETIME", + "REPLACE", 1, false, true, &tablet_schema); + schema.push_back(tablet_schema); int size = 6; std::vector return_columns; for (int i = 0; i < schema.size(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(schema, return_columns, size); + InitVectorizedBatch(&schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls diff --git a/be/test/olap/row_block_test.cpp b/be/test/olap/row_block_test.cpp index 4b108e94522eff..4b52d13cb3fc58 100644 --- a/be/test/olap/row_block_test.cpp +++ b/be/test/olap/row_block_test.cpp @@ -45,46 +45,51 @@ class TestRowBlock : public testing::Test { } }; -TEST_F(TestRowBlock, init) { - std::vector fields; +void init_tablet_schema(TabletSchema* tablet_schema) { + TabletSchemaPB tablet_schema_pb; { // k1: bigint { - FieldInfo info; - info.name = "k1"; - info.type = OLAP_FIELD_TYPE_BIGINT; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 8; - info.is_key = true; - fields.push_back(info); + ColumnPB* column_1 = tablet_schema_pb.add_column(); + column_1->set_unique_id(1); + column_1->set_name("k1"); + column_1->set_type("BIGINT"); + column_1->set_is_key(true); + column_1->set_length(8); + column_1->set_aggregation("NONE"); } // k2: char { - FieldInfo info; - info.name = "k2"; - info.type = OLAP_FIELD_TYPE_CHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 10; - info.is_key = true; - fields.push_back(info); + ColumnPB* column_2 = tablet_schema_pb.add_column(); + column_2->set_unique_id(2); + column_2->set_name("k2"); + column_2->set_type("CHAR"); + column_2->set_is_key(true); + column_2->set_length(10); + column_2->set_aggregation("NONE"); } // k3: varchar { - FieldInfo info; - info.name = "k3"; - info.type = OLAP_FIELD_TYPE_VARCHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 20; - info.is_key = true; - fields.push_back(info); + ColumnPB* column_3 = tablet_schema_pb.add_column(); + column_3->set_unique_id(3); + column_3->set_name("k3"); + column_3->set_type("VARCHAR"); + column_3->set_is_key(true); + column_3->set_length(20); + column_3->set_aggregation("NONE"); } } + tablet_schema->init_from_pb(tablet_schema_pb); +} + +TEST_F(TestRowBlock, init) { + TabletSchema tablet_schema; + init_tablet_schema(&tablet_schema); { // has nullbyte - RowBlock block(fields); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); @@ -93,63 +98,29 @@ TEST_F(TestRowBlock, init) { } { // has nullbyte - RowBlock block(fields); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = false; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); // num_rows * (num_nullbytes + bigint + char + varchar) - ASSERT_EQ(1024 * (3 + 8 + 10 + (4 + 20)), block.buf_len()); + ASSERT_EQ(1024 * (8 + 10 + (4 + 20)), block.buf_len()); } } TEST_F(TestRowBlock, write_and_read) { - std::vector fields; - { - // k1: bigint - { - FieldInfo info; - info.name = "k1"; - info.type = OLAP_FIELD_TYPE_BIGINT; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 8; - info.is_key = true; - fields.push_back(info); - } - // k2: char - { - FieldInfo info; - info.name = "k2"; - info.type = OLAP_FIELD_TYPE_CHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 10; - info.is_key = true; - fields.push_back(info); - } - // k3: varchar - { - FieldInfo info; - info.name = "k3"; - info.type = OLAP_FIELD_TYPE_VARCHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 20; - info.is_key = true; - fields.push_back(info); - } - } - // - RowBlock block(fields); + TabletSchema tablet_schema; + init_tablet_schema(&tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); RowCursor row; - row.init(fields); + row.init(tablet_schema); for (int i = 0; i < 5; ++i) { block.get_row(i, &row); @@ -185,7 +156,7 @@ TEST_F(TestRowBlock, write_and_read) { ASSERT_EQ(OLAP_SUCCESS, res); { - RowBlock resolve_block(fields); + RowBlock resolve_block(&tablet_schema); block_info.checksum = block.row_block_info().checksum; block_info.row_num = 5; res = resolve_block.init(block_info); @@ -222,50 +193,17 @@ TEST_F(TestRowBlock, write_and_read) { } TEST_F(TestRowBlock, write_and_read_without_nullbyte) { - std::vector fields; - { - // k1: bigint - { - FieldInfo info; - info.name = "k1"; - info.type = OLAP_FIELD_TYPE_BIGINT; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 8; - info.is_key = true; - fields.push_back(info); - } - // k2: char - { - FieldInfo info; - info.name = "k2"; - info.type = OLAP_FIELD_TYPE_CHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 10; - info.is_key = true; - fields.push_back(info); - } - // k3: varchar - { - FieldInfo info; - info.name = "k3"; - info.type = OLAP_FIELD_TYPE_VARCHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 20; - info.is_key = true; - fields.push_back(info); - } - } - // - RowBlock block(fields); + TabletSchema tablet_schema; + init_tablet_schema(&tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = false; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); RowCursor row; - row.init(fields); + row.init(tablet_schema); for (int i = 0; i < 5; ++i) { block.get_row(i, &row); @@ -301,7 +239,7 @@ TEST_F(TestRowBlock, write_and_read_without_nullbyte) { ASSERT_EQ(OLAP_SUCCESS, res); { - RowBlock resolve_block(fields); + RowBlock resolve_block(&tablet_schema); block_info.checksum = block.row_block_info().checksum; block_info.row_num = 5; res = resolve_block.init(block_info); @@ -338,50 +276,17 @@ TEST_F(TestRowBlock, write_and_read_without_nullbyte) { } TEST_F(TestRowBlock, compress_failed) { - std::vector fields; - { - // k1: bigint - { - FieldInfo info; - info.name = "k1"; - info.type = OLAP_FIELD_TYPE_BIGINT; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 8; - info.is_key = true; - fields.push_back(info); - } - // k2: char - { - FieldInfo info; - info.name = "k2"; - info.type = OLAP_FIELD_TYPE_CHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 10; - info.is_key = true; - fields.push_back(info); - } - // k3: varchar - { - FieldInfo info; - info.name = "k3"; - info.type = OLAP_FIELD_TYPE_VARCHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 20; - info.is_key = true; - fields.push_back(info); - } - } - // - RowBlock block(fields); + TabletSchema tablet_schema; + init_tablet_schema(&tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); RowCursor row; - row.init(fields); + row.init(tablet_schema); for (int i = 0; i < 5; ++i) { block.get_row(i, &row); @@ -415,50 +320,17 @@ TEST_F(TestRowBlock, compress_failed) { } TEST_F(TestRowBlock, decompress_failed) { - std::vector fields; - { - // k1: bigint - { - FieldInfo info; - info.name = "k1"; - info.type = OLAP_FIELD_TYPE_BIGINT; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 8; - info.is_key = true; - fields.push_back(info); - } - // k2: char - { - FieldInfo info; - info.name = "k2"; - info.type = OLAP_FIELD_TYPE_CHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 10; - info.is_key = true; - fields.push_back(info); - } - // k3: varchar - { - FieldInfo info; - info.name = "k3"; - info.type = OLAP_FIELD_TYPE_VARCHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 20; - info.is_key = true; - fields.push_back(info); - } - } - // - RowBlock block(fields); + TabletSchema tablet_schema; + init_tablet_schema(&tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); RowCursor row; - row.init(fields); + row.init(tablet_schema); for (int i = 0; i < 5; ++i) { block.get_row(i, &row); @@ -492,7 +364,7 @@ TEST_F(TestRowBlock, decompress_failed) { { // checksum failed - RowBlock resolve_block(fields); + RowBlock resolve_block(&tablet_schema); block_info.checksum = 0; block_info.row_num = 5; res = resolve_block.init(block_info); @@ -503,7 +375,7 @@ TEST_F(TestRowBlock, decompress_failed) { } { // buffer is not ok - RowBlock resolve_block(fields); + RowBlock resolve_block(&tablet_schema); block_info.checksum = block.row_block_info().checksum; block_info.row_num = 5; res = resolve_block.init(block_info); @@ -515,50 +387,17 @@ TEST_F(TestRowBlock, decompress_failed) { } TEST_F(TestRowBlock, find_row) { - std::vector fields; - { - // k1: bigint - { - FieldInfo info; - info.name = "k1"; - info.type = OLAP_FIELD_TYPE_BIGINT; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 8; - info.is_key = true; - fields.push_back(info); - } - // k2: char - { - FieldInfo info; - info.name = "k2"; - info.type = OLAP_FIELD_TYPE_CHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 10; - info.is_key = true; - fields.push_back(info); - } - // k3: varchar - { - FieldInfo info; - info.name = "k3"; - info.type = OLAP_FIELD_TYPE_VARCHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 20; - info.is_key = true; - fields.push_back(info); - } - } - // - RowBlock block(fields); + TabletSchema tablet_schema; + init_tablet_schema(&tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); RowCursor row; - row.init(fields); + row.init(tablet_schema); for (int i = 0; i < 5; ++i) { block.get_row(i, &row); @@ -590,7 +429,7 @@ TEST_F(TestRowBlock, find_row) { { RowCursor find_row; - find_row.init(fields); + find_row.init(tablet_schema); for (int i = 0; i < 5; ++i) { // bigint { @@ -677,44 +516,11 @@ TEST_F(TestRowBlock, find_row) { } TEST_F(TestRowBlock, clear) { - std::vector fields; - { - // k1: bigint - { - FieldInfo info; - info.name = "k1"; - info.type = OLAP_FIELD_TYPE_BIGINT; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 8; - info.is_key = true; - fields.push_back(info); - } - // k2: char - { - FieldInfo info; - info.name = "k2"; - info.type = OLAP_FIELD_TYPE_CHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 10; - info.is_key = true; - fields.push_back(info); - } - // k3: varchar - { - FieldInfo info; - info.name = "k3"; - info.type = OLAP_FIELD_TYPE_VARCHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 20; - info.is_key = true; - fields.push_back(info); - } - } - // - RowBlock block(fields); + TabletSchema tablet_schema; + init_tablet_schema(&tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); @@ -727,44 +533,11 @@ TEST_F(TestRowBlock, clear) { } TEST_F(TestRowBlock, pos_limit) { - std::vector fields; - { - // k1: bigint - { - FieldInfo info; - info.name = "k1"; - info.type = OLAP_FIELD_TYPE_BIGINT; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 8; - info.is_key = true; - fields.push_back(info); - } - // k2: char - { - FieldInfo info; - info.name = "k2"; - info.type = OLAP_FIELD_TYPE_CHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 10; - info.is_key = true; - fields.push_back(info); - } - // k3: varchar - { - FieldInfo info; - info.name = "k3"; - info.type = OLAP_FIELD_TYPE_VARCHAR; - info.aggregation = OLAP_FIELD_AGGREGATION_NONE; - info.length = 20; - info.is_key = true; - fields.push_back(info); - } - } - // - RowBlock block(fields); + TabletSchema tablet_schema; + init_tablet_schema(&tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); diff --git a/be/test/olap/row_cursor_test.cpp b/be/test/olap/row_cursor_test.cpp index 604126d0f0cd05..3d2a00499787d4 100644 --- a/be/test/olap/row_cursor_test.cpp +++ b/be/test/olap/row_cursor_test.cpp @@ -18,219 +18,286 @@ #include #include "olap/row_cursor.h" +#include "olap/tablet_schema.h" #include "runtime/mem_tracker.h" #include "runtime/mem_pool.h" #include "util/logging.h" namespace doris { -void set_tablet_schema_for_init(std::vector* tablet_schema) { - FieldInfo k1; - k1.name = "k1"; - k1.type = OLAP_FIELD_TYPE_TINYINT; - k1.length = 1; - k1.is_key = true; - k1.index_length = 1; - k1.is_allow_null = true; +void set_tablet_schema_for_init(std::vector* tablet_schema) { + ColumnPB column_1; + column_1.set_unique_id(1); + column_1.set_name("column_1"); + column_1.set_type("TINYINT"); + column_1.set_is_key(true); + column_1.set_is_nullable(true); + column_1.set_length(1); + column_1.set_index_length(1); + TabletColumn k1; + k1.init_from_pb(column_1); tablet_schema->push_back(k1); - FieldInfo k2; - k2.name = "k2"; - k2.type = OLAP_FIELD_TYPE_SMALLINT; - k2.length = 2; - k2.default_value = "0"; - k2.is_key = true; - k2.index_length = 2; - k2.is_allow_null = true; + ColumnPB column_2; + column_2.set_unique_id(2); + column_2.set_name("column_2"); + column_2.set_type("SMALLINT"); + column_2.set_is_key(true); + column_2.set_is_nullable(true); + column_2.set_length(2); + column_2.set_index_length(2); + column_2.set_default_value("0"); + TabletColumn k2; + k2.init_from_pb(column_2); tablet_schema->push_back(k2); - FieldInfo k3; - k3.name = "k3"; - k3.type = OLAP_FIELD_TYPE_INT; - k3.length = 4; - k3.is_key = true; - k3.index_length = 4; - k3.is_allow_null = true; + ColumnPB column_3; + column_3.set_unique_id(3); + column_3.set_name("column_3"); + column_3.set_type("INT"); + column_3.set_is_key(true); + column_3.set_is_nullable(true); + column_3.set_length(4); + column_3.set_index_length(4); + TabletColumn k3; + k3.init_from_pb(column_3); tablet_schema->push_back(k3); - FieldInfo k4; - k4.name = "k4"; - k4.type = OLAP_FIELD_TYPE_DATE; - k4.length = 3; - k4.is_key = true; - k4.index_length = 3; - k4.is_allow_null = true; + ColumnPB column_4; + column_4.set_unique_id(4); + column_4.set_name("column_4"); + column_4.set_type("DATE"); + column_4.set_is_key(true); + column_4.set_is_nullable(true); + column_4.set_length(3); + column_4.set_index_length(3); + TabletColumn k4; + k4.init_from_pb(column_4); tablet_schema->push_back(k4); - FieldInfo k5; - k5.name = "k5"; - k5.type = OLAP_FIELD_TYPE_DATETIME; - k5.length = 8; - k5.is_key = true; - k5.index_length = 8; - k5.is_allow_null = true; + ColumnPB column_5; + column_5.set_unique_id(5); + column_5.set_name("column_5"); + column_5.set_type("DATETIME"); + column_5.set_is_key(true); + column_5.set_is_nullable(true); + column_5.set_length(8); + column_5.set_index_length(8); + TabletColumn k5; + k5.init_from_pb(column_5); tablet_schema->push_back(k5); - FieldInfo k6; - k6.name = "k6"; - k6.type = OLAP_FIELD_TYPE_DECIMAL; - k6.length = 12; - k6.precision = 6; - k6.frac = 3; - k6.is_key = true; - k6.index_length = 12; - k6.is_allow_null = true; + ColumnPB column_6; + column_6.set_unique_id(6); + column_6.set_name("column_6"); + column_6.set_type("DECIMAL"); + column_6.set_is_key(true); + column_6.set_is_nullable(true); + column_6.set_length(12); + column_6.set_index_length(12); + column_6.set_frac(3); + column_6.set_precision(6); + TabletColumn k6; + k6.init_from_pb(column_6); tablet_schema->push_back(k6); - FieldInfo k7; - k7.name = "k7"; - k7.type = OLAP_FIELD_TYPE_CHAR; - k7.length = 4; - k7.default_value = "char"; - k7.is_key = true; - k7.index_length = 4; - k7.is_allow_null = true; + ColumnPB column_7; + column_7.set_unique_id(7); + column_7.set_name("column_7"); + column_7.set_type("CHAR"); + column_7.set_is_key(true); + column_7.set_is_nullable(true); + column_7.set_length(4); + column_7.set_index_length(4); + column_7.set_default_value("char"); + TabletColumn k7; + k7.init_from_pb(column_7); tablet_schema->push_back(k7); - FieldInfo v1; - v1.name = "v1"; - v1.type = OLAP_FIELD_TYPE_BIGINT; - v1.length = 8; - v1.aggregation = OLAP_FIELD_AGGREGATION_SUM; - v1.is_key = false; - v1.is_allow_null = true; + ColumnPB column_8; + column_8.set_unique_id(8); + column_8.set_name("column_8"); + column_8.set_type("BIGINT"); + column_8.set_is_nullable(true); + column_8.set_length(8); + column_8.set_aggregation("SUM"); + column_8.set_is_key(false); + TabletColumn v1; + v1.init_from_pb(column_8); tablet_schema->push_back(v1); - FieldInfo v2; - v2.name = "v2"; - v2.type = OLAP_FIELD_TYPE_VARCHAR; - v2.length = 16 + OLAP_STRING_MAX_BYTES; - v2.aggregation = OLAP_FIELD_AGGREGATION_REPLACE; - v2.is_key = false; - v2.is_allow_null = true; + ColumnPB column_9; + column_9.set_unique_id(9); + column_9.set_name("column_9"); + column_9.set_type("VARCHAR"); + column_9.set_is_nullable(true); + column_9.set_length(16 + OLAP_STRING_MAX_BYTES); + column_9.set_aggregation("REPLACE"); + column_9.set_is_key(false); + TabletColumn v2; + v2.init_from_pb(column_9); tablet_schema->push_back(v2); - FieldInfo v3; - v3.name = "v3"; - v3.type = OLAP_FIELD_TYPE_LARGEINT; - v3.length = 16; - v3.aggregation = OLAP_FIELD_AGGREGATION_MAX; - v3.is_key = false; - v3.is_allow_null = true; + ColumnPB column_10; + column_10.set_unique_id(10); + column_10.set_name("column_10"); + column_10.set_type("LARGEINT"); + column_10.set_is_nullable(true); + column_10.set_length(16); + column_10.set_aggregation("MAX"); + column_10.set_is_key(false); + TabletColumn v3; + v3.init_from_pb(column_10); tablet_schema->push_back(v3); - FieldInfo v4; - v4.name = "v4"; - v4.type = OLAP_FIELD_TYPE_DECIMAL; - v4.length = 12; - v4.aggregation = OLAP_FIELD_AGGREGATION_MIN; - v4.is_key = false; - v4.is_allow_null = true; + ColumnPB column_11; + column_11.set_unique_id(11); + column_11.set_name("column_11"); + column_11.set_type("DECIMAL"); + column_11.set_is_nullable(true); + column_11.set_length(12); + column_11.set_aggregation("MIN"); + column_11.set_is_key(false); + TabletColumn v4; + v4.init_from_pb(column_11); tablet_schema->push_back(v4); - FieldInfo v5; - v5.name = "v5"; - v5.type = OLAP_FIELD_TYPE_HLL; - v5.length = HLL_COLUMN_DEFAULT_LEN; - v5.aggregation = OLAP_FIELD_AGGREGATION_HLL_UNION; - v5.is_key = false; - v5.is_allow_null = true; + ColumnPB column_12; + column_12.set_unique_id(12); + column_12.set_name("column_12"); + column_12.set_type("HLL"); + column_12.set_is_nullable(true); + column_12.set_length(HLL_COLUMN_DEFAULT_LEN); + column_12.set_aggregation("HLL_UNION"); + column_12.set_is_key(false); + TabletColumn v5; + v5.init_from_pb(column_12); tablet_schema->push_back(v5); } -void set_tablet_schema_for_scan_key(std::vector* tablet_schema) { - FieldInfo k1; - k1.name = "k1"; - k1.type = OLAP_FIELD_TYPE_CHAR; - k1.length = 4; - k1.index_length = 4; - k1.default_value = "char"; - k1.is_key = true; - k1.is_allow_null = true; +void set_tablet_schema_for_scan_key(std::vector* tablet_schema) { + ColumnPB column_1; + column_1.set_unique_id(1); + column_1.set_name("column_1"); + column_1.set_type("CHAR"); + column_1.set_is_key(true); + column_1.set_is_nullable(true); + column_1.set_length(4); + column_1.set_index_length(4); + column_1.set_default_value("char"); + TabletColumn k1; + k1.init_from_pb(column_1); tablet_schema->push_back(k1); - FieldInfo k2; - k2.name = "k2"; - k2.type = OLAP_FIELD_TYPE_VARCHAR; - k2.length = 16 + OLAP_STRING_MAX_BYTES; - k2.index_length = 20; - k2.is_key = true; - k2.is_allow_null = true; + ColumnPB column_2; + column_2.set_unique_id(2); + column_2.set_name("column_2"); + column_2.set_type("VARCHAR"); + column_2.set_is_key(true); + column_2.set_is_nullable(true); + column_2.set_length(16 + OLAP_STRING_MAX_BYTES); + column_2.set_index_length(20); + TabletColumn k2; + k2.init_from_pb(column_2); tablet_schema->push_back(k2); - FieldInfo v1; - v1.name = "v1"; - v1.type = OLAP_FIELD_TYPE_LARGEINT; - v1.length = 16; - v1.aggregation = OLAP_FIELD_AGGREGATION_MAX; - v1.is_key = false; - v1.is_allow_null = true; + ColumnPB column_3; + column_3.set_unique_id(3); + column_3.set_name("column_3"); + column_3.set_type("LARGEINT"); + column_3.set_is_nullable(true); + column_3.set_length(16); + column_3.set_aggregation("MAX"); + column_3.set_is_key(false); + TabletColumn v1; + v1.init_from_pb(column_3); tablet_schema->push_back(v1); - FieldInfo v2; - v2.name = "v2"; - v2.type = OLAP_FIELD_TYPE_DECIMAL; - v2.length = 12; - v2.aggregation = OLAP_FIELD_AGGREGATION_MIN; - v2.is_key = false; - v2.is_allow_null = true; + ColumnPB column_4; + column_4.set_unique_id(9); + column_4.set_name("column_4"); + column_4.set_type("DECIMAL"); + column_4.set_is_nullable(true); + column_4.set_length(12); + column_4.set_aggregation("MIN"); + column_4.set_is_key(false); + TabletColumn v2; + v2.init_from_pb(column_4); tablet_schema->push_back(v2); } -void set_tablet_schema_for_cmp_and_aggregate(std::vector* tablet_schema) { - FieldInfo k1; - k1.name = "k1"; - k1.type = OLAP_FIELD_TYPE_CHAR; - k1.length = 4; - k1.default_value = "char"; - k1.is_key = true; - k1.index_length = 4; - k1.is_allow_null = true; +void set_tablet_schema_for_cmp_and_aggregate(std::vector* tablet_schema) { + ColumnPB column_1; + column_1.set_unique_id(1); + column_1.set_name("column_1"); + column_1.set_type("CHAR"); + column_1.set_is_key(true); + column_1.set_is_nullable(true); + column_1.set_length(4); + column_1.set_index_length(4); + column_1.set_default_value("char"); + TabletColumn k1; + k1.init_from_pb(column_1); tablet_schema->push_back(k1); - FieldInfo k2; - k2.name = "k2"; - k2.type = OLAP_FIELD_TYPE_INT; - k2.length = 4; - k2.is_key = true; - k2.index_length = 4; - k2.is_allow_null = true; + ColumnPB column_2; + column_2.set_unique_id(2); + column_2.set_name("column_2"); + column_2.set_type("INT"); + column_2.set_is_key(true); + column_2.set_is_nullable(true); + column_2.set_length(4); + column_2.set_index_length(4); + TabletColumn k2; + k2.init_from_pb(column_2); tablet_schema->push_back(k2); - FieldInfo v1; - v1.name = "v1"; - v1.type = OLAP_FIELD_TYPE_LARGEINT; - v1.length = 16; - v1.aggregation = OLAP_FIELD_AGGREGATION_SUM; - v1.is_key = false; - v1.is_allow_null = true; + ColumnPB column_3; + column_3.set_unique_id(3); + column_3.set_name("column_3"); + column_3.set_type("LARGEINT"); + column_3.set_is_nullable(true); + column_3.set_length(16); + column_3.set_aggregation("SUM"); + column_3.set_is_key(false); + TabletColumn v1; + v1.init_from_pb(column_3); tablet_schema->push_back(v1); - FieldInfo v2; - v2.name = "v2"; - v2.type = OLAP_FIELD_TYPE_DOUBLE; - v2.length = 8; - v2.aggregation = OLAP_FIELD_AGGREGATION_MIN; - v2.is_key = false; - v2.is_allow_null = true; + ColumnPB column_4; + column_4.set_unique_id(9); + column_4.set_name("column_4"); + column_4.set_type("DOUBLE"); + column_4.set_is_nullable(true); + column_4.set_length(8); + column_4.set_aggregation("MIN"); + column_4.set_is_key(false); + TabletColumn v2; + v2.init_from_pb(column_4); tablet_schema->push_back(v2); - FieldInfo v3; - v3.name = "v3"; - v3.type = OLAP_FIELD_TYPE_DECIMAL; - v3.length = 12; - v3.aggregation = OLAP_FIELD_AGGREGATION_MAX; - v3.is_key = false; - v3.is_allow_null = true; + ColumnPB column_5; + column_5.set_unique_id(3); + column_5.set_name("column_5"); + column_5.set_type("DECIMAL"); + column_5.set_is_nullable(true); + column_5.set_length(12); + column_5.set_aggregation("MAX"); + column_5.set_is_key(false); + TabletColumn v3; + v3.init_from_pb(column_5); tablet_schema->push_back(v3); - FieldInfo v4; - v4.name = "v4"; - v4.type = OLAP_FIELD_TYPE_VARCHAR; - v4.length = 16 + OLAP_STRING_MAX_BYTES; - v4.aggregation = OLAP_FIELD_AGGREGATION_REPLACE; - v4.is_key = false; - v4.is_allow_null = true; + ColumnPB column_6; + column_6.set_unique_id(9); + column_6.set_name("column_6"); + column_6.set_type("VARCHAR"); + column_6.set_is_nullable(true); + column_6.set_length(16 + OLAP_STRING_MAX_BYTES); + column_6.set_aggregation("REPLACE"); + column_6.set_is_key(false); + TabletColumn v4; + v4.init_from_pb(column_6); tablet_schema->push_back(v4); } @@ -250,7 +317,7 @@ class TestRowCursor : public testing::Test { }; TEST_F(TestRowCursor, InitRowCursor) { - std::vector tablet_schema; + std::vector tablet_schema; set_tablet_schema_for_init(&tablet_schema); RowCursor row; OLAPStatus res = row.init(tablet_schema); @@ -260,7 +327,7 @@ TEST_F(TestRowCursor, InitRowCursor) { } TEST_F(TestRowCursor, InitRowCursorWithColumnCount) { - std::vector tablet_schema; + std::vector tablet_schema; set_tablet_schema_for_init(&tablet_schema); RowCursor row; OLAPStatus res = row.init(tablet_schema, 5); @@ -272,7 +339,7 @@ TEST_F(TestRowCursor, InitRowCursorWithColumnCount) { } TEST_F(TestRowCursor, InitRowCursorWithColIds) { - std::vector tablet_schema; + std::vector tablet_schema; set_tablet_schema_for_init(&tablet_schema); std::vector col_ids; @@ -288,7 +355,7 @@ TEST_F(TestRowCursor, InitRowCursorWithColIds) { } TEST_F(TestRowCursor, InitRowCursorWithScanKey) { - std::vector tablet_schema; + std::vector tablet_schema; set_tablet_schema_for_scan_key(&tablet_schema); std::vector scan_keys; @@ -311,7 +378,7 @@ TEST_F(TestRowCursor, InitRowCursorWithScanKey) { } TEST_F(TestRowCursor, SetMinAndMaxKey) { - std::vector tablet_schema; + std::vector tablet_schema; set_tablet_schema_for_init(&tablet_schema); RowCursor min_row; @@ -332,7 +399,7 @@ TEST_F(TestRowCursor, SetMinAndMaxKey) { } TEST_F(TestRowCursor, EqualAndCompare) { - std::vector tablet_schema; + std::vector tablet_schema; set_tablet_schema_for_cmp_and_aggregate(&tablet_schema); RowCursor left; @@ -373,7 +440,7 @@ TEST_F(TestRowCursor, EqualAndCompare) { } TEST_F(TestRowCursor, IndexCmp) { - std::vector tablet_schema; + std::vector tablet_schema; set_tablet_schema_for_cmp_and_aggregate(&tablet_schema); RowCursor left; @@ -414,7 +481,7 @@ TEST_F(TestRowCursor, IndexCmp) { } TEST_F(TestRowCursor, FullKeyCmp) { - std::vector tablet_schema; + std::vector tablet_schema; set_tablet_schema_for_cmp_and_aggregate(&tablet_schema); RowCursor left; @@ -454,7 +521,7 @@ TEST_F(TestRowCursor, FullKeyCmp) { } TEST_F(TestRowCursor, AggregateWithoutNull) { - std::vector tablet_schema; + std::vector tablet_schema; set_tablet_schema_for_cmp_and_aggregate(&tablet_schema); RowCursor row; @@ -514,7 +581,7 @@ TEST_F(TestRowCursor, AggregateWithoutNull) { } TEST_F(TestRowCursor, AggregateWithNull) { - std::vector tablet_schema; + std::vector tablet_schema; set_tablet_schema_for_cmp_and_aggregate(&tablet_schema); RowCursor row; diff --git a/be/test/olap/rowset/rowset_meta_manager_test.cpp b/be/test/olap/rowset/rowset_meta_manager_test.cpp index 0d9d3345d241d4..cce0693cedf74a 100644 --- a/be/test/olap/rowset/rowset_meta_manager_test.cpp +++ b/be/test/olap/rowset/rowset_meta_manager_test.cpp @@ -75,27 +75,27 @@ TEST_F(RowsetMetaManagerTest, TestSaveAndGetAndRemove) { uint64_t rowset_id = 10000; RowsetMeta rowset_meta; rowset_meta.init_from_json(_json_rowset_meta); - ASSERT_EQ(rowset_meta.get_rowset_id(), rowset_id); - NewStatus status = RowsetMetaManager::save(_meta, rowset_id, &rowset_meta); - ASSERT_TRUE(status.ok()); + ASSERT_EQ(rowset_meta.rowset_id(), rowset_id); + OLAPStatus status = RowsetMetaManager::save(_meta, rowset_id, &rowset_meta); + ASSERT_TRUE(status == OLAP_SUCCESS); std::string json_rowset_meta_read; status = RowsetMetaManager::get_json_rowset_meta(_meta, rowset_id, &json_rowset_meta_read); - ASSERT_TRUE(status.ok()); + ASSERT_TRUE(status == OLAP_SUCCESS); ASSERT_EQ(_json_rowset_meta, json_rowset_meta_read); status = RowsetMetaManager::remove(_meta, rowset_id); - ASSERT_TRUE(status.ok()); - RowsetMeta rowset_meta_read; - status = RowsetMetaManager::get_rowset_meta(_meta, rowset_id, &rowset_meta_read); - ASSERT_TRUE(!status.ok()); + ASSERT_TRUE(status == OLAP_SUCCESS); + RowsetMetaSharedPtr rowset_meta_read(new RowsetMeta()); + status = RowsetMetaManager::get_rowset_meta(_meta, rowset_id, rowset_meta_read); + ASSERT_TRUE(status != OLAP_SUCCESS); } TEST_F(RowsetMetaManagerTest, TestLoad) { uint64_t rowset_id = 10000; - NewStatus status = RowsetMetaManager::load_json_rowset_meta(_meta, rowset_meta_path); - ASSERT_TRUE(status.ok()); + OLAPStatus status = RowsetMetaManager::load_json_rowset_meta(_meta, rowset_meta_path); + ASSERT_TRUE(status == OLAP_SUCCESS); std::string json_rowset_meta_read; status = RowsetMetaManager::get_json_rowset_meta(_meta, rowset_id, &json_rowset_meta_read); - ASSERT_TRUE(status.ok()); + ASSERT_TRUE(status == OLAP_SUCCESS); ASSERT_EQ(_json_rowset_meta, json_rowset_meta_read); } diff --git a/be/test/olap/run_length_byte_test.cpp b/be/test/olap/run_length_byte_test.cpp index 65db8a833b3aa9..d6a7535fcfa420 100755 --- a/be/test/olap/run_length_byte_test.cpp +++ b/be/test/olap/run_length_byte_test.cpp @@ -21,9 +21,9 @@ #include "olap/out_stream.h" #include "olap/in_stream.h" #include "olap/file_stream.h" -#include "olap/run_length_byte_writer.h" -#include "olap/run_length_byte_reader.h" -#include "olap/column_reader.h" +#include "olap/rowset/run_length_byte_writer.h" +#include "olap/rowset/run_length_byte_reader.h" +#include "olap/rowset/column_reader.h" #include "olap/stream_index_reader.h" #include "olap/stream_index_writer.h" #include "util/logging.h" diff --git a/be/test/olap/run_length_integer_test.cpp b/be/test/olap/run_length_integer_test.cpp index fafb4a035a74b6..ff27fdac9aef11 100755 --- a/be/test/olap/run_length_integer_test.cpp +++ b/be/test/olap/run_length_integer_test.cpp @@ -20,8 +20,8 @@ #include "olap/byte_buffer.h" #include "olap/out_stream.h" #include "olap/in_stream.h" -#include "olap/run_length_integer_writer.h" -#include "olap/run_length_integer_reader.h" +#include "olap/rowset/run_length_integer_writer.h" +#include "olap/rowset/run_length_integer_reader.h" #include "olap/stream_index_writer.h" #include "olap/stream_index_reader.h" #include "util/logging.h" diff --git a/be/test/olap/tablet_meta_manager_test.cpp b/be/test/olap/tablet_meta_manager_test.cpp index 4dc4d8def74483..28eef409fa18f3 100755 --- a/be/test/olap/tablet_meta_manager_test.cpp +++ b/be/test/olap/tablet_meta_manager_test.cpp @@ -45,9 +45,9 @@ class TabletMetaManagerTest : public testing::Test { virtual void SetUp() { std::string root_path = "./store"; ASSERT_TRUE(boost::filesystem::create_directory(root_path)); - _store = new(std::nothrow) DataDir(root_path); - ASSERT_NE(nullptr, _store); - Status st = _store->init(); + _data_dir = new(std::nothrow) DataDir(root_path); + ASSERT_NE(nullptr, _data_dir); + Status st = _data_dir->init(); ASSERT_TRUE(st.ok()); ASSERT_TRUE(boost::filesystem::exists("./store/meta")); @@ -59,55 +59,47 @@ class TabletMetaManagerTest : public testing::Test { } _json_header = _json_header.substr(0, _json_header.size() - 1); _json_header = _json_header.substr(0, _json_header.size() - 1); - std::cout << "set up finish" << std::endl; } virtual void TearDown() { - delete _store; + delete _data_dir; ASSERT_TRUE(boost::filesystem::remove_all("./store")); } private: - DataDir* _store; + DataDir* _data_dir; std::string _json_header; }; -TEST_F(TabletMetaManagerTest, TestConvertedFlag) { - bool converted_flag; - OLAPStatus s = TabletMetaManager::get_header_converted(_store, converted_flag); - ASSERT_EQ(false, converted_flag); - s = TabletMetaManager::set_converted_flag(_store); - ASSERT_EQ(OLAP_SUCCESS, s); - s = TabletMetaManager::get_header_converted(_store, converted_flag); - ASSERT_EQ(true, converted_flag); -} - TEST_F(TabletMetaManagerTest, TestSaveAndGetAndRemove) { - const TTabletId tablet_id = 20487; - const TSchemaHash schema_hash = 1520686811; - TabletMeta header; - bool ret = json2pb::JsonToProtoMessage(_json_header, &header); + const TTabletId tablet_id = 15672; + const TSchemaHash schema_hash = 567997577; + TabletMetaPB tablet_meta_pb; + bool ret = json2pb::JsonToProtoMessage(_json_header, &tablet_meta_pb); ASSERT_TRUE(ret); - OLAPStatus s = TabletMetaManager::save(_store, tablet_id, schema_hash, &header); + TabletMeta tablet_meta; + OLAPStatus s = tablet_meta.init_from_pb(tablet_meta_pb); + ASSERT_EQ(OLAP_SUCCESS, s); + s = TabletMetaManager::save(_data_dir, tablet_id, schema_hash, &tablet_meta); ASSERT_EQ(OLAP_SUCCESS, s); std::string json_header_read; - s = TabletMetaManager::get_json_header(_store, tablet_id, schema_hash, &json_header_read); + s = TabletMetaManager::get_json_header(_data_dir, tablet_id, schema_hash, &json_header_read); ASSERT_EQ(OLAP_SUCCESS, s); ASSERT_EQ(_json_header, json_header_read); - s = TabletMetaManager::remove(_store, tablet_id, schema_hash); + s = TabletMetaManager::remove(_data_dir, tablet_id, schema_hash); ASSERT_EQ(OLAP_SUCCESS, s); TabletMeta header_read; - s = TabletMetaManager::get_header(_store, tablet_id, schema_hash, &header_read); + s = TabletMetaManager::get_header(_data_dir, tablet_id, schema_hash, &header_read); ASSERT_EQ(OLAP_ERR_META_KEY_NOT_FOUND, s); } TEST_F(TabletMetaManagerTest, TestLoad) { - const TTabletId tablet_id = 20487; - const TSchemaHash schema_hash = 1520686811; - OLAPStatus s = TabletMetaManager::load_json_header(_store, header_path); + const TTabletId tablet_id = 15672; + const TSchemaHash schema_hash = 567997577; + OLAPStatus s = TabletMetaManager::load_json_header(_data_dir, header_path); ASSERT_EQ(OLAP_SUCCESS, s); std::string json_header_read; - s = TabletMetaManager::get_json_header(_store, tablet_id, schema_hash, &json_header_read); + s = TabletMetaManager::get_json_header(_data_dir, tablet_id, schema_hash, &json_header_read); ASSERT_EQ(OLAP_SUCCESS, s); ASSERT_EQ(_json_header, json_header_read); } diff --git a/be/test/olap/test_data/header.txt b/be/test/olap/test_data/header.txt index a657c2c40b12b2..b56aa5e3c0d5c0 100644 --- a/be/test/olap/test_data/header.txt +++ b/be/test/olap/test_data/header.txt @@ -1,152 +1,114 @@ { - "num_rows_per_data_block": 1024, - "file_version": [ + "table_id": 15670, + "partition_id": 15671, + "tablet_id": 15672, + "schema_hash": 567997577, + "shard_id": 34, + "creation_time": 1553765664, + "cumulative_layer_point": 2, + "schema": { + "keys_type": "AGG_KEYS", + "column": [ + { + "unique_id": 0, + "name": "k1", + "type": "BIGINT", + "is_key": true, + "aggregation": "NONE", + "is_nullable": false, + "length": 8, + "index_length": 8 + }, + { + "unique_id": 1, + "name": "v1", + "type": "HLL", + "is_key": false, + "aggregation": "HLL_UNION", + "is_nullable": false, + "default_value": "MA==", + "length": 16387, + "index_length": 16 + }, + { + "unique_id": 2, + "name": "v2", + "type": "INT", + "is_key": false, + "aggregation": "SUM", + "is_nullable": false, + "length": 4, + "index_length": 4 + } + ], + "num_short_key_columns": 1, + "num_rows_per_row_block": 1024, + "compress_kind": "COMPRESS_LZ4", + "next_column_unique_id": 3 + }, + "rs_metas": [ { - "num_segments": 1, + "rowset_id": 540072, + "tablet_id": 15673, + "tablet_schema_hash": 567997577, + "rowset_type": "ALPHA_ROWSET", + "rowset_state": "VISIBLE", "start_version": 0, "end_version": 1, "version_hash": 0, - "max_timestamp": 0, - "index_size": 67, - "data_size": 477, "num_rows": 0, - "creation_time": 1534750461, - "delta_pruning": { - "column_pruning": [ - { - "min": "OTk5OS0xMi0zMQ==", - "max": "MC0wMC0wMA==", - "null_flag": false - }, - { - "min": "MjE0NzQ4MzY0Nw==", - "max": "MA==", - "null_flag": false - }, - { - "min": "MzI3Njc=", - "max": "MA==", - "null_flag": false - }, - { - "min": "/w==", - "max": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "null_flag": false - } - ] - } + "total_disk_size": 0, + "data_disk_size": 0, + "index_disk_size": 0, + "empty": true, + "creation_time": 1553765664, + "extra_properties": "CgwIABAAGAAgACgAOAE=" }, { - "num_segments": 1, + "rowset_id": 540081, + "tablet_id": 15673, + "txn_id": 4042, + "tablet_schema_hash": 567997577, + "rowset_type": "ALPHA_ROWSET", + "rowset_state": "VISIBLE", "start_version": 2, "end_version": 2, - "version_hash": 0, - "max_timestamp": 0, - "index_size": 67, - "data_size": 477, - "num_rows": 0, - "creation_time": 1534750461, - "delta_pruning": { - "column_pruning": [ - { - "min": "OTk5OS0xMi0zMQ==", - "max": "MC0wMC0wMA==", - "null_flag": false - }, - { - "min": "MjE0NzQ4MzY0Nw==", - "max": "MA==", - "null_flag": false - }, - { - "min": "MzI3Njc=", - "max": "MA==", - "null_flag": false - }, - { - "min": "/w==", - "max": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "null_flag": false - } - ] - } + "version_hash": 8391828013814912580, + "num_rows": 3929, + "total_disk_size": 84699, + "data_disk_size": 84464, + "index_disk_size": 235, + "empty": false, + "load_id": { + "hi": -5350970832824939812, + "lo": -6717994719194512122 + }, + "creation_time": 1553765670, + "extra_properties": "Cj8IABABGIkBIMKRBCj3GDItChQtOTIyMTU1MjY3MjkzMDc2NDM3MBITOTIxMzI0NDQwNTY3MzIwODI3MhgAOAAKPggBEAEYYiCuggEo4gUyLQoULTkyMDQ4NjgyNzgzOTU5NTgyOTYSEzkyMjE4NjQ3ODUyMjk2NDE3MDYYADgA" } ], - "cumulative_layer_point": 2, - "num_short_key_fields": 4, - "column": [ + "inc_rs_metas": [ { - "name": "event_day", - "type": "DATE", - "aggregation": "NONE", - "length": 3, - "is_key": true, - "index_length": 3, - "is_allow_null": true, - "unique_id": 0, - "is_root_column": true - }, - { - "name": "siteid", - "type": "INT", - "aggregation": "NONE", - "length": 4, - "is_key": true, - "default_value": "10", - "index_length": 4, - "is_allow_null": true, - "unique_id": 1, - "is_root_column": true - }, - { - "name": "citycode", - "type": "SMALLINT", - "aggregation": "NONE", - "length": 2, - "is_key": true, - "index_length": 2, - "is_allow_null": true, - "unique_id": 2, - "is_root_column": true - }, - { - "name": "username", - "type": "VARCHAR", - "aggregation": "NONE", - "length": 34, - "is_key": true, - "default_value": "", - "index_length": 20, - "is_allow_null": true, - "unique_id": 3, - "is_root_column": true - }, - { - "name": "pv", - "type": "BIGINT", - "aggregation": "SUM", - "length": 8, - "is_key": false, - "default_value": "0", - "index_length": 8, - "is_allow_null": true, - "unique_id": 4, - "is_root_column": true + "rowset_id": 540081, + "tablet_id": 15673, + "txn_id": 4042, + "tablet_schema_hash": 567997577, + "rowset_type": "ALPHA_ROWSET", + "rowset_state": "VISIBLE", + "start_version": 2, + "end_version": 2, + "version_hash": 8391828013814912580, + "num_rows": 3929, + "total_disk_size": 84699, + "data_disk_size": 84464, + "index_disk_size": 235, + "empty": false, + "load_id": { + "hi": -5350970832824939812, + "lo": -6717994719194512122 + }, + "creation_time": 1553765670, + "extra_properties": "Cj8IABABGIkBIMKRBCj3GDItChQtOTIyMTU1MjY3MjkzMDc2NDM3MBITOTIxMzI0NDQwNTY3MzIwODI3MhgAOAAKPggBEAEYYiCuggEo4gUyLQoULTkyMDQ4NjgyNzgzOTU5NTgyOTYSEzkyMjE4NjQ3ODUyMjk2NDE3MDYYADgA" } - ], - "creation_time": 1534750461, - "selectivity": [ - 1, - 1, - 1, - 1 - ], - "data_file_type": "COLUMN_ORIENTED_FILE", - "next_column_unique_id": 5, - "compress_kind": "COMPRESS_LZ4", - "segment_size": 268435456, - "keys_type": "AGG_KEYS", - "tablet_id": 20487, - "schema_hash": 1520686811, - "shard": 0 + ] } diff --git a/be/test/olap/test_data/rowset_meta.json b/be/test/olap/test_data/rowset_meta.json index db962f2323cab7..866b4c8596e683 100644 --- a/be/test/olap/test_data/rowset_meta.json +++ b/be/test/olap/test_data/rowset_meta.json @@ -1,12 +1,17 @@ { "rowset_id": 10000, - "tablet_id": 20487, - "tablet_schema_hash": 1520686811, + "tablet_id": 12046, + "tablet_schema_hash": 365187263, "rowset_type": "ALPHA_ROWSET", "rowset_state": "VISIBLE", "start_version": 0, "end_version": 1, - "row_number": 123456, - "total_disk_size": 100000, - "data_disk_size": 95000 + "version_hash": 0, + "num_rows": 0, + "total_disk_size": 0, + "data_disk_size": 0, + "index_disk_size": 0, + "empty": true, + "creation_time": 1552911435, + "extra_properties": "CgwIABAAGAAgACgAOAE=" } diff --git a/be/test/util/doris_metrics_test.cpp b/be/test/util/doris_metrics_test.cpp index 1765ffaafa9b12..08631c818f8c20 100644 --- a/be/test/util/doris_metrics_test.cpp +++ b/be/test/util/doris_metrics_test.cpp @@ -227,14 +227,6 @@ TEST_F(DorisMetricsTest, Normal) { ASSERT_TRUE(metric != nullptr); ASSERT_STREQ("22", ((SimpleMetric*)metric)->to_string().c_str()); } - { - DorisMetrics::cancel_delete_requests_total.increment(23); - auto metric = metrics->get_metric("engine_requests_total", - MetricLabels().add("type", "cancel_delete") - .add("status", "total")); - ASSERT_TRUE(metric != nullptr); - ASSERT_STREQ("23", ((SimpleMetric*)metric)->to_string().c_str()); - } // comapction { DorisMetrics::base_compaction_deltas_total.increment(30); diff --git a/run-ut.sh b/run-ut.sh index cc594a22e59a6a..62f6106b097abd 100755 --- a/run-ut.sh +++ b/run-ut.sh @@ -151,10 +151,10 @@ ${DORIS_TEST_BINARY_DIR}/util/uid_util_test ${DORIS_TEST_BINARY_DIR}/util/aes_util_test ${DORIS_TEST_BINARY_DIR}/util/string_util_test -## Running common Unittest +# Running common Unittest ${DORIS_TEST_BINARY_DIR}/common/resource_tls_test -## Running exec unit test +# Running exec unit test ${DORIS_TEST_BINARY_DIR}/exec/plain_text_line_reader_uncompressed_test ${DORIS_TEST_BINARY_DIR}/exec/plain_text_line_reader_gzip_test ${DORIS_TEST_BINARY_DIR}/exec/plain_text_line_reader_bzip_test @@ -165,7 +165,7 @@ ${DORIS_TEST_BINARY_DIR}/exec/broker_scan_node_test ${DORIS_TEST_BINARY_DIR}/exec/tablet_info_test ${DORIS_TEST_BINARY_DIR}/exec/tablet_sink_test -## Running runtime Unittest +# Running runtime Unittest ${DORIS_TEST_BINARY_DIR}/runtime/fragment_mgr_test ${DORIS_TEST_BINARY_DIR}/runtime/decimal_value_test ${DORIS_TEST_BINARY_DIR}/runtime/datetime_value_test @@ -177,7 +177,6 @@ ${DORIS_TEST_BINARY_DIR}/runtime/stream_load_pipe_test ${DORIS_TEST_BINARY_DIR}/runtime/tablet_writer_mgr_test ${DORIS_TEST_BINARY_DIR}/runtime/snapshot_loader_test ${DORIS_TEST_BINARY_DIR}/runtime/user_function_cache_test -## Running expr Unittest # Running http ${DORIS_TEST_BINARY_DIR}/http/metrics_action_test @@ -200,7 +199,7 @@ ${DORIS_TEST_BINARY_DIR}/olap/in_list_predicate_test ${DORIS_TEST_BINARY_DIR}/olap/null_predicate_test ${DORIS_TEST_BINARY_DIR}/olap/file_helper_test ${DORIS_TEST_BINARY_DIR}/olap/file_utils_test -#${DORIS_TEST_BINARY_DIR}/olap/delete_handler_test +${DORIS_TEST_BINARY_DIR}/olap/delete_handler_test ${DORIS_TEST_BINARY_DIR}/olap/column_reader_test ${DORIS_TEST_BINARY_DIR}/olap/row_cursor_test ${DORIS_TEST_BINARY_DIR}/olap/skiplist_test From c46add0c9a7d280eee66d6eea7d0a9786d05d999 Mon Sep 17 00:00:00 2001 From: huangkangping Date: Mon, 8 Apr 2019 17:05:41 +0800 Subject: [PATCH 2/2] use TabletSchema to replace TabletColumn vector --- be/src/olap/row_block.cpp | 29 +- be/src/olap/row_block.h | 4 - be/src/olap/row_cursor.cpp | 33 +- be/src/olap/row_cursor.h | 12 - be/src/olap/rowset/column_data.cpp | 6 +- be/src/olap/rowset/column_writer.cpp | 4 +- be/src/olap/rowset/column_writer.h | 2 +- be/src/olap/rowset/segment_writer.cpp | 2 +- be/src/olap/tablet_meta.cpp | 3 + be/src/runtime/vectorized_row_batch.cpp | 6 +- be/src/runtime/vectorized_row_batch.h | 4 +- be/test/olap/column_reader_test.cpp | 213 +++++---- be/test/olap/comparison_predicate_test.cpp | 202 ++++---- be/test/olap/in_list_predicate_test.cpp | 116 +++-- be/test/olap/null_predicate_test.cpp | 103 ++--- be/test/olap/row_cursor_test.cpp | 509 +++++++++------------ 16 files changed, 547 insertions(+), 701 deletions(-) diff --git a/be/src/olap/row_block.cpp b/be/src/olap/row_block.cpp index 0a4755bac9aed6..f0c381fa80c456 100644 --- a/be/src/olap/row_block.cpp +++ b/be/src/olap/row_block.cpp @@ -38,33 +38,18 @@ namespace doris { RowBlock::RowBlock(const TabletSchema* schema) : _capacity(0), - _schema(schema), - _columns(nullptr) { - if (_schema != nullptr) { - _columns = &_schema->columns(); - } + _schema(schema) { _tracker.reset(new MemTracker(-1)); _mem_pool.reset(new MemPool(_tracker.get())); } -RowBlock::RowBlock(const std::vector& columns) : - _capacity(0), - _schema(nullptr), - _columns(&columns) { - _tracker.reset(new MemTracker(-1)); - _mem_pool.reset(new MemPool(_tracker.get())); -} - RowBlock::~RowBlock() { delete[] _mem_buf; delete[] _storage_buf; } OLAPStatus RowBlock::init(const RowBlockInfo& block_info) { - if (_columns == nullptr) { - return OLAP_ERR_INIT_FAILED; - } - _field_count = _columns->size(); + _field_count = _schema->num_columns(); _info = block_info; _null_supported = block_info.null_supported; _capacity = _info.row_num; @@ -129,7 +114,7 @@ void RowBlock::_convert_storage_to_memory() { // some data file in history not suppored null size_t null_byte = has_nullbyte() ? 1 : 0; for (int col = 0; col < _field_count; ++col) { - const TabletColumn& column = _columns->at(col); + const TabletColumn& column = _schema->column(col); char* memory_ptr = _mem_buf + _field_offset_in_memory[col]; if (column.type() == OLAP_FIELD_TYPE_VARCHAR || column.type() == OLAP_FIELD_TYPE_HLL) { @@ -207,7 +192,7 @@ void RowBlock::_convert_memory_to_storage(uint32_t num_rows) { char* storage_variable_ptr = _storage_buf + num_rows * _storage_row_fixed_bytes; size_t null_byte = has_nullbyte() ? 1 : 0; for (int col = 0; col < _field_count; ++col) { - const TabletColumn& column = _columns->at(col);; + const TabletColumn& column = _schema->column(col); char* memory_ptr = _mem_buf + _field_offset_in_memory[col]; if (column.type() == OLAP_FIELD_TYPE_VARCHAR || column.type() == OLAP_FIELD_TYPE_HLL) { @@ -296,7 +281,7 @@ OLAPStatus RowBlock::find_row(const RowCursor& key, OLAPStatus res = OLAP_SUCCESS; RowCursor helper_cursor; - if ((res = helper_cursor.init(*_columns)) != OLAP_SUCCESS) { + if ((res = helper_cursor.init(*_schema)) != OLAP_SUCCESS) { OLAP_LOG_WARNING("Init helper cursor fail. [res=%d]", res); return OLAP_ERR_INIT_FAILED; } @@ -335,8 +320,8 @@ void RowBlock::_compute_layout() { size_t memory_size = 0; size_t storage_fixed_bytes = 0; size_t storage_variable_bytes = 0; - for (size_t col_id = 0; col_id < _columns->size(); ++col_id) { - const TabletColumn& column = _columns->at(col_id); + for (size_t col_id = 0; col_id < _schema->num_columns(); ++col_id) { + const TabletColumn& column = _schema->column(col_id); _field_offset_in_memory.push_back(memory_size); // All column has a nullbyte in memory diff --git a/be/src/olap/row_block.h b/be/src/olap/row_block.h index 21f4f6ee33f140..f33916088f595c 100644 --- a/be/src/olap/row_block.h +++ b/be/src/olap/row_block.h @@ -61,9 +61,6 @@ class RowBlock { public: RowBlock(const TabletSchema* schema); - // this constructor is just for unit test - RowBlock(const std::vector& columns); - // 注意回收内部buffer ~RowBlock(); @@ -189,7 +186,6 @@ class RowBlock { uint32_t _capacity; RowBlockInfo _info; const TabletSchema* _schema; // 内部保存的schema句柄 - const std::vector* _columns; bool _null_supported; diff --git a/be/src/olap/row_cursor.cpp b/be/src/olap/row_cursor.cpp index 94f72887c2125d..e590e45d30717f 100644 --- a/be/src/olap/row_cursor.cpp +++ b/be/src/olap/row_cursor.cpp @@ -162,24 +162,13 @@ OLAPStatus RowCursor::init(const TabletSchema& schema, return OLAP_SUCCESS; } -OLAPStatus RowCursor::init(const std::vector& schema, - const vector& columns) { - RETURN_NOT_OK(_init(schema, columns)); - return OLAP_SUCCESS; -} - OLAPStatus RowCursor::init_scan_key(const TabletSchema& schema, const std::vector& scan_keys) { - return init_scan_key(schema.columns(), scan_keys); -} - -OLAPStatus RowCursor::init_scan_key(const std::vector& schema, - const std::vector& scan_keys) { size_t scan_key_size = scan_keys.size(); - if (scan_key_size > schema.size()) { + if (scan_key_size > schema.num_columns()) { LOG(WARNING) << "Input param are invalid. Column count is bigger than num_columns of schema. " << "column_count=" << scan_key_size - << ", schema.num_columns=" << schema.size(); + << ", schema.num_columns=" << schema.num_columns(); return OLAP_ERR_INPUT_PARAMETER_ERROR; } @@ -188,13 +177,13 @@ OLAPStatus RowCursor::init_scan_key(const std::vector& schema, columns.push_back(i); } - RETURN_NOT_OK(_init(schema, columns)); + RETURN_NOT_OK(_init(schema.columns(), columns)); // NOTE: cid equal with column index // Hyperloglog cannot be key, no need to handle it _variable_len = 0; for (auto cid : _columns) { - const TabletColumn& column = schema[cid]; + const TabletColumn& column = schema.column(cid); FieldType type = column.type(); if (type == OLAP_FIELD_TYPE_VARCHAR) { _variable_len += scan_keys[cid].length(); @@ -205,7 +194,7 @@ OLAPStatus RowCursor::init_scan_key(const std::vector& schema, } // variable_len for null bytes - _variable_buf = new (nothrow) char[_variable_len]; + _variable_buf = new(nothrow) char[_variable_len]; if (_variable_buf == nullptr) { OLAP_LOG_WARNING("Fail to malloc _variable_buf."); return OLAP_ERR_MALLOC_ERROR; @@ -214,7 +203,7 @@ OLAPStatus RowCursor::init_scan_key(const std::vector& schema, char* fixed_ptr = _fixed_buf; char* variable_ptr = _variable_buf; for (auto cid : _columns) { - const TabletColumn& column = schema[cid]; + const TabletColumn& column = schema.column(cid); fixed_ptr = _fixed_buf + _field_array[cid]->get_offset(); FieldType type = column.type(); if (type == OLAP_FIELD_TYPE_VARCHAR) { @@ -234,13 +223,7 @@ OLAPStatus RowCursor::init_scan_key(const std::vector& schema, } OLAPStatus RowCursor::allocate_memory_for_string_type( - const TabletSchema& schema, - MemPool* mem_pool) { - return allocate_memory_for_string_type(schema.columns(), mem_pool); -} - -OLAPStatus RowCursor::allocate_memory_for_string_type( - const std::vector& schema, + const TabletSchema& schema, MemPool* mem_pool) { // allocate memory for string type(char, varchar, hll) // The memory allocated in this function is used in aggregate and copy function @@ -264,7 +247,7 @@ OLAPStatus RowCursor::allocate_memory_for_string_type( char* fixed_ptr = _fixed_buf; char* variable_ptr = _variable_buf; for (auto cid : _columns) { - const TabletColumn& column = schema[cid]; + const TabletColumn& column = schema.column(cid); fixed_ptr = _fixed_buf + _field_array[cid]->get_offset(); FieldType type = column.type(); if (type == OLAP_FIELD_TYPE_VARCHAR) { diff --git a/be/src/olap/row_cursor.h b/be/src/olap/row_cursor.h index cf58a513a96b21..3066a4213988f4 100644 --- a/be/src/olap/row_cursor.h +++ b/be/src/olap/row_cursor.h @@ -73,26 +73,14 @@ class RowCursor { OLAPStatus init(const TabletSchema& schema, const std::vector& columns); - OLAPStatus init(const std::vector& schema, - const std::vector& columns); - // 用传入的key的size来初始化 // 目前仅用在拆分key区间的时候 OLAPStatus init_scan_key(const TabletSchema& schema, const std::vector& keys); - OLAPStatus init_scan_key(const std::vector& schema, - const std::vector& keys); - - OLAPStatus init_scan_key(const TabletSchema& schema, - const std::vector& field_lengths); - //allocate memory for string type, which include char, varchar, hyperloglog OLAPStatus allocate_memory_for_string_type(const TabletSchema& schema, MemPool* mem_pool = nullptr); - - OLAPStatus allocate_memory_for_string_type(const std::vector& schema, - MemPool* mem_pool = nullptr); // 两个RowCurosr做比较，返回-1，0，1 int cmp(const RowCursor& other) const; diff --git a/be/src/olap/rowset/column_data.cpp b/be/src/olap/rowset/column_data.cpp index c71c1383a152b5..302d879cd3ec5d 100644 --- a/be/src/olap/rowset/column_data.cpp +++ b/be/src/olap/rowset/column_data.cpp @@ -453,10 +453,10 @@ void ColumnData::set_read_params( } _read_vector_batch.reset(new VectorizedRowBatch( - &(_segment_group->get_tablet_schema().columns()), _return_columns, _num_rows_per_block)); + &(_segment_group->get_tablet_schema()), _return_columns, _num_rows_per_block)); _seek_vector_batch.reset(new VectorizedRowBatch( - &(_segment_group->get_tablet_schema().columns()), _seek_columns, _num_rows_per_block)); + &(_segment_group->get_tablet_schema()), _seek_columns, _num_rows_per_block)); _read_block.reset(new RowBlock(&(_segment_group->get_tablet_schema()))); RowBlockInfo block_info; @@ -602,7 +602,7 @@ OLAPStatus ColumnData::schema_change_init() { } _read_vector_batch.reset(new VectorizedRowBatch( - &(_segment_group->get_tablet_schema().columns()), _return_columns, _num_rows_per_block)); + &(_segment_group->get_tablet_schema()), _return_columns, _num_rows_per_block)); _read_block.reset(new RowBlock(&(_segment_group->get_tablet_schema()))); diff --git a/be/src/olap/rowset/column_writer.cpp b/be/src/olap/rowset/column_writer.cpp index 0f19901b6cc93a..b8fa9ef2b00614 100755 --- a/be/src/olap/rowset/column_writer.cpp +++ b/be/src/olap/rowset/column_writer.cpp @@ -23,12 +23,12 @@ namespace doris { ColumnWriter* ColumnWriter::create(uint32_t column_id, - const std::vector& schema, + const TabletSchema& schema, OutStreamFactory* stream_factory, size_t num_rows_per_row_block, double bf_fpp) { ColumnWriter* column_writer = nullptr; - const TabletColumn& column = schema[column_id]; + const TabletColumn& column = schema.column(column_id); switch (column.type()) { case OLAP_FIELD_TYPE_TINYINT: diff --git a/be/src/olap/rowset/column_writer.h b/be/src/olap/rowset/column_writer.h index 3725011fbbc55c..fa3fb77b922830 100644 --- a/be/src/olap/rowset/column_writer.h +++ b/be/src/olap/rowset/column_writer.h @@ -53,7 +53,7 @@ class ColumnWriter { // stream_factory: 用于创建输出流的工厂对象, 该对象的生命期由调用者所有 static ColumnWriter* create( uint32_t column_id, - const std::vector& schema, + const TabletSchema& schema, OutStreamFactory* stream_factory, size_t num_rows_per_row_block, double bf_fpp); diff --git a/be/src/olap/rowset/segment_writer.cpp b/be/src/olap/rowset/segment_writer.cpp index 7ef23546444a22..09a77ecd367c19 100644 --- a/be/src/olap/rowset/segment_writer.cpp +++ b/be/src/olap/rowset/segment_writer.cpp @@ -63,7 +63,7 @@ OLAPStatus SegmentWriter::init(uint32_t write_mbytes_per_sec) { // 创建writer for (uint32_t i = 0; i < _segment_group->get_tablet_schema().num_columns(); i++) { - ColumnWriter* writer = ColumnWriter::create(i, _segment_group->get_tablet_schema().columns(), + ColumnWriter* writer = ColumnWriter::create(i, _segment_group->get_tablet_schema(), _stream_factory, _segment_group->get_num_rows_per_row_block(), _bloom_filter_fpp); diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index a322ebf2263b8c..02502c96107f0e 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -270,6 +270,9 @@ OLAPStatus TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) { for (auto& it : tablet_meta_pb.inc_rs_metas()) { RowsetMetaSharedPtr rs_meta(new AlphaRowsetMeta()); rs_meta->init_from_pb(it); + if (rs_meta->has_delete_predicate()) { + add_delete_predicate(rs_meta->delete_predicate(), rs_meta->version().first); + } _inc_rs_metas.push_back(std::move(rs_meta)); } diff --git a/be/src/runtime/vectorized_row_batch.cpp b/be/src/runtime/vectorized_row_batch.cpp index 07789da476bc51..d082c083e0579a 100644 --- a/be/src/runtime/vectorized_row_batch.cpp +++ b/be/src/runtime/vectorized_row_batch.cpp @@ -23,7 +23,7 @@ namespace doris { VectorizedRowBatch::VectorizedRowBatch( - const std::vector* schema, + const TabletSchema* schema, const std::vector& cols, int capacity) : _schema(schema), _cols(cols), _capacity(capacity), _limit(capacity) { @@ -49,7 +49,7 @@ void VectorizedRowBatch::dump_to_row_block(RowBlock* row_block) { // pointer of this field in row block char* row_field_ptr = row_block->_mem_buf + row_block->_field_offset_in_memory[column_id]; - const TabletColumn& column = _schema->at(column_id); + const TabletColumn& column = _schema->column(column_id); size_t field_size = 0; if (column.type() == OLAP_FIELD_TYPE_CHAR || column.type() == OLAP_FIELD_TYPE_VARCHAR || @@ -93,7 +93,7 @@ void VectorizedRowBatch::dump_to_row_block(RowBlock* row_block) { char* row_field_ptr = row_block->_mem_buf + row_block->_field_offset_in_memory[column_id]; - const TabletColumn& column = _schema->at(column_id); + const TabletColumn& column = _schema->column(column_id); size_t field_size = 0; if (column.type() == OLAP_FIELD_TYPE_CHAR || column.type() == OLAP_FIELD_TYPE_VARCHAR || diff --git a/be/src/runtime/vectorized_row_batch.h b/be/src/runtime/vectorized_row_batch.h index a2c29100984cc6..4960d35b0e0a94 100644 --- a/be/src/runtime/vectorized_row_batch.h +++ b/be/src/runtime/vectorized_row_batch.h @@ -73,7 +73,7 @@ class ColumnVector { class VectorizedRowBatch { public: VectorizedRowBatch( - const std::vector* schema, + const TabletSchema* schema, const std::vector& cols, int capacity); @@ -139,7 +139,7 @@ class VectorizedRowBatch { void dump_to_row_block(RowBlock* row_block); private: - const std::vector* _schema; + const TabletSchema* _schema; const std::vector& _cols; const uint16_t _capacity; uint16_t _size = 0; diff --git a/be/test/olap/column_reader_test.cpp b/be/test/olap/column_reader_test.cpp index 78a9530f2a16d6..500b7d8bd1c0ef 100644 --- a/be/test/olap/column_reader_test.cpp +++ b/be/test/olap/column_reader_test.cpp @@ -99,7 +99,7 @@ class TestColumn : public testing::Test { _length_buffers.clear(); } - void CreateColumnWriter(const std::vector& tablet_schema) { + void CreateColumnWriter(const TabletSchema& tablet_schema) { _column_writer = ColumnWriter::create( 0, tablet_schema, _stream_factory, 1024, BLOOM_FILTER_DEFAULT_FPP); @@ -107,7 +107,7 @@ class TestColumn : public testing::Test { ASSERT_EQ(_column_writer->init(), OLAP_SUCCESS); } - void CreateColumnReader(const std::vector &tablet_schema) { + void CreateColumnReader(const TabletSchema& tablet_schema) { UniqueIdEncodingMap encodings; encodings[0] = ColumnEncodingMessage(); encodings[0].set_kind(ColumnEncodingMessage::DIRECT); @@ -116,7 +116,7 @@ class TestColumn : public testing::Test { } void CreateColumnReader( - const std::vector &tablet_schema, + const TabletSchema& tablet_schema, UniqueIdEncodingMap &encodings) { UniqueIdToColumnIdMap included; included[0] = 0; @@ -211,21 +211,20 @@ class TestColumn : public testing::Test { std::string aggregation, uint32_t length, bool is_allow_null, - bool is_key, std::vector* tablet_schema) { - ColumnPB column; - column.set_unique_id(0); - column.set_name(name); - column.set_type(type); - column.set_is_key(is_key); - column.set_is_nullable(is_allow_null); - column.set_length(length); - column.set_aggregation(aggregation); - column.set_precision(1000); - column.set_frac(1000); - column.set_is_bf_column(false); - TabletColumn tablet_column; - tablet_column.init_from_pb(column); - tablet_schema->push_back(tablet_column); + bool is_key, TabletSchema* tablet_schema) { + TabletSchemaPB tablet_schema_pb; + ColumnPB* column = tablet_schema_pb.add_column(); + column->set_unique_id(0); + column->set_name(name); + column->set_type(type); + column->set_is_key(is_key); + column->set_is_nullable(is_allow_null); + column->set_length(length); + column->set_aggregation(aggregation); + column->set_precision(1000); + column->set_frac(1000); + column->set_is_bf_column(false); + tablet_schema->init_from_pb(tablet_schema_pb); } void create_and_save_last_position() { @@ -264,9 +263,9 @@ class TestColumn : public testing::Test { TEST_F(TestColumn, VectorizedTinyColumnWithoutPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( - "TinyColumn", + "TinyColumn", "TINYINT", "REPLACE", 1, @@ -277,7 +276,7 @@ TEST_F(TestColumn, VectorizedTinyColumnWithoutPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -317,7 +316,7 @@ TEST_F(TestColumn, VectorizedTinyColumnWithoutPresent) { TEST_F(TestColumn, SeekTinyColumnWithoutPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "TinyColumn", "TINYINT", @@ -332,7 +331,7 @@ TEST_F(TestColumn, SeekTinyColumnWithoutPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -403,7 +402,7 @@ TEST_F(TestColumn, SeekTinyColumnWithoutPresent) { TEST_F(TestColumn, SkipTinyColumnWithoutPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "TinyColumn", "TINYINT", @@ -418,7 +417,7 @@ TEST_F(TestColumn, SkipTinyColumnWithoutPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -458,7 +457,7 @@ TEST_F(TestColumn, SkipTinyColumnWithoutPresent) { TEST_F(TestColumn, VectorizedTinyColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "TinyColumn", "TINYINT", @@ -472,7 +471,7 @@ TEST_F(TestColumn, VectorizedTinyColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -510,7 +509,7 @@ TEST_F(TestColumn, VectorizedTinyColumnWithPresent) { TEST_F(TestColumn, TinyColumnIndex) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "TinyColumn", "TINYINT", @@ -524,7 +523,7 @@ TEST_F(TestColumn, TinyColumnIndex) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -562,7 +561,7 @@ TEST_F(TestColumn, TinyColumnIndex) { TEST_F(TestColumn, SeekTinyColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "TinyColumn", "TINYINT", @@ -576,7 +575,7 @@ TEST_F(TestColumn, SeekTinyColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -646,7 +645,7 @@ TEST_F(TestColumn, SeekTinyColumnWithPresent) { TEST_F(TestColumn, SkipTinyColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "TinyColumn", "TINYINT", @@ -659,7 +658,7 @@ TEST_F(TestColumn, SkipTinyColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -699,7 +698,7 @@ TEST_F(TestColumn, SkipTinyColumnWithPresent) { TEST_F(TestColumn, VectorizedShortColumnWithoutPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "ShortColumn", "SMALLINT", @@ -712,7 +711,7 @@ TEST_F(TestColumn, VectorizedShortColumnWithoutPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -750,7 +749,7 @@ TEST_F(TestColumn, VectorizedShortColumnWithoutPresent) { TEST_F(TestColumn, SeekShortColumnWithoutPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "ShortColumn", "SMALLINT", @@ -763,7 +762,7 @@ TEST_F(TestColumn, SeekShortColumnWithoutPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -832,7 +831,7 @@ TEST_F(TestColumn, SeekShortColumnWithoutPresent) { TEST_F(TestColumn, SkipShortColumnWithoutPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "ShortColumn", "SMALLINT", @@ -845,7 +844,7 @@ TEST_F(TestColumn, SkipShortColumnWithoutPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -885,7 +884,7 @@ TEST_F(TestColumn, SkipShortColumnWithoutPresent) { TEST_F(TestColumn, SeekShortColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "ShortColumn", "SMALLINT", @@ -898,7 +897,7 @@ TEST_F(TestColumn, SeekShortColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -964,7 +963,7 @@ TEST_F(TestColumn, SeekShortColumnWithPresent) { TEST_F(TestColumn, VectorizedShortColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "ShortColumn", @@ -978,7 +977,7 @@ TEST_F(TestColumn, VectorizedShortColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -1018,7 +1017,7 @@ TEST_F(TestColumn, VectorizedShortColumnWithPresent) { TEST_F(TestColumn, SkipShortColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "ShortColumn", "SMALLINT", @@ -1031,7 +1030,7 @@ TEST_F(TestColumn, SkipShortColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -1070,7 +1069,7 @@ TEST_F(TestColumn, SkipShortColumnWithPresent) { TEST_F(TestColumn, VectorizedIntColumnWithoutPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "IntColumn", "INT", @@ -1083,7 +1082,7 @@ TEST_F(TestColumn, VectorizedIntColumnWithoutPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 1024; block.init(block_info); @@ -1121,7 +1120,7 @@ TEST_F(TestColumn, VectorizedIntColumnWithoutPresent) { TEST_F(TestColumn, VectorizedIntColumnMassWithoutPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "IntColumn", "INT", @@ -1134,7 +1133,7 @@ TEST_F(TestColumn, VectorizedIntColumnMassWithoutPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1174,7 +1173,7 @@ TEST_F(TestColumn, VectorizedIntColumnMassWithoutPresent) { TEST_F(TestColumn, VectorizedIntColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "IntColumn", "INT", @@ -1187,7 +1186,7 @@ TEST_F(TestColumn, VectorizedIntColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1228,7 +1227,7 @@ TEST_F(TestColumn, VectorizedIntColumnWithPresent) { TEST_F(TestColumn, VectorizedLongColumnWithoutPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "LongColumnWithoutPresent", "BIGINT", @@ -1241,7 +1240,7 @@ TEST_F(TestColumn, VectorizedLongColumnWithoutPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1280,7 +1279,7 @@ TEST_F(TestColumn, VectorizedLongColumnWithoutPresent) { TEST_F(TestColumn, VectorizedLongColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "LongColumnWithPresent", "BIGINT", @@ -1293,7 +1292,7 @@ TEST_F(TestColumn, VectorizedLongColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1334,7 +1333,7 @@ TEST_F(TestColumn, VectorizedLongColumnWithPresent) { TEST_F(TestColumn, VectorizedFloatColumnWithoutPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "FloatColumnWithoutPresent", "FLOAT", @@ -1347,7 +1346,7 @@ TEST_F(TestColumn, VectorizedFloatColumnWithoutPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1387,7 +1386,7 @@ TEST_F(TestColumn, VectorizedFloatColumnWithoutPresent) { TEST_F(TestColumn, VectorizedFloatColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "FloatColumnWithPresent", @@ -1401,7 +1400,7 @@ TEST_F(TestColumn, VectorizedFloatColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1442,7 +1441,7 @@ TEST_F(TestColumn, VectorizedFloatColumnWithPresent) { TEST_F(TestColumn, SeekFloatColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "FloatColumnWithPresent", @@ -1456,7 +1455,7 @@ TEST_F(TestColumn, SeekFloatColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1513,7 +1512,7 @@ TEST_F(TestColumn, SeekFloatColumnWithPresent) { TEST_F(TestColumn, SkipFloatColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "FloatColumnWithPresent", @@ -1527,7 +1526,7 @@ TEST_F(TestColumn, SkipFloatColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1564,7 +1563,7 @@ TEST_F(TestColumn, SkipFloatColumnWithPresent) { TEST_F(TestColumn, VectorizedDoubleColumnWithoutPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "DoubleColumnWithoutPresent", @@ -1578,7 +1577,7 @@ TEST_F(TestColumn, VectorizedDoubleColumnWithoutPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1618,7 +1617,7 @@ TEST_F(TestColumn, VectorizedDoubleColumnWithoutPresent) { TEST_F(TestColumn, VectorizedDoubleColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "DoubleColumnWithPresent", @@ -1632,7 +1631,7 @@ TEST_F(TestColumn, VectorizedDoubleColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1674,7 +1673,7 @@ TEST_F(TestColumn, VectorizedDoubleColumnWithPresent) { TEST_F(TestColumn, VectorizedDatetimeColumnWithoutPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "DatetimeColumnWithoutPresent", @@ -1688,7 +1687,7 @@ TEST_F(TestColumn, VectorizedDatetimeColumnWithoutPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1721,7 +1720,7 @@ TEST_F(TestColumn, VectorizedDatetimeColumnWithoutPresent) { TEST_F(TestColumn, VectorizedDatetimeColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "DatetimeColumnWithoutPresent", @@ -1735,7 +1734,7 @@ TEST_F(TestColumn, VectorizedDatetimeColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1783,7 +1782,7 @@ TEST_F(TestColumn, VectorizedDatetimeColumnWithPresent) { TEST_F(TestColumn, VectorizedDateColumnWithoutPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "DateColumnWithoutoutPresent", @@ -1797,7 +1796,7 @@ TEST_F(TestColumn, VectorizedDateColumnWithoutPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1829,7 +1828,7 @@ TEST_F(TestColumn, VectorizedDateColumnWithoutPresent) { TEST_F(TestColumn, VectorizedDateColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "DateColumnWithoutoutPresent", @@ -1843,7 +1842,7 @@ TEST_F(TestColumn, VectorizedDateColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1891,7 +1890,7 @@ TEST_F(TestColumn, VectorizedDateColumnWithPresent) { TEST_F(TestColumn, VectorizedDecimalColumnWithoutPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "DecimalColumnWithoutoutPresent", @@ -1905,7 +1904,7 @@ TEST_F(TestColumn, VectorizedDecimalColumnWithoutPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -1950,7 +1949,7 @@ TEST_F(TestColumn, VectorizedDecimalColumnWithoutPresent) { TEST_F(TestColumn, VectorizedDecimalColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "DecimalColumnWithoutoutPresent", @@ -1964,7 +1963,7 @@ TEST_F(TestColumn, VectorizedDecimalColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2009,7 +2008,7 @@ TEST_F(TestColumn, VectorizedDecimalColumnWithPresent) { TEST_F(TestColumn, SkipDecimalColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "DecimalColumnWithPresent", @@ -2023,7 +2022,7 @@ TEST_F(TestColumn, SkipDecimalColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2064,7 +2063,7 @@ TEST_F(TestColumn, SkipDecimalColumnWithPresent) { TEST_F(TestColumn, SeekDecimalColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "DecimalColumnWithPresent", "DECIMAL", @@ -2077,7 +2076,7 @@ TEST_F(TestColumn, SeekDecimalColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2145,7 +2144,7 @@ TEST_F(TestColumn, SeekDecimalColumnWithPresent) { TEST_F(TestColumn, VectorizedLargeIntColumnWithoutPresent) { // init tablet schema - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "LargeIntColumnWithoutoutPresent", @@ -2163,7 +2162,7 @@ TEST_F(TestColumn, VectorizedLargeIntColumnWithoutPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2207,7 +2206,7 @@ TEST_F(TestColumn, VectorizedLargeIntColumnWithoutPresent) { TEST_F(TestColumn, VectorizedLargeIntColumnWithPresent) { // init tablet schema - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "LargeIntColumnWithoutoutPresent", @@ -2227,7 +2226,7 @@ TEST_F(TestColumn, VectorizedLargeIntColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2286,7 +2285,7 @@ TEST_F(TestColumn, VectorizedLargeIntColumnWithPresent) { TEST_F(TestColumn, SkipLargeIntColumnWithPresent) { // init tablet schema - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "LargeIntColumnWithPresent", @@ -2304,7 +2303,7 @@ TEST_F(TestColumn, SkipLargeIntColumnWithPresent) { RowCursor write_row; write_row.init(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2345,7 +2344,7 @@ TEST_F(TestColumn, SkipLargeIntColumnWithPresent) { TEST_F(TestColumn, VectorizedDirectVarcharColumnWithoutPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "DirectVarcharColumnWithoutoutPresent", @@ -2361,7 +2360,7 @@ TEST_F(TestColumn, VectorizedDirectVarcharColumnWithoutPresent) { write_row.init(tablet_schema); write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2418,7 +2417,7 @@ TEST_F(TestColumn, VectorizedDirectVarcharColumnWithoutPresent) { TEST_F(TestColumn, VectorizedDirectVarcharColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "DirectVarcharColumnWithoutoutPresent", "VARCHAR", @@ -2432,7 +2431,7 @@ TEST_F(TestColumn, VectorizedDirectVarcharColumnWithPresent) { write_row.init(tablet_schema); write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2475,7 +2474,7 @@ TEST_F(TestColumn, VectorizedDirectVarcharColumnWithPresent) { TEST_F(TestColumn, SkipDirectVarcharColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "DirectVarcharColumnWithPresent", @@ -2490,7 +2489,7 @@ TEST_F(TestColumn, SkipDirectVarcharColumnWithPresent) { write_row.init(tablet_schema); write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2533,7 +2532,7 @@ TEST_F(TestColumn, SkipDirectVarcharColumnWithPresent) { TEST_F(TestColumn, SeekDirectVarcharColumnWithoutPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "DirectVarcharColumnWithPresent", @@ -2548,7 +2547,7 @@ TEST_F(TestColumn, SeekDirectVarcharColumnWithoutPresent) { write_row.init(tablet_schema); write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2613,7 +2612,7 @@ TEST_F(TestColumn, SeekDirectVarcharColumnWithoutPresent) { TEST_F(TestColumn, SeekDirectVarcharColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "DirectVarcharColumnWithPresent", @@ -2628,7 +2627,7 @@ TEST_F(TestColumn, SeekDirectVarcharColumnWithPresent) { write_row.init(tablet_schema); write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2693,7 +2692,7 @@ TEST_F(TestColumn, SeekDirectVarcharColumnWithPresent) { TEST_F(TestColumn, VectorizedStringColumnWithoutPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "VarcharColumnWithoutoutPresent", "CHAR", @@ -2707,7 +2706,7 @@ TEST_F(TestColumn, VectorizedStringColumnWithoutPresent) { write_row.init(tablet_schema); write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2764,7 +2763,7 @@ TEST_F(TestColumn, VectorizedStringColumnWithoutPresent) { TEST_F(TestColumn, VectorizedStringColumnWithPresent) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "VarcharColumnWithoutoutPresent", "CHAR", @@ -2778,7 +2777,7 @@ TEST_F(TestColumn, VectorizedStringColumnWithPresent) { write_row.init(tablet_schema); write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2819,7 +2818,7 @@ TEST_F(TestColumn, VectorizedStringColumnWithPresent) { TEST_F(TestColumn, VectorizedStringColumnWithoutoutPresent2) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "VarcharColumnWithoutoutPresent", "CHAR", @@ -2833,7 +2832,7 @@ TEST_F(TestColumn, VectorizedStringColumnWithoutoutPresent2) { write_row.init(tablet_schema); write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); @@ -2910,7 +2909,7 @@ TEST_F(TestColumn, VectorizedStringColumnWithoutoutPresent2) { TEST_F(TestColumn, VectorizedDirectVarcharColumnWith65533) { // write data - std::vector tablet_schema; + TabletSchema tablet_schema; SetTabletSchemaWithOneColumn( "DirectVarcharColumnWithoutoutPresent", "VARCHAR", @@ -2924,7 +2923,7 @@ TEST_F(TestColumn, VectorizedDirectVarcharColumnWith65533) { write_row.init(tablet_schema); write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(tablet_schema); + RowBlock block(&tablet_schema); RowBlockInfo block_info; block_info.row_num = 10000; block.init(block_info); diff --git a/be/test/olap/comparison_predicate_test.cpp b/be/test/olap/comparison_predicate_test.cpp index c9e9db3d259e82..8d582a70c09fc2 100644 --- a/be/test/olap/comparison_predicate_test.cpp +++ b/be/test/olap/comparison_predicate_test.cpp @@ -99,27 +99,28 @@ public: \ delete _vectorized_batch; \ } \ } \ - void SetTabletColumn(std::string name, \ - std::string type, std::string aggregation, \ - uint32_t length, bool is_allow_null, bool is_key, TabletColumn* tablet_column) { \ + void SetTabletSchema(std::string name, \ + const std::string& type, const std::string& aggregation, \ + uint32_t length, bool is_allow_null, bool is_key, TabletSchema* tablet_schema) { \ + TabletSchemaPB tablet_schema_pb; \ static int id = 0; \ - ColumnPB column; \ - column.set_unique_id(++id); \ - column.set_name(name); \ - column.set_type(type); \ - column.set_is_key(is_key); \ - column.set_is_nullable(is_allow_null); \ - column.set_length(length); \ - column.set_aggregation(aggregation); \ - column.set_precision(1000); \ - column.set_frac(1000); \ - column.set_is_bf_column(false); \ - tablet_column->init_from_pb(column); \ + ColumnPB* column = tablet_schema_pb.add_column(); \ + column->set_unique_id(++id); \ + column->set_name(name); \ + column->set_type(type); \ + column->set_is_key(is_key); \ + column->set_is_nullable(is_allow_null); \ + column->set_length(length); \ + column->set_aggregation(aggregation); \ + column->set_precision(1000); \ + column->set_frac(1000); \ + column->set_is_bf_column(false); \ + tablet_schema->init_from_pb(tablet_schema_pb); \ } \ - void InitVectorizedBatch(const std::vector* schema, \ + void InitVectorizedBatch(const TabletSchema* tablet_schema, \ const std::vector&ids, \ int size) { \ - _vectorized_batch = new VectorizedRowBatch(schema, ids, size); \ + _vectorized_batch = new VectorizedRowBatch(tablet_schema, ids, size); \ _vectorized_batch->set_size(size); \ } \ std::unique_ptr _mem_tracker; \ @@ -132,17 +133,15 @@ TEST_PREDICATE_DEFINITION(TestLessPredicate) #define TEST_EQUAL_PREDICATE(TYPE, TYPE_NAME, FIELD_TYPE) \ TEST_F(TestEqualPredicate, TYPE_NAME##_COLUMN) { \ - std::vector schema; \ - TabletColumn tablet_column; \ - SetTabletColumn(std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, \ - "REPLACE", 1, false, true, &tablet_column); \ - schema.push_back(tablet_column); \ + TabletSchema tablet_schema; \ + SetTabletSchema(std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, \ + "REPLACE", 1, false, true, &tablet_schema); \ int size = 10; \ std::vector return_columns; \ - for (int i = 0; i < schema.size(); ++i) { \ + for (int i = 0; i < tablet_schema.num_columns(); ++i) { \ return_columns.push_back(i); \ } \ - InitVectorizedBatch(&schema, return_columns, size); \ + InitVectorizedBatch(&tablet_schema, return_columns, size); \ ColumnVector* col_vector = _vectorized_batch->column(0); \ \ /* for no nulls */ \ @@ -186,17 +185,15 @@ TEST_EQUAL_PREDICATE(int64_t, BIGINT, "BIGINT") TEST_EQUAL_PREDICATE(int128_t, LARGEINT, "LARGEINT") TEST_F(TestEqualPredicate, FLOAT_COLUMN) { - std::vector schema; - TabletColumn tablet_column; - SetTabletColumn(std::string("FLOAT_COLUMN"), "FLOAT", - "REPLACE", 1, false, true, &tablet_column); - schema.push_back(tablet_column); + TabletSchema tablet_schema; + SetTabletSchema(std::string("FLOAT_COLUMN"), "FLOAT", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -234,17 +231,15 @@ TEST_F(TestEqualPredicate, FLOAT_COLUMN) { } TEST_F(TestEqualPredicate, DOUBLE_COLUMN) { - std::vector schema; - TabletColumn tablet_column; - SetTabletColumn(std::string("DOUBLE_COLUMN"), "DOUBLE", - "REPLACE", 1, false, true, &tablet_column); - schema.push_back(tablet_column); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DOUBLE_COLUMN"), "DOUBLE", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -282,17 +277,15 @@ TEST_F(TestEqualPredicate, DOUBLE_COLUMN) { } TEST_F(TestEqualPredicate, DECIMAL_COLUMN) { - std::vector schema; - TabletColumn tablet_column; - SetTabletColumn(std::string("DECIMAL_COLUMN"), "DECIMAL", - "REPLACE", 1, false, true, &tablet_column); - schema.push_back(tablet_column); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DECIMAL_COLUMN"), "DECIMAL", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -333,17 +326,15 @@ TEST_F(TestEqualPredicate, DECIMAL_COLUMN) { } TEST_F(TestEqualPredicate, STRING_COLUMN) { - std::vector schema; - TabletColumn tablet_column; - SetTabletColumn(std::string("STRING_COLUMN"), "VARCHAR", - "REPLACE", 1, false, true, &tablet_column); - schema.push_back(tablet_column); + TabletSchema tablet_schema; + SetTabletSchema(std::string("STRING_COLUMN"), "VARCHAR", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -401,17 +392,15 @@ TEST_F(TestEqualPredicate, STRING_COLUMN) { } TEST_F(TestEqualPredicate, DATE_COLUMN) { - std::vector schema; - TabletColumn tablet_column; - SetTabletColumn(std::string("DATE_COLUMN"), "DATA", - "REPLACE", 1, false, true, &tablet_column); - schema.push_back(tablet_column); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DATE_COLUMN"), "DATA", + "REPLACE", 1, false, true, &tablet_schema); int size = 6; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -464,17 +453,15 @@ TEST_F(TestEqualPredicate, DATE_COLUMN) { } TEST_F(TestEqualPredicate, DATETIME_COLUMN) { - std::vector schema; - TabletColumn tablet_column; - SetTabletColumn(std::string("DATETIME_COLUMN"), "DATETIME", - "REPLACE", 1, false, true, &tablet_column); - schema.push_back(tablet_column); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DATETIME_COLUMN"), "DATETIME", + "REPLACE", 1, false, true, &tablet_schema); int size = 6; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -528,17 +515,15 @@ TEST_F(TestEqualPredicate, DATETIME_COLUMN) { #define TEST_LESS_PREDICATE(TYPE, TYPE_NAME, FIELD_TYPE) \ TEST_F(TestLessPredicate, TYPE_NAME##_COLUMN) { \ - std::vector schema; \ - TabletColumn tablet_column; \ - SetTabletColumn(std::string("TYPE_NAME_COLUMN"), FIELD_TYPE, \ - "REPLACE", 1, false, true, &tablet_column); \ - schema.push_back(tablet_column); \ + TabletSchema tablet_schema; \ + SetTabletSchema(std::string("TYPE_NAME_COLUMN"), FIELD_TYPE, \ + "REPLACE", 1, false, true, &tablet_schema); \ int size = 10; \ std::vector return_columns; \ - for (int i = 0; i < schema.size(); ++i) { \ + for (int i = 0; i < tablet_schema.num_columns(); ++i) { \ return_columns.push_back(i); \ } \ - InitVectorizedBatch(&schema, return_columns, size); \ + InitVectorizedBatch(&tablet_schema, return_columns, size); \ ColumnVector* col_vector = _vectorized_batch->column(0); \ \ /* for no nulls */ \ @@ -590,17 +575,15 @@ TEST_LESS_PREDICATE(int64_t, BIGINT, "BIGINT") TEST_LESS_PREDICATE(int128_t, LARGEINT, "LARGEINT") TEST_F(TestLessPredicate, FLOAT_COLUMN) { - std::vector schema; - TabletColumn tablet_column; - SetTabletColumn(std::string("FLOAT_COLUMN"), "FLOAT", - "REPLACE", 1, false, true, &tablet_column); - schema.push_back(tablet_column); + TabletSchema tablet_schema; + SetTabletSchema(std::string("FLOAT_COLUMN"), "FLOAT", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -647,17 +630,15 @@ TEST_F(TestLessPredicate, FLOAT_COLUMN) { } TEST_F(TestLessPredicate, DOUBLE_COLUMN) { - std::vector schema; - TabletColumn tablet_column; - SetTabletColumn(std::string("DOUBLE_COLUMN"), "DOUBLE", - "REPLACE", 1, false, true, &tablet_column); - schema.push_back(tablet_column); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DOUBLE_COLUMN"), "DOUBLE", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -704,17 +685,15 @@ TEST_F(TestLessPredicate, DOUBLE_COLUMN) { } TEST_F(TestLessPredicate, DECIMAL_COLUMN) { - std::vector schema; - TabletColumn tablet_column; - SetTabletColumn(std::string("DECIMAL_COLUMN"), "DECIMAL", - "REPLACE", 1, false, true, &tablet_column); - schema.push_back(tablet_column); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DECIMAL_COLUMN"), "DECIMAL", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -765,17 +744,15 @@ TEST_F(TestLessPredicate, DECIMAL_COLUMN) { } TEST_F(TestLessPredicate, STRING_COLUMN) { - std::vector schema; - TabletColumn tablet_column; - SetTabletColumn(std::string("STRING_COLUMN"), "VARCHAR", - "REPLACE", 1, false, true, &tablet_column); - schema.push_back(tablet_column); + TabletSchema tablet_schema; + SetTabletSchema(std::string("STRING_COLUMN"), "VARCHAR", + "REPLACE", 1, false, true, &tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -831,17 +808,15 @@ TEST_F(TestLessPredicate, STRING_COLUMN) { } TEST_F(TestLessPredicate, DATE_COLUMN) { - std::vector schema; - TabletColumn tablet_column; - SetTabletColumn(std::string("DATE_COLUMN"), "DATE", - "REPLACE", 1, false, true, &tablet_column); - schema.push_back(tablet_column); + TabletSchema tablet_schema; + SetTabletSchema(std::string("DATE_COLUMN"), "DATE", + "REPLACE", 1, false, true, &tablet_schema); int size = 6; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -891,17 +866,16 @@ TEST_F(TestLessPredicate, DATE_COLUMN) { } TEST_F(TestLessPredicate, DATETIME_COLUMN) { - std::vector schema; + TabletSchema tablet_schema; TabletColumn tablet_column; - SetTabletColumn(std::string("DATETIME_COLUMN"), "DATETIME", - "REPLACE", 1, false, true, &tablet_column); - schema.push_back(tablet_column); + SetTabletSchema(std::string("DATETIME_COLUMN"), "DATETIME", + "REPLACE", 1, false, true, &tablet_schema); int size = 6; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls diff --git a/be/test/olap/in_list_predicate_test.cpp b/be/test/olap/in_list_predicate_test.cpp index de2acb52a5301a..e34ea89c45f828 100644 --- a/be/test/olap/in_list_predicate_test.cpp +++ b/be/test/olap/in_list_predicate_test.cpp @@ -100,28 +100,30 @@ class TestInListPredicate : public testing::Test { } } - void SetTabletColumn(std::string name, - std::string type, std::string aggregation, - uint32_t length, bool is_allow_null, bool is_key, TabletColumn* tablet_column) { + void SetTabletSchema(std::string name, + const std::string& type, const std::string& aggregation, + uint32_t length, bool is_allow_null, bool is_key, TabletSchema* tablet_schema) { + TabletSchemaPB tablet_schema_pb; static int id = 0; - ColumnPB column; - column.set_unique_id(++id); - column.set_name(name); - column.set_type(type); - column.set_is_key(is_key); - column.set_is_nullable(is_allow_null); - column.set_length(length); - column.set_aggregation(aggregation); - column.set_precision(1000); - column.set_frac(1000); - column.set_is_bf_column(false); - tablet_column->init_from_pb(column); + ColumnPB* column = tablet_schema_pb.add_column(); + column->set_unique_id(++id); + column->set_name(name); + column->set_type(type); + column->set_is_key(is_key); + column->set_is_nullable(is_allow_null); + column->set_length(length); + column->set_aggregation(aggregation); + column->set_precision(1000); + column->set_frac(1000); + column->set_is_bf_column(false); + + tablet_schema->init_from_pb(tablet_schema_pb); } - void InitVectorizedBatch(const std::vector* schema, + void InitVectorizedBatch(const TabletSchema* tablet_schema, const std::vector& ids, int size) { - _vectorized_batch = new VectorizedRowBatch(schema, ids, size); + _vectorized_batch = new VectorizedRowBatch(tablet_schema, ids, size); _vectorized_batch->set_size(size); } std::unique_ptr _mem_tracker; @@ -131,17 +133,15 @@ class TestInListPredicate : public testing::Test { #define TEST_IN_LIST_PREDICATE(TYPE, TYPE_NAME, FIELD_TYPE) \ TEST_F(TestInListPredicate, TYPE_NAME##_COLUMN) { \ - std::vector schema; \ - TabletColumn tablet_schema; \ - SetTabletColumn(std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, \ + TabletSchema tablet_schema; \ + SetTabletSchema(std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, \ "REPLACE", 1, false, true, &tablet_schema); \ - schema.push_back(tablet_schema); \ int size = 10; \ std::vector return_columns; \ - for (int i = 0; i < schema.size(); ++i) { \ + for (int i = 0; i < tablet_schema.num_columns(); ++i) { \ return_columns.push_back(i); \ } \ - InitVectorizedBatch(&schema, return_columns, size); \ + InitVectorizedBatch(&tablet_schema, return_columns, size); \ ColumnVector* col_vector = _vectorized_batch->column(0); \ \ /* for no nulls */ \ @@ -191,17 +191,15 @@ TEST_IN_LIST_PREDICATE(int64_t, BIGINT, "BIGINT") TEST_IN_LIST_PREDICATE(int128_t, LARGEINT, "LARGEINT") TEST_F(TestInListPredicate, FLOAT_COLUMN) { - std::vector schema; - TabletColumn tablet_schema; - SetTabletColumn(std::string("FLOAT_COLUMN"), "FLOAT", + TabletSchema tablet_schema; + SetTabletSchema(std::string("FLOAT_COLUMN"), "FLOAT", "REPLACE", 1, false, true, &tablet_schema); - schema.push_back(tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -244,17 +242,15 @@ TEST_F(TestInListPredicate, FLOAT_COLUMN) { } TEST_F(TestInListPredicate, DOUBLE_COLUMN) { - std::vector schema; - TabletColumn tablet_schema; - SetTabletColumn(std::string("DOUBLE_COLUMN"), "DOUBLE", + TabletSchema tablet_schema; + SetTabletSchema(std::string("DOUBLE_COLUMN"), "DOUBLE", "REPLACE", 1, false, true, &tablet_schema); - schema.push_back(tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -298,17 +294,15 @@ TEST_F(TestInListPredicate, DOUBLE_COLUMN) { } TEST_F(TestInListPredicate, DECIMAL_COLUMN) { - std::vector schema; - TabletColumn tablet_schema; - SetTabletColumn(std::string("DECIMAL_COLUMN"), "DECIMAL", + TabletSchema tablet_schema; + SetTabletSchema(std::string("DECIMAL_COLUMN"), "DECIMAL", "REPLACE", 1, false, true, &tablet_schema); - schema.push_back(tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -361,17 +355,15 @@ TEST_F(TestInListPredicate, DECIMAL_COLUMN) { } TEST_F(TestInListPredicate, CHAR_COLUMN) { - std::vector schema; - TabletColumn tablet_schema; - SetTabletColumn(std::string("STRING_COLUMN"), "CHAR", + TabletSchema tablet_schema; + SetTabletSchema(std::string("STRING_COLUMN"), "CHAR", "REPLACE", 1, false, true, &tablet_schema); - schema.push_back(tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -446,17 +438,15 @@ TEST_F(TestInListPredicate, CHAR_COLUMN) { } TEST_F(TestInListPredicate, VARCHAR_COLUMN) { - std::vector schema; - TabletColumn tablet_schema; - SetTabletColumn(std::string("STRING_COLUMN"), "VARCHAR", + TabletSchema tablet_schema; + SetTabletSchema(std::string("STRING_COLUMN"), "VARCHAR", "REPLACE", 1, false, true, &tablet_schema); - schema.push_back(tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -529,17 +519,15 @@ TEST_F(TestInListPredicate, VARCHAR_COLUMN) { } TEST_F(TestInListPredicate, DATE_COLUMN) { - std::vector schema; - TabletColumn tablet_schema; - SetTabletColumn(std::string("DATE_COLUMN"), "DATE", + TabletSchema tablet_schema; + SetTabletSchema(std::string("DATE_COLUMN"), "DATE", "REPLACE", 1, false, true, &tablet_schema); - schema.push_back(tablet_schema); int size = 6; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -600,17 +588,15 @@ TEST_F(TestInListPredicate, DATE_COLUMN) { } TEST_F(TestInListPredicate, DATETIME_COLUMN) { - std::vector schema; - TabletColumn tablet_schema; - SetTabletColumn(std::string("DATETIME_COLUMN"), "DATETIME", + TabletSchema tablet_schema; + SetTabletSchema(std::string("DATETIME_COLUMN"), "DATETIME", "REPLACE", 1, false, true, &tablet_schema); - schema.push_back(tablet_schema); int size = 6; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls diff --git a/be/test/olap/null_predicate_test.cpp b/be/test/olap/null_predicate_test.cpp index 7a8fa448db20ab..f59b4a5b27a844 100644 --- a/be/test/olap/null_predicate_test.cpp +++ b/be/test/olap/null_predicate_test.cpp @@ -70,28 +70,29 @@ class TestNullPredicate : public testing::Test { } } - void SetTabletColumn(std::string name, + void SetTabletSchema(std::string name, std::string type, std::string aggregation, - uint32_t length, bool is_allow_null, bool is_key, TabletColumn* tablet_column) { + uint32_t length, bool is_allow_null, bool is_key, TabletSchema* tablet_schema) { + TabletSchemaPB tablet_schema_pb; static int id = 0; - ColumnPB column; - column.set_unique_id(++id); - column.set_name(name); - column.set_type(type); - column.set_is_key(is_key); - column.set_is_nullable(is_allow_null); - column.set_length(length); - column.set_aggregation(aggregation); - column.set_precision(1000); - column.set_frac(1000); - column.set_is_bf_column(false); - tablet_column->init_from_pb(column); + ColumnPB* column = tablet_schema_pb.add_column();; + column->set_unique_id(++id); + column->set_name(name); + column->set_type(type); + column->set_is_key(is_key); + column->set_is_nullable(is_allow_null); + column->set_length(length); + column->set_aggregation(aggregation); + column->set_precision(1000); + column->set_frac(1000); + column->set_is_bf_column(false); + tablet_schema->init_from_pb(tablet_schema_pb); } - void InitVectorizedBatch(const std::vector* schema, + void InitVectorizedBatch(const TabletSchema* tablet_schema, const std::vector&ids, int size) { - _vectorized_batch = new VectorizedRowBatch(schema, ids, size); + _vectorized_batch = new VectorizedRowBatch(tablet_schema, ids, size); _vectorized_batch->set_size(size); } std::unique_ptr _mem_tracker; @@ -101,17 +102,15 @@ class TestNullPredicate : public testing::Test { #define TEST_IN_LIST_PREDICATE(TYPE, TYPE_NAME, FIELD_TYPE) \ TEST_F(TestNullPredicate, TYPE_NAME##_COLUMN) { \ - std::vector schema; \ - TabletColumn tablet_schema; \ - SetTabletColumn(std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, \ + TabletSchema tablet_schema; \ + SetTabletSchema(std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, \ "REPLACE", 1, false, true, &tablet_schema); \ - schema.push_back(tablet_schema); \ int size = 10; \ std::vector return_columns; \ - for (int i = 0; i < schema.size(); ++i) { \ + for (int i = 0; i < tablet_schema.num_columns(); ++i) { \ return_columns.push_back(i); \ } \ - InitVectorizedBatch(&schema, return_columns, size); \ + InitVectorizedBatch(&tablet_schema, return_columns, size); \ ColumnVector* col_vector = _vectorized_batch->column(0); \ \ /* for no nulls */ \ @@ -151,17 +150,15 @@ TEST_IN_LIST_PREDICATE(int64_t, BIGINT, "BIGINT") TEST_IN_LIST_PREDICATE(int128_t, LARGEINT, "LARGEINT") TEST_F(TestNullPredicate, FLOAT_COLUMN) { - std::vector schema; - TabletColumn tablet_schema; - SetTabletColumn(std::string("FLOAT_COLUMN"), "FLOAT", + TabletSchema tablet_schema; + SetTabletSchema(std::string("FLOAT_COLUMN"), "FLOAT", "REPLACE", 1, false, true, &tablet_schema); - schema.push_back(tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -194,17 +191,15 @@ TEST_F(TestNullPredicate, FLOAT_COLUMN) { } TEST_F(TestNullPredicate, DOUBLE_COLUMN) { - std::vector schema; - TabletColumn tablet_schema; - SetTabletColumn(std::string("DOUBLE_COLUMN"), "DOUBLE", + TabletSchema tablet_schema; + SetTabletSchema(std::string("DOUBLE_COLUMN"), "DOUBLE", "REPLACE", 1, false, true, &tablet_schema); - schema.push_back(tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -238,17 +233,15 @@ TEST_F(TestNullPredicate, DOUBLE_COLUMN) { } TEST_F(TestNullPredicate, DECIMAL_COLUMN) { - std::vector schema; - TabletColumn tablet_schema; - SetTabletColumn(std::string("DECIMAL_COLUMN"), "DECIMAL", + TabletSchema tablet_schema; + SetTabletSchema(std::string("DECIMAL_COLUMN"), "DECIMAL", "REPLACE", 1, false, true, &tablet_schema); - schema.push_back(tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -285,17 +278,15 @@ TEST_F(TestNullPredicate, DECIMAL_COLUMN) { } TEST_F(TestNullPredicate, STRING_COLUMN) { - std::vector schema; - TabletColumn tablet_schema; - SetTabletColumn(std::string("STRING_COLUMN"), "VARCHAR", + TabletSchema tablet_schema; + SetTabletSchema(std::string("STRING_COLUMN"), "VARCHAR", "REPLACE", 1, false, true, &tablet_schema); - schema.push_back(tablet_schema); int size = 10; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -342,17 +333,15 @@ TEST_F(TestNullPredicate, STRING_COLUMN) { } TEST_F(TestNullPredicate, DATE_COLUMN) { - std::vector schema; - TabletColumn tablet_schema; - SetTabletColumn(std::string("DATE_COLUMN"), "DATE", + TabletSchema tablet_schema; + SetTabletSchema(std::string("DATE_COLUMN"), "DATE", "REPLACE", 1, false, true, &tablet_schema); - schema.push_back(tablet_schema); int size = 6; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls @@ -397,17 +386,15 @@ TEST_F(TestNullPredicate, DATE_COLUMN) { } TEST_F(TestNullPredicate, DATETIME_COLUMN) { - std::vector schema; - TabletColumn tablet_schema; - SetTabletColumn(std::string("DATETIME_COLUMN"), "DATETIME", + TabletSchema tablet_schema; + SetTabletSchema(std::string("DATETIME_COLUMN"), "DATETIME", "REPLACE", 1, false, true, &tablet_schema); - schema.push_back(tablet_schema); int size = 6; std::vector return_columns; - for (int i = 0; i < schema.size(); ++i) { + for (int i = 0; i < tablet_schema.num_columns(); ++i) { return_columns.push_back(i); } - InitVectorizedBatch(&schema, return_columns, size); + InitVectorizedBatch(&tablet_schema, return_columns, size); ColumnVector* col_vector = _vectorized_batch->column(0); // for no nulls diff --git a/be/test/olap/row_cursor_test.cpp b/be/test/olap/row_cursor_test.cpp index 3d2a00499787d4..1f4fae9659e0d5 100644 --- a/be/test/olap/row_cursor_test.cpp +++ b/be/test/olap/row_cursor_test.cpp @@ -25,280 +25,225 @@ namespace doris { -void set_tablet_schema_for_init(std::vector* tablet_schema) { - ColumnPB column_1; - column_1.set_unique_id(1); - column_1.set_name("column_1"); - column_1.set_type("TINYINT"); - column_1.set_is_key(true); - column_1.set_is_nullable(true); - column_1.set_length(1); - column_1.set_index_length(1); - TabletColumn k1; - k1.init_from_pb(column_1); - tablet_schema->push_back(k1); - - ColumnPB column_2; - column_2.set_unique_id(2); - column_2.set_name("column_2"); - column_2.set_type("SMALLINT"); - column_2.set_is_key(true); - column_2.set_is_nullable(true); - column_2.set_length(2); - column_2.set_index_length(2); - column_2.set_default_value("0"); - TabletColumn k2; - k2.init_from_pb(column_2); - tablet_schema->push_back(k2); - - ColumnPB column_3; - column_3.set_unique_id(3); - column_3.set_name("column_3"); - column_3.set_type("INT"); - column_3.set_is_key(true); - column_3.set_is_nullable(true); - column_3.set_length(4); - column_3.set_index_length(4); - TabletColumn k3; - k3.init_from_pb(column_3); - tablet_schema->push_back(k3); - - ColumnPB column_4; - column_4.set_unique_id(4); - column_4.set_name("column_4"); - column_4.set_type("DATE"); - column_4.set_is_key(true); - column_4.set_is_nullable(true); - column_4.set_length(3); - column_4.set_index_length(3); - TabletColumn k4; - k4.init_from_pb(column_4); - tablet_schema->push_back(k4); - - ColumnPB column_5; - column_5.set_unique_id(5); - column_5.set_name("column_5"); - column_5.set_type("DATETIME"); - column_5.set_is_key(true); - column_5.set_is_nullable(true); - column_5.set_length(8); - column_5.set_index_length(8); - TabletColumn k5; - k5.init_from_pb(column_5); - tablet_schema->push_back(k5); - - ColumnPB column_6; - column_6.set_unique_id(6); - column_6.set_name("column_6"); - column_6.set_type("DECIMAL"); - column_6.set_is_key(true); - column_6.set_is_nullable(true); - column_6.set_length(12); - column_6.set_index_length(12); - column_6.set_frac(3); - column_6.set_precision(6); - TabletColumn k6; - k6.init_from_pb(column_6); - tablet_schema->push_back(k6); - - ColumnPB column_7; - column_7.set_unique_id(7); - column_7.set_name("column_7"); - column_7.set_type("CHAR"); - column_7.set_is_key(true); - column_7.set_is_nullable(true); - column_7.set_length(4); - column_7.set_index_length(4); - column_7.set_default_value("char"); - TabletColumn k7; - k7.init_from_pb(column_7); - tablet_schema->push_back(k7); - - ColumnPB column_8; - column_8.set_unique_id(8); - column_8.set_name("column_8"); - column_8.set_type("BIGINT"); - column_8.set_is_nullable(true); - column_8.set_length(8); - column_8.set_aggregation("SUM"); - column_8.set_is_key(false); - TabletColumn v1; - v1.init_from_pb(column_8); - tablet_schema->push_back(v1); - - ColumnPB column_9; - column_9.set_unique_id(9); - column_9.set_name("column_9"); - column_9.set_type("VARCHAR"); - column_9.set_is_nullable(true); - column_9.set_length(16 + OLAP_STRING_MAX_BYTES); - column_9.set_aggregation("REPLACE"); - column_9.set_is_key(false); - TabletColumn v2; - v2.init_from_pb(column_9); - tablet_schema->push_back(v2); - - ColumnPB column_10; - column_10.set_unique_id(10); - column_10.set_name("column_10"); - column_10.set_type("LARGEINT"); - column_10.set_is_nullable(true); - column_10.set_length(16); - column_10.set_aggregation("MAX"); - column_10.set_is_key(false); - TabletColumn v3; - v3.init_from_pb(column_10); - tablet_schema->push_back(v3); - - ColumnPB column_11; - column_11.set_unique_id(11); - column_11.set_name("column_11"); - column_11.set_type("DECIMAL"); - column_11.set_is_nullable(true); - column_11.set_length(12); - column_11.set_aggregation("MIN"); - column_11.set_is_key(false); - TabletColumn v4; - v4.init_from_pb(column_11); - tablet_schema->push_back(v4); - - ColumnPB column_12; - column_12.set_unique_id(12); - column_12.set_name("column_12"); - column_12.set_type("HLL"); - column_12.set_is_nullable(true); - column_12.set_length(HLL_COLUMN_DEFAULT_LEN); - column_12.set_aggregation("HLL_UNION"); - column_12.set_is_key(false); - TabletColumn v5; - v5.init_from_pb(column_12); - tablet_schema->push_back(v5); +void set_tablet_schema_for_init(TabletSchema* tablet_schema) { + TabletSchemaPB tablet_schema_pb; + ColumnPB* column_1 = tablet_schema_pb.add_column(); + column_1->set_unique_id(1); + column_1->set_name("column_1"); + column_1->set_type("TINYINT"); + column_1->set_is_key(true); + column_1->set_is_nullable(true); + column_1->set_length(1); + column_1->set_index_length(1); + + ColumnPB* column_2 = tablet_schema_pb.add_column(); + column_2->set_unique_id(2); + column_2->set_name("column_2"); + column_2->set_type("SMALLINT"); + column_2->set_is_key(true); + column_2->set_is_nullable(true); + column_2->set_length(2); + column_2->set_index_length(2); + column_2->set_default_value("0"); + + ColumnPB* column_3 = tablet_schema_pb.add_column(); + column_3->set_unique_id(3); + column_3->set_name("column_3"); + column_3->set_type("INT"); + column_3->set_is_key(true); + column_3->set_is_nullable(true); + column_3->set_length(4); + column_3->set_index_length(4); + + ColumnPB* column_4 = tablet_schema_pb.add_column(); + column_4->set_unique_id(4); + column_4->set_name("column_4"); + column_4->set_type("DATE"); + column_4->set_is_key(true); + column_4->set_is_nullable(true); + column_4->set_length(3); + column_4->set_index_length(3); + + ColumnPB* column_5 = tablet_schema_pb.add_column(); + column_5->set_unique_id(5); + column_5->set_name("column_5"); + column_5->set_type("DATETIME"); + column_5->set_is_key(true); + column_5->set_is_nullable(true); + column_5->set_length(8); + column_5->set_index_length(8); + + ColumnPB* column_6 = tablet_schema_pb.add_column(); + column_6->set_unique_id(6); + column_6->set_name("column_6"); + column_6->set_type("DECIMAL"); + column_6->set_is_key(true); + column_6->set_is_nullable(true); + column_6->set_length(12); + column_6->set_index_length(12); + column_6->set_frac(3); + column_6->set_precision(6); + + ColumnPB* column_7 = tablet_schema_pb.add_column(); + column_7->set_unique_id(7); + column_7->set_name("column_7"); + column_7->set_type("CHAR"); + column_7->set_is_key(true); + column_7->set_is_nullable(true); + column_7->set_length(4); + column_7->set_index_length(4); + column_7->set_default_value("char"); + + ColumnPB* column_8 = tablet_schema_pb.add_column(); + column_8->set_unique_id(8); + column_8->set_name("column_8"); + column_8->set_type("BIGINT"); + column_8->set_is_nullable(true); + column_8->set_length(8); + column_8->set_aggregation("SUM"); + column_8->set_is_key(false); + + ColumnPB* column_9 = tablet_schema_pb.add_column(); + column_9->set_unique_id(9); + column_9->set_name("column_9"); + column_9->set_type("VARCHAR"); + column_9->set_is_nullable(true); + column_9->set_length(16 + OLAP_STRING_MAX_BYTES); + column_9->set_aggregation("REPLACE"); + column_9->set_is_key(false); + + ColumnPB* column_10 = tablet_schema_pb.add_column(); + column_10->set_unique_id(10); + column_10->set_name("column_10"); + column_10->set_type("LARGEINT"); + column_10->set_is_nullable(true); + column_10->set_length(16); + column_10->set_aggregation("MAX"); + column_10->set_is_key(false); + + ColumnPB* column_11 = tablet_schema_pb.add_column(); + column_11->set_unique_id(11); + column_11->set_name("column_11"); + column_11->set_type("DECIMAL"); + column_11->set_is_nullable(true); + column_11->set_length(12); + column_11->set_aggregation("MIN"); + column_11->set_is_key(false); + + ColumnPB* column_12 = tablet_schema_pb.add_column(); + column_12->set_unique_id(12); + column_12->set_name("column_12"); + column_12->set_type("HLL"); + column_12->set_is_nullable(true); + column_12->set_length(HLL_COLUMN_DEFAULT_LEN); + column_12->set_aggregation("HLL_UNION"); + column_12->set_is_key(false); + + tablet_schema->init_from_pb(tablet_schema_pb); } -void set_tablet_schema_for_scan_key(std::vector* tablet_schema) { - ColumnPB column_1; - column_1.set_unique_id(1); - column_1.set_name("column_1"); - column_1.set_type("CHAR"); - column_1.set_is_key(true); - column_1.set_is_nullable(true); - column_1.set_length(4); - column_1.set_index_length(4); - column_1.set_default_value("char"); - TabletColumn k1; - k1.init_from_pb(column_1); - tablet_schema->push_back(k1); - - ColumnPB column_2; - column_2.set_unique_id(2); - column_2.set_name("column_2"); - column_2.set_type("VARCHAR"); - column_2.set_is_key(true); - column_2.set_is_nullable(true); - column_2.set_length(16 + OLAP_STRING_MAX_BYTES); - column_2.set_index_length(20); - TabletColumn k2; - k2.init_from_pb(column_2); - tablet_schema->push_back(k2); - - ColumnPB column_3; - column_3.set_unique_id(3); - column_3.set_name("column_3"); - column_3.set_type("LARGEINT"); - column_3.set_is_nullable(true); - column_3.set_length(16); - column_3.set_aggregation("MAX"); - column_3.set_is_key(false); - TabletColumn v1; - v1.init_from_pb(column_3); - tablet_schema->push_back(v1); - - ColumnPB column_4; - column_4.set_unique_id(9); - column_4.set_name("column_4"); - column_4.set_type("DECIMAL"); - column_4.set_is_nullable(true); - column_4.set_length(12); - column_4.set_aggregation("MIN"); - column_4.set_is_key(false); - TabletColumn v2; - v2.init_from_pb(column_4); - tablet_schema->push_back(v2); +void set_tablet_schema_for_scan_key(TabletSchema* tablet_schema) { + TabletSchemaPB tablet_schema_pb; + + ColumnPB* column_1 = tablet_schema_pb.add_column(); + column_1->set_unique_id(1); + column_1->set_name("column_1"); + column_1->set_type("CHAR"); + column_1->set_is_key(true); + column_1->set_is_nullable(true); + column_1->set_length(4); + column_1->set_index_length(4); + column_1->set_default_value("char"); + + ColumnPB* column_2 = tablet_schema_pb.add_column(); + column_2->set_unique_id(2); + column_2->set_name("column_2"); + column_2->set_type("VARCHAR"); + column_2->set_is_key(true); + column_2->set_is_nullable(true); + column_2->set_length(16 + OLAP_STRING_MAX_BYTES); + column_2->set_index_length(20); + + ColumnPB* column_3 = tablet_schema_pb.add_column(); + column_3->set_unique_id(3); + column_3->set_name("column_3"); + column_3->set_type("LARGEINT"); + column_3->set_is_nullable(true); + column_3->set_length(16); + column_3->set_aggregation("MAX"); + column_3->set_is_key(false); + + ColumnPB* column_4 = tablet_schema_pb.add_column(); + column_4->set_unique_id(9); + column_4->set_name("column_4"); + column_4->set_type("DECIMAL"); + column_4->set_is_nullable(true); + column_4->set_length(12); + column_4->set_aggregation("MIN"); + column_4->set_is_key(false); + + tablet_schema->init_from_pb(tablet_schema_pb); } -void set_tablet_schema_for_cmp_and_aggregate(std::vector* tablet_schema) { - ColumnPB column_1; - column_1.set_unique_id(1); - column_1.set_name("column_1"); - column_1.set_type("CHAR"); - column_1.set_is_key(true); - column_1.set_is_nullable(true); - column_1.set_length(4); - column_1.set_index_length(4); - column_1.set_default_value("char"); - TabletColumn k1; - k1.init_from_pb(column_1); - tablet_schema->push_back(k1); - - ColumnPB column_2; - column_2.set_unique_id(2); - column_2.set_name("column_2"); - column_2.set_type("INT"); - column_2.set_is_key(true); - column_2.set_is_nullable(true); - column_2.set_length(4); - column_2.set_index_length(4); - TabletColumn k2; - k2.init_from_pb(column_2); - tablet_schema->push_back(k2); - - ColumnPB column_3; - column_3.set_unique_id(3); - column_3.set_name("column_3"); - column_3.set_type("LARGEINT"); - column_3.set_is_nullable(true); - column_3.set_length(16); - column_3.set_aggregation("SUM"); - column_3.set_is_key(false); - TabletColumn v1; - v1.init_from_pb(column_3); - tablet_schema->push_back(v1); - - ColumnPB column_4; - column_4.set_unique_id(9); - column_4.set_name("column_4"); - column_4.set_type("DOUBLE"); - column_4.set_is_nullable(true); - column_4.set_length(8); - column_4.set_aggregation("MIN"); - column_4.set_is_key(false); - TabletColumn v2; - v2.init_from_pb(column_4); - tablet_schema->push_back(v2); - - ColumnPB column_5; - column_5.set_unique_id(3); - column_5.set_name("column_5"); - column_5.set_type("DECIMAL"); - column_5.set_is_nullable(true); - column_5.set_length(12); - column_5.set_aggregation("MAX"); - column_5.set_is_key(false); - TabletColumn v3; - v3.init_from_pb(column_5); - tablet_schema->push_back(v3); - - ColumnPB column_6; - column_6.set_unique_id(9); - column_6.set_name("column_6"); - column_6.set_type("VARCHAR"); - column_6.set_is_nullable(true); - column_6.set_length(16 + OLAP_STRING_MAX_BYTES); - column_6.set_aggregation("REPLACE"); - column_6.set_is_key(false); - TabletColumn v4; - v4.init_from_pb(column_6); - tablet_schema->push_back(v4); +void set_tablet_schema_for_cmp_and_aggregate(TabletSchema* tablet_schema) { + TabletSchemaPB tablet_schema_pb; + + ColumnPB* column_1 = tablet_schema_pb.add_column(); + column_1->set_unique_id(1); + column_1->set_name("column_1"); + column_1->set_type("CHAR"); + column_1->set_is_key(true); + column_1->set_is_nullable(true); + column_1->set_length(4); + column_1->set_index_length(4); + column_1->set_default_value("char"); + + ColumnPB* column_2 = tablet_schema_pb.add_column(); + column_2->set_unique_id(2); + column_2->set_name("column_2"); + column_2->set_type("INT"); + column_2->set_is_key(true); + column_2->set_is_nullable(true); + column_2->set_length(4); + column_2->set_index_length(4); + + ColumnPB* column_3 = tablet_schema_pb.add_column(); + column_3->set_unique_id(3); + column_3->set_name("column_3"); + column_3->set_type("LARGEINT"); + column_3->set_is_nullable(true); + column_3->set_length(16); + column_3->set_aggregation("SUM"); + column_3->set_is_key(false); + + ColumnPB* column_4 = tablet_schema_pb.add_column(); + column_4->set_unique_id(9); + column_4->set_name("column_4"); + column_4->set_type("DOUBLE"); + column_4->set_is_nullable(true); + column_4->set_length(8); + column_4->set_aggregation("MIN"); + column_4->set_is_key(false); + + ColumnPB* column_5 = tablet_schema_pb.add_column(); + column_5->set_unique_id(3); + column_5->set_name("column_5"); + column_5->set_type("DECIMAL"); + column_5->set_is_nullable(true); + column_5->set_length(12); + column_5->set_aggregation("MAX"); + column_5->set_is_key(false); + + ColumnPB* column_6 = tablet_schema_pb.add_column(); + column_6->set_unique_id(9); + column_6->set_name("column_6"); + column_6->set_type("VARCHAR"); + column_6->set_is_nullable(true); + column_6->set_length(16 + OLAP_STRING_MAX_BYTES); + column_6->set_aggregation("REPLACE"); + column_6->set_is_key(false); + + tablet_schema->init_from_pb(tablet_schema_pb); } class TestRowCursor : public testing::Test { @@ -317,7 +262,7 @@ class TestRowCursor : public testing::Test { }; TEST_F(TestRowCursor, InitRowCursor) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_init(&tablet_schema); RowCursor row; OLAPStatus res = row.init(tablet_schema); @@ -327,7 +272,7 @@ TEST_F(TestRowCursor, InitRowCursor) { } TEST_F(TestRowCursor, InitRowCursorWithColumnCount) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_init(&tablet_schema); RowCursor row; OLAPStatus res = row.init(tablet_schema, 5); @@ -339,11 +284,11 @@ TEST_F(TestRowCursor, InitRowCursorWithColumnCount) { } TEST_F(TestRowCursor, InitRowCursorWithColIds) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_init(&tablet_schema); std::vector col_ids; - for (size_t i = 0; i < tablet_schema.size() / 2; ++i) { + for (size_t i = 0; i < tablet_schema.num_columns() / 2; ++i) { col_ids.push_back(i * 2); } @@ -355,7 +300,7 @@ TEST_F(TestRowCursor, InitRowCursorWithColIds) { } TEST_F(TestRowCursor, InitRowCursorWithScanKey) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_scan_key(&tablet_schema); std::vector scan_keys; @@ -378,7 +323,7 @@ TEST_F(TestRowCursor, InitRowCursorWithScanKey) { } TEST_F(TestRowCursor, SetMinAndMaxKey) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_init(&tablet_schema); RowCursor min_row; @@ -388,7 +333,7 @@ TEST_F(TestRowCursor, SetMinAndMaxKey) { res = min_row.build_min_key(); ASSERT_EQ(res, OLAP_SUCCESS); - for (size_t i = 0; i < tablet_schema.size(); ++i) { + for (size_t i = 0; i < tablet_schema.num_columns(); ++i) { ASSERT_TRUE(min_row.is_min(i)); } @@ -399,7 +344,7 @@ TEST_F(TestRowCursor, SetMinAndMaxKey) { } TEST_F(TestRowCursor, EqualAndCompare) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_cmp_and_aggregate(&tablet_schema); RowCursor left; @@ -440,7 +385,7 @@ TEST_F(TestRowCursor, EqualAndCompare) { } TEST_F(TestRowCursor, IndexCmp) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_cmp_and_aggregate(&tablet_schema); RowCursor left; @@ -481,7 +426,7 @@ TEST_F(TestRowCursor, IndexCmp) { } TEST_F(TestRowCursor, FullKeyCmp) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_cmp_and_aggregate(&tablet_schema); RowCursor left; @@ -521,7 +466,7 @@ TEST_F(TestRowCursor, FullKeyCmp) { } TEST_F(TestRowCursor, AggregateWithoutNull) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_cmp_and_aggregate(&tablet_schema); RowCursor row; @@ -581,7 +526,7 @@ TEST_F(TestRowCursor, AggregateWithoutNull) { } TEST_F(TestRowCursor, AggregateWithNull) { - std::vector tablet_schema; + TabletSchema tablet_schema; set_tablet_schema_for_cmp_and_aggregate(&tablet_schema); RowCursor row;