Skip to content

[Bug] fix wrong null flag #10883

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dataroaring merged 3 commits into from
Jul 16, 2022
Merged

[Bug] fix wrong null flag #10883

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
96c31ea
[Bug] fix wrong null flag
Gabriel39 Jul 15, 2022
0756a6e
update
Gabriel39 Jul 15, 2022
8292301
update
Gabriel39 Jul 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 22 additions & 22 deletions be/src/vec/olap/olap_data_convertor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,6 @@ void OlapBlockDataConvertor::OlapColumnDataConvertorBase::set_source_column(
auto nullable_column =
assert_cast<const vectorized::ColumnNullable*>(_typed_column.column.get());
_nullmap = nullable_column->get_null_map_data().data();
_nullmap += row_pos;
}
}

Expand All @@ -157,9 +156,11 @@ void OlapBlockDataConvertor::OlapColumnDataConvertorBase::clear_source_column()
_typed_column.column = nullptr;
}

// This should be called only in SegmentWriter. If you want to access nullmap in Convertor,
// use `_nullmap` directly.
const UInt8* OlapBlockDataConvertor::OlapColumnDataConvertorBase::get_nullmap() const {
assert(_typed_column.column);
return _nullmap;
return _nullmap + _row_pos;
}

// class OlapBlockDataConvertor::OlapColumnDataConvertorObject
Expand All @@ -177,7 +178,6 @@ const void* OlapBlockDataConvertor::OlapColumnDataConvertorObject::get_data() co

const void* OlapBlockDataConvertor::OlapColumnDataConvertorObject::get_data_at(
size_t offset) const {
assert(offset < _num_rows && _num_rows == _slice.size());
UInt8 null_flag = 0;
if (_nullmap) {
null_flag = _nullmap[offset];
Expand Down Expand Up @@ -205,7 +205,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorBitMap::convert_to_olap()

size_t total_size = 0;
if (_nullmap) {
const UInt8* nullmap_cur = _nullmap;
const UInt8* nullmap_cur = _nullmap + _row_pos;
while (bitmap_value_cur != bitmap_value_end) {
if (!*nullmap_cur) {
total_size += bitmap_value_cur->getSizeInBytes();
Expand All @@ -226,7 +226,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorBitMap::convert_to_olap()
char* raw_data = _raw_data.data();
Slice* slice = _slice.data();
if (_nullmap) {
const UInt8* nullmap_cur = _nullmap;
const UInt8* nullmap_cur = _nullmap + _row_pos;
while (bitmap_value_cur != bitmap_value_end) {
if (!*nullmap_cur) {
slice_size = bitmap_value_cur->getSizeInBytes();
Expand All @@ -244,7 +244,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorBitMap::convert_to_olap()
++nullmap_cur;
++bitmap_value_cur;
}
assert(nullmap_cur == _nullmap + _num_rows && slice == _slice.get_end_ptr());
assert(nullmap_cur == _nullmap + _row_pos + _num_rows && slice == _slice.get_end_ptr());
} else {
while (bitmap_value_cur != bitmap_value_end) {
slice_size = bitmap_value_cur->getSizeInBytes();
Expand Down Expand Up @@ -281,7 +281,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorHLL::convert_to_olap() {

size_t total_size = 0;
if (_nullmap) {
const UInt8* nullmap_cur = _nullmap;
const UInt8* nullmap_cur = _nullmap + _row_pos;
while (hll_value_cur != hll_value_end) {
if (!*nullmap_cur) {
total_size += hll_value_cur->max_serialized_size();
Expand All @@ -303,7 +303,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorHLL::convert_to_olap() {

hll_value_cur = hll_value;
if (_nullmap) {
const UInt8* nullmap_cur = _nullmap;
const UInt8* nullmap_cur = _nullmap + _row_pos;
while (hll_value_cur != hll_value_end) {
if (!*nullmap_cur) {
slice_size = hll_value_cur->serialize((uint8_t*)raw_data);
Expand All @@ -320,7 +320,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorHLL::convert_to_olap() {
++nullmap_cur;
++hll_value_cur;
}
assert(nullmap_cur == _nullmap + _num_rows && slice == _slice.get_end_ptr());
assert(nullmap_cur == _nullmap + _row_pos + _num_rows && slice == _slice.get_end_ptr());
} else {
while (hll_value_cur != hll_value_end) {
slice_size = hll_value_cur->serialize((uint8_t*)raw_data);
Expand Down Expand Up @@ -355,7 +355,6 @@ const void* OlapBlockDataConvertor::OlapColumnDataConvertorChar::get_data() cons
}

const void* OlapBlockDataConvertor::OlapColumnDataConvertorChar::get_data_at(size_t offset) const {
assert(offset < _num_rows && _num_rows == _slice.size());
UInt8 null_flag = 0;
if (_nullmap) {
null_flag = _nullmap[offset];
Expand All @@ -382,7 +381,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorChar::convert_to_olap() {
}

for (size_t i = 0; i < _num_rows; i++) {
if (!_nullmap || !_nullmap[i]) {
if (!_nullmap || !_nullmap[i + _row_pos]) {
_slice[i] = column_string->get_data_at(i + _row_pos).to_slice();
DCHECK(_slice[i].size == _length)
<< "char type data length not equal to schema, schema=" << _length
Expand Down Expand Up @@ -411,7 +410,6 @@ const void* OlapBlockDataConvertor::OlapColumnDataConvertorVarChar::get_data() c

const void* OlapBlockDataConvertor::OlapColumnDataConvertorVarChar::get_data_at(
size_t offset) const {
assert(offset < _num_rows && _num_rows == _slice.size());
UInt8 null_flag = 0;
if (_nullmap) {
null_flag = _nullmap[offset];
Expand Down Expand Up @@ -440,7 +438,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorVarChar::convert_to_olap()
Slice* slice = _slice.data();
size_t string_offset = *(offset_cur - 1);
if (_nullmap) {
const UInt8* nullmap_cur = _nullmap;
const UInt8* nullmap_cur = _nullmap + _row_pos;
while (offset_cur != offset_end) {
if (!*nullmap_cur) {
slice->data = const_cast<char*>(char_data + string_offset);
Expand All @@ -461,7 +459,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorVarChar::convert_to_olap()
++slice;
++offset_cur;
}
assert(nullmap_cur == _nullmap + _num_rows && slice == _slice.get_end_ptr());
assert(nullmap_cur == _nullmap + _row_pos + _num_rows && slice == _slice.get_end_ptr());
} else {
while (offset_cur != offset_end) {
slice->data = const_cast<char*>(char_data + string_offset);
Expand Down Expand Up @@ -513,7 +511,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDate::convert_to_olap() {
const DateV2Value* datetime_end = datetime_cur + _num_rows;
uint24_t* value = _values.data();
if (_nullmap) {
const UInt8* nullmap_cur = _nullmap;
const UInt8* nullmap_cur = _nullmap + _row_pos;
while (datetime_cur != datetime_end) {
if (!*nullmap_cur) {
*value = datetime_cur->to_olap_date();
Expand All @@ -524,7 +522,8 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDate::convert_to_olap() {
++datetime_cur;
++nullmap_cur;
}
assert(nullmap_cur == _nullmap + _num_rows && value == _values.get_end_ptr());
assert(nullmap_cur == _nullmap + _row_pos + _num_rows &&
value == _values.get_end_ptr());
} else {
while (datetime_cur != datetime_end) {
*value = datetime_cur->to_olap_date();
Expand Down Expand Up @@ -553,7 +552,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDate::convert_to_olap() {
const VecDateTimeValue* datetime_end = datetime_cur + _num_rows;
uint24_t* value = _values.data();
if (_nullmap) {
const UInt8* nullmap_cur = _nullmap;
const UInt8* nullmap_cur = _nullmap + _row_pos;
while (datetime_cur != datetime_end) {
if (!*nullmap_cur) {
*value = datetime_cur->to_olap_date();
Expand All @@ -564,7 +563,8 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDate::convert_to_olap() {
++datetime_cur;
++nullmap_cur;
}
assert(nullmap_cur == _nullmap + _num_rows && value == _values.get_end_ptr());
assert(nullmap_cur == _nullmap + _row_pos + _num_rows &&
value == _values.get_end_ptr());
} else {
while (datetime_cur != datetime_end) {
*value = datetime_cur->to_olap_date();
Expand Down Expand Up @@ -597,7 +597,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDateTime::convert_to_olap(
const VecDateTimeValue* datetime_end = datetime_cur + _num_rows;
uint64_t* value = _values.data();
if (_nullmap) {
const UInt8* nullmap_cur = _nullmap;
const UInt8* nullmap_cur = _nullmap + _row_pos;
while (datetime_cur != datetime_end) {
if (!*nullmap_cur) {
*value = datetime_cur->to_olap_datetime();
Expand All @@ -608,7 +608,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDateTime::convert_to_olap(
++datetime_cur;
++nullmap_cur;
}
assert(nullmap_cur == _nullmap + _num_rows && value == _values.get_end_ptr());
assert(nullmap_cur == _nullmap + _row_pos + _num_rows && value == _values.get_end_ptr());
} else {
while (datetime_cur != datetime_end) {
*value = datetime_cur->to_olap_datetime();
Expand Down Expand Up @@ -652,7 +652,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDecimal::convert_to_olap()
++decimal_cur;
++nullmap_cur;
}
assert(nullmap_cur == _nullmap + _num_rows && value == _values.get_end_ptr());
assert(nullmap_cur == _nullmap + _row_pos + _num_rows && value == _values.get_end_ptr());
} else {
while (decimal_cur != decimal_end) {
value->integer = decimal_cur->int_value();
Expand Down Expand Up @@ -713,7 +713,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorArray::convert_to_olap(
for (size_t i = 0; i < _num_rows; ++i, ++collection_value) {
int64_t cur_pos = _row_pos + i;
int64_t prev_pos = cur_pos - 1;
if (_nullmap && _nullmap[cur_pos - _row_pos]) {
if (_nullmap && _nullmap[cur_pos]) {
continue;
}
auto offset = offsets[prev_pos];
Expand Down
3 changes: 1 addition & 2 deletions be/src/vec/olap/olap_data_convertor.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,6 @@ class OlapBlockDataConvertor {
}
const void* get_data() const override { return _values.data(); }
const void* get_data_at(size_t offset) const override {
assert(offset < _num_rows && _num_rows == _values.size());
UInt8 null_flag = 0;
if (_nullmap) {
null_flag = _nullmap[offset];
Expand Down Expand Up @@ -298,7 +297,7 @@ class OlapBlockDataConvertor {
const VecDateTimeValue* datetime_end = datetime_cur + _num_rows;
uint32_t* value = const_cast<uint32_t*>(values_);
if (_nullmap) {
const UInt8* nullmap_cur = _nullmap;
const UInt8* nullmap_cur = _nullmap + _row_pos;
while (datetime_cur != datetime_end) {
if (!*nullmap_cur) {
*value = datetime_cur->to_date_v2();
Expand Down