Skip to content

[refactor] simplify column permute #8329

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
zbtzbtzbt wants to merge 1 commit into from
Closed

[refactor] simplify column permute #8329

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 1 addition & 11 deletions be/src/vec/columns/column_const.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,17 +72,7 @@ void ColumnConst::replicate(const uint32_t* counts, size_t target_size, IColumn&
}

ColumnPtr ColumnConst::permute(const Permutation& perm, size_t limit) const {
if (limit == 0) {
limit = s;
} else {
limit = std::min(s, limit);
}

if (perm.size() < limit) {
LOG(FATAL) << fmt::format("Size of permutation ({}) is less than required ({})",
perm.size(), limit);
}

limit = getLimitForPermutation(*this, perm, limit);
return ColumnConst::create(data, limit);
}

Expand Down
13 changes: 2 additions & 11 deletions be/src/vec/columns/column_decimal.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
// https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/ColumnDecimal.cpp
// and modified by Doris

#include "vec/columns/columns_common.h"
#include "vec/columns/column_decimal.h"

#include "vec/common/arena.h"
Expand Down Expand Up @@ -90,17 +91,7 @@ void ColumnDecimal<T>::get_permutation(bool reverse, size_t limit, int,

template <typename T>
ColumnPtr ColumnDecimal<T>::permute(const IColumn::Permutation& perm, size_t limit) const {
size_t size = limit ? std::min(data.size(), limit) : data.size();
if (perm.size() < size) {
LOG(FATAL) << "Size of permutation is less than required.";
}

auto res = this->create(size, scale);
typename Self::Container& res_data = res->get_data();

for (size_t i = 0; i < size; ++i) res_data[i] = data[perm[i]];

return res;
return permuteImpl(*this, perm, limit);
}

template <typename T>
Expand Down
7 changes: 1 addition & 6 deletions be/src/vec/columns/column_decimal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,12 +228,7 @@ class ColumnDecimal final : public COWHelper<ColumnVectorHelper, ColumnDecimal<T
template <typename T>
template <typename Type>
ColumnPtr ColumnDecimal<T>::index_impl(const PaddedPODArray<Type>& indexes, size_t limit) const {
size_t size = indexes.size();

if (limit == 0)
limit = size;
else
limit = std::min(size, limit);
DCHECK(limit <= indexes.size());

auto res = this->create(limit, scale);
typename Self::Container& res_data = res->get_data();
Expand Down
45 changes: 2 additions & 43 deletions be/src/vec/columns/column_string.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
// https://github.com/ClickHouse/ClickHouse/blob/master/src/Columns/ColumnString.cpp
// and modified by Doris

#include "vec/columns/columns_common.h"
#include "vec/columns/column_string.h"

#include "vec/columns/collator.h"
Expand Down Expand Up @@ -116,49 +117,7 @@ ColumnPtr ColumnString::filter(const Filter& filt, ssize_t result_size_hint) con
}

ColumnPtr ColumnString::permute(const Permutation& perm, size_t limit) const {
size_t size = offsets.size();

if (limit == 0)
limit = size;
else
limit = std::min(size, limit);

if (perm.size() < limit) {
LOG(FATAL) << "Size of permutation is less than required.";
}

if (limit == 0) return ColumnString::create();

auto res = ColumnString::create();

Chars& res_chars = res->chars;
Offsets& res_offsets = res->offsets;

if (limit == size)
res_chars.resize(chars.size());
else {
size_t new_chars_size = 0;
for (size_t i = 0; i < limit; ++i) new_chars_size += size_at(perm[i]);
res_chars.resize(new_chars_size);
}

res_offsets.resize(limit);

Offset current_new_offset = 0;

for (size_t i = 0; i < limit; ++i) {
size_t j = perm[i];
size_t string_offset = offsets[j - 1];
size_t string_size = offsets[j] - string_offset;

memcpy_small_allow_read_write_overflow15(&res_chars[current_new_offset],
&chars[string_offset], string_size);

current_new_offset += string_size;
res_offsets[i] = current_new_offset;
}

return res;
return permuteImpl(*this, perm, limit);
}

StringRef ColumnString::serialize_value_into_arena(size_t n, Arena& arena,
Expand Down
18 changes: 2 additions & 16 deletions be/src/vec/columns/column_vector.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
// https://github.com/ClickHouse/ClickHouse/blob/master/src/Columns/ColumnVector.cpp
// and modified by Doris

#include "vec/columns/columns_common.h"
#include "vec/columns/column_vector.h"

#include <pdqsort.h>
Expand Down Expand Up @@ -291,22 +292,7 @@ ColumnPtr ColumnVector<T>::filter(const IColumn::Filter& filt, ssize_t result_si

template <typename T>
ColumnPtr ColumnVector<T>::permute(const IColumn::Permutation& perm, size_t limit) const {
size_t size = data.size();

if (limit == 0)
limit = size;
else
limit = std::min(size, limit);

if (perm.size() < limit) {
LOG(FATAL) << "Size of permutation is less than required.";
}

auto res = this->create(limit);
typename Self::Container& res_data = res->get_data();
for (size_t i = 0; i < limit; ++i) res_data[i] = data[perm[i]];

return res;
return permuteImpl(*this, perm, limit);
}

template <typename T>
Expand Down
7 changes: 1 addition & 6 deletions be/src/vec/columns/column_vector.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -360,12 +360,7 @@ class ColumnVector final : public COWHelper<ColumnVectorHelper, ColumnVector<T>>
template <typename T>
template <typename Type>
ColumnPtr ColumnVector<T>::index_impl(const PaddedPODArray<Type>& indexes, size_t limit) const {
size_t size = indexes.size();

if (limit == 0)
limit = size;
else
limit = std::min(size, limit);
DCHECK(limit <= indexes.size());

auto res = this->create(limit);
typename Self::Container& res_data = res->get_data();
Expand Down
23 changes: 23 additions & 0 deletions be/src/vec/columns/columns_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#pragma once

#include "vec/columns/column.h"
#include "vec/common/pod_array.h"

/// Common helper methods for implementation of different columns.

Expand Down Expand Up @@ -79,6 +80,28 @@ ColumnPtr select_index_impl(const Column& column, const IColumn& indexes, size_t
}
}

template <typename Column>
size_t getLimitForPermutation(const Column& column, const IColumn::Permutation& perm, size_t limit)
{
if (limit == 0)
limit = column.size();
else
limit = std::min(column.size(), limit);

if (perm.size() < limit){
LOG(FATAL) << "Size of permutation is less than required";
}

return limit;
}

template <typename Column>
ColumnPtr permuteImpl(const Column& column, const IColumn::Permutation& perm, size_t limit)
{
limit = getLimitForPermutation(column, perm, limit);
return column.index_impl(perm, limit);
}

#define INSTANTIATE_INDEX_IMPL(Column) \
template ColumnPtr Column::indexImpl<UInt8>(const PaddedPODArray<UInt8>& indexes, \
size_t limit) const; \
Expand Down