Skip to content

branch-4.1: [Feature](multimodal_embed) Support multimodal(file) embed #66461

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 2 commits into from
Aug 7, 2026
Merged

branch-4.1: [Feature](multimodal_embed) Support multimodal(file) embed #66461

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5c0df0e
[Feature](multimodal_embed) Support multimodal(file) embed (#62147)
linrrzqqq Apr 14, 2026
66f1783
[Enhancement](ai_func) Skip Null inputs in AI functions (#66242)
linrrzqqq Aug 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
433 changes: 419 additions & 14 deletions be/src/exprs/function/ai/ai_adapter.h
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions be/src/exprs/function/ai/ai_classify.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,13 @@ class FunctionAIClassify : public AIFunction<FunctionAIClassify> {

static constexpr size_t number_of_arguments = 3;

DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
DataTypePtr get_nested_return_type_impl(const DataTypes& /*arguments*/) const {
return std::make_shared<DataTypeString>();
}

static FunctionPtr create() { return std::make_shared<FunctionAIClassify>(); }

Status build_prompt(const Block& block, const ColumnNumbers& arguments, size_t row_num,
Status build_prompt(const Columns& prompt_columns, size_t row_num,
std::string& prompt) const override;
};
} // namespace doris
4 changes: 2 additions & 2 deletions be/src/exprs/function/ai/ai_extract.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,13 @@ class FunctionAIExtract : public AIFunction<FunctionAIExtract> {

static constexpr size_t number_of_arguments = 3;

DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
DataTypePtr get_nested_return_type_impl(const DataTypes& /*arguments*/) const {
return std::make_shared<DataTypeString>();
}

static FunctionPtr create() { return std::make_shared<FunctionAIExtract>(); }

Status build_prompt(const Block& block, const ColumnNumbers& arguments, size_t row_num,
Status build_prompt(const Columns& prompt_columns, size_t row_num,
std::string& prompt) const override;
};

Expand Down
4 changes: 3 additions & 1 deletion be/src/exprs/function/ai/ai_filter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ class FunctionAIFilter : public AIFunction<FunctionAIFilter> {

static constexpr size_t number_of_arguments = 2;

DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
DataTypePtr get_nested_return_type_impl(const DataTypes& /*arguments*/) const {
return std::make_shared<DataTypeBool>();
}

Expand All @@ -47,6 +47,8 @@ class FunctionAIFilter : public AIFunction<FunctionAIFilter> {
private:
MutableColumnPtr create_result_column() const { return ColumnUInt8::create(); }

// AI_FILTER-private helper.
// Converts one parsed batch of string flags into BOOL results.
Status append_batch_results(const std::vector<std::string>& batch_results,
IColumn& col_result) const {
auto& bool_col = assert_cast<ColumnUInt8&>(col_result);
Expand Down
2 changes: 1 addition & 1 deletion be/src/exprs/function/ai/ai_fix_grammar.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ class FunctionAIFixGrammar : public AIFunction<FunctionAIFixGrammar> {

static constexpr size_t number_of_arguments = 2;

DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
DataTypePtr get_nested_return_type_impl(const DataTypes& /*arguments*/) const {
return std::make_shared<DataTypeString>();
}

Expand Down
164 changes: 52 additions & 112 deletions be/src/exprs/function/ai/ai_functions.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

#include "core/column/column_array.h"
#include "core/column/column_array_view.h"
#include "exprs/function/ai/ai_classify.h"
#include "exprs/function/ai/ai_extract.h"
#include "exprs/function/ai/ai_filter.h"
Expand All @@ -30,168 +30,108 @@
#include "exprs/function/simple_function_factory.h"

namespace doris {
Status FunctionAIClassify::build_prompt(const Block& block, const ColumnNumbers& arguments,
size_t row_num, std::string& prompt) const {
// Get the text column
const ColumnWithTypeAndName& text_column = block.get_by_position(arguments[1]);
StringRef text = text_column.column->get_data_at(row_num);
std::string text_str = std::string(text.data, text.size);

// Get the labels array column
const ColumnWithTypeAndName& labels_column = block.get_by_position(arguments[2]);
const auto& [array_column, array_row_num] =
check_column_const_set_readability(*labels_column.column, row_num);
const auto* col_array = check_and_get_column<ColumnArray>(*array_column);
if (col_array == nullptr) {
static Status format_labels(const ColumnPtr& labels_column, size_t row_num,
std::string_view function_name, std::string& labels_str) {
auto readable_column = check_column_const_set_readability(*labels_column, row_num);
if (!is_column<ColumnArray>(*readable_column.first)) {
return Status::InternalError(
"labels argument for {} must be Array(String) or Array(Varchar)", name);
}

std::vector<std::string> label_values;
const auto& data = col_array->get_data();
const auto& offsets = col_array->get_offsets();
size_t start = array_row_num > 0 ? offsets[array_row_num - 1] : 0;
size_t end = offsets[array_row_num];
for (size_t i = start; i < end; ++i) {
Field field;
data.get(i, field);
label_values.emplace_back(field.template get<TYPE_STRING>());
"labels argument for {} must be Array(String) or Array(Varchar)", function_name);
}

std::string labels_str = "[";
for (size_t i = 0; i < label_values.size(); ++i) {
if (i > 0) {
auto labels_view = ColumnArrayView<TYPE_STRING>::create(labels_column);
auto labels = labels_view[row_num];
labels_str = "[";
bool is_first_label = true;
for (size_t i = 0; i < labels.size(); ++i) {
if (labels.is_null_at(i)) {
continue;
}
if (!is_first_label) {
labels_str += ", ";
}
labels_str += "\"" + label_values[i] + "\"";
StringRef label = labels.value_at(i);
labels_str += "\"";
labels_str.append(label.data, label.size);
labels_str += "\"";
is_first_label = false;
}
labels_str += "]";
return Status::OK();
}

Status FunctionAIClassify::build_prompt(const Columns& prompt_columns, size_t row_num,
std::string& prompt) const {
// Get the text column
StringRef text = prompt_columns[0]->get_data_at(row_num);
std::string text_str = std::string(text.data, text.size);

std::string labels_str;
RETURN_IF_ERROR(format_labels(prompt_columns[1], row_num, name, labels_str));

prompt = "Labels: " + labels_str + "\nText: " + text_str;

return Status::OK();
}

Status FunctionAIExtract::build_prompt(const Block& block, const ColumnNumbers& arguments,
size_t row_num, std::string& prompt) const {
Status FunctionAIExtract::build_prompt(const Columns& prompt_columns, size_t row_num,
std::string& prompt) const {
// Get the text column
const ColumnWithTypeAndName& text_column = block.get_by_position(arguments[1]);
StringRef text = text_column.column->get_data_at(row_num);
StringRef text = prompt_columns[0]->get_data_at(row_num);
std::string text_str = std::string(text.data, text.size);

// Get the labels array column
const ColumnWithTypeAndName& labels_column = block.get_by_position(arguments[2]);
const auto& [array_column, array_row_num] =
check_column_const_set_readability(*labels_column.column, row_num);
const auto* col_array = check_and_get_column<ColumnArray>(*array_column);
if (col_array == nullptr) {
return Status::InternalError(
"labels argument for {} must be Array(String) or Array(Varchar)", name);
}

std::vector<std::string> label_values;
const auto& offsets = col_array->get_offsets();
const auto& data = col_array->get_data();
size_t start = array_row_num > 0 ? offsets[array_row_num - 1] : 0;
size_t end = offsets[array_row_num];
for (size_t i = start; i < end; ++i) {
Field field;
data.get(i, field);
label_values.emplace_back(field.template get<TYPE_STRING>());
}

std::string labels_str = "[";
for (size_t i = 0; i < label_values.size(); ++i) {
if (i > 0) {
labels_str += ", ";
}
labels_str += "\"" + label_values[i] + "\"";
}
labels_str += "]";
std::string labels_str;
RETURN_IF_ERROR(format_labels(prompt_columns[1], row_num, name, labels_str));

prompt = "Labels: " + labels_str + "\nText: " + text_str;

return Status::OK();
}

Status FunctionAIGenerate::build_prompt(const Block& block, const ColumnNumbers& arguments,
size_t row_num, std::string& prompt) const {
const ColumnWithTypeAndName& text_column = block.get_by_position(arguments[1]);
StringRef text_ref = text_column.column->get_data_at(row_num);
Status FunctionAIGenerate::build_prompt(const Columns& prompt_columns, size_t row_num,
std::string& prompt) const {
StringRef text_ref = prompt_columns[0]->get_data_at(row_num);
prompt = std::string(text_ref.data, text_ref.size);

return Status::OK();
}

Status FunctionAIMask::build_prompt(const Block& block, const ColumnNumbers& arguments,
size_t row_num, std::string& prompt) const {
Status FunctionAIMask::build_prompt(const Columns& prompt_columns, size_t row_num,
std::string& prompt) const {
// Get the text column
const ColumnWithTypeAndName& text_column = block.get_by_position(arguments[1]);
StringRef text = text_column.column->get_data_at(row_num);
StringRef text = prompt_columns[0]->get_data_at(row_num);
std::string text_str = std::string(text.data, text.size);

// Get the labels array column
const ColumnWithTypeAndName& labels_column = block.get_by_position(arguments[2]);
const auto& [array_column, array_row_num] =
check_column_const_set_readability(*labels_column.column, row_num);
const auto* col_array = check_and_get_column<ColumnArray>(*array_column);
if (col_array == nullptr) {
return Status::InternalError(
"labels argument for {} must be Array(String) or Array(Varchar)", name);
}

std::vector<std::string> label_values;
const auto& offsets = col_array->get_offsets();
const auto& data = col_array->get_data();
size_t start = array_row_num > 0 ? offsets[array_row_num - 1] : 0;
size_t end = offsets[array_row_num];
for (size_t i = start; i < end; ++i) {
Field field;
data.get(i, field);
label_values.emplace_back(field.template get<TYPE_STRING>());
}

std::string labels_str = "[";
for (size_t i = 0; i < label_values.size(); ++i) {
if (i > 0) {
labels_str += ", ";
}
labels_str += "\"" + label_values[i] + "\"";
}
labels_str += "]";
std::string labels_str;
RETURN_IF_ERROR(format_labels(prompt_columns[1], row_num, name, labels_str));

prompt = "Labels: " + labels_str + "\nText: " + text_str;

return Status::OK();
}

Status FunctionAISimilarity::build_prompt(const Block& block, const ColumnNumbers& arguments,
size_t row_num, std::string& prompt) const {
Status FunctionAISimilarity::build_prompt(const Columns& prompt_columns, size_t row_num,
std::string& prompt) const {
// text1
const ColumnWithTypeAndName& text_column_1 = block.get_by_position(arguments[1]);
StringRef text_1 = text_column_1.column.get()->get_data_at(row_num);
StringRef text_1 = prompt_columns[0]->get_data_at(row_num);
std::string text_str_1 = std::string(text_1.data, text_1.size);

// text2
const ColumnWithTypeAndName& text_column_2 = block.get_by_position(arguments[2]);
StringRef text_2 = text_column_2.column.get()->get_data_at(row_num);
StringRef text_2 = prompt_columns[1]->get_data_at(row_num);
std::string text_str_2 = std::string(text_2.data, text_2.size);

prompt = "Text 1: " + text_str_1 + "\nText 2: " + text_str_2;

return Status::OK();
}

Status FunctionAITranslate::build_prompt(const Block& block, const ColumnNumbers& arguments,
size_t row_num, std::string& prompt) const {
Status FunctionAITranslate::build_prompt(const Columns& prompt_columns, size_t row_num,
std::string& prompt) const {
// text
const ColumnWithTypeAndName& text_column = block.get_by_position(arguments[1]);
StringRef text = text_column.column.get()->get_data_at(row_num);
StringRef text = prompt_columns[0]->get_data_at(row_num);
std::string text_str = std::string(text.data, text.size);

// target language
const ColumnWithTypeAndName& lang_column = block.get_by_position(arguments[2]);
StringRef lang = lang_column.column.get()->get_data_at(row_num);
StringRef lang = prompt_columns[1]->get_data_at(row_num);
std::string target_lang = std::string(lang.data, lang.size);

prompt = "Translate the following text to " + target_lang + ".\nText: " + text_str;
Expand Down
Loading
Loading