Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions be/src/runtime/runtime_state.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -214,10 +214,12 @@ Status RuntimeState::init(const TUniqueId& fragment_instance_id, const TQueryOpt
const TQueryGlobals& query_globals, ExecEnv* exec_env) {
_fragment_instance_id = fragment_instance_id;
_query_options = query_options;
_lc_time_names = query_globals.lc_time_names;
if (query_globals.__isset.time_zone && query_globals.__isset.nano_seconds) {
_timezone = query_globals.time_zone;
_timestamp_ms = query_globals.timestamp_ms;
_nano_seconds = query_globals.nano_seconds;

} else if (query_globals.__isset.time_zone) {
_timezone = query_globals.time_zone;
_timestamp_ms = query_globals.timestamp_ms;
Expand Down
2 changes: 2 additions & 0 deletions be/src/runtime/runtime_state.h
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@ class RuntimeState {
// if possible, use timezone_obj() rather than timezone()
const std::string& timezone() const { return _timezone; }
const cctz::time_zone& timezone_obj() const { return _timezone_obj; }
const std::string& lc_time_names() const { return _lc_time_names; }
const std::string& user() const { return _user; }
const TUniqueId& query_id() const { return _query_id; }
const TUniqueId& fragment_instance_id() const { return _fragment_instance_id; }
Expand Down Expand Up @@ -747,6 +748,7 @@ class RuntimeState {
int32_t _nano_seconds;
std::string _timezone;
cctz::time_zone _timezone_obj;
std::string _lc_time_names;

TUniqueId _query_id;
// fragment id for each TPipelineFragmentParams
Expand Down
121 changes: 121 additions & 0 deletions be/src/vec/runtime/locale_value.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include "locale_value.h"

#include <cctype>
#include <cstddef>
#include <cstdint>
#include <cstring>

namespace doris::vectorized {
namespace {

constexpr size_t kMonthsPerYear = 12;
constexpr size_t kDaysPerWeek = 7;

// The name arrays keep the trailing nullptr sentinel so the data layout mirrors
// the MySQL TYPELIB-based representation.
const char* month_names_en_US[kMonthsPerYear + 1] = {
"January", "February", "March", "April", "May", "June", "July",
"August", "September", "October", "November", "December", nullptr};
const char* ab_month_names_en_US[kMonthsPerYear + 1] = {"Jan", "Feb", "Mar", "Apr", "May",
"Jun", "Jul", "Aug", "Sep", "Oct",
"Nov", "Dec", nullptr};
const char* day_names_en_US[kDaysPerWeek + 1] = {"Monday", "Tuesday", "Wednesday", "Thursday",
"Friday", "Saturday", "Sunday", nullptr};
const char* ab_day_names_en_US[kDaysPerWeek + 1] = {"Mon", "Tue", "Wed", "Thu",
"Fri", "Sat", "Sun", nullptr};

LocaleLib locale_lib_month_names_en_US {
.count = kMonthsPerYear,
.name = "month_names",
.type_name = month_names_en_US,
};
LocaleLib locale_lib_ab_month_names_en_US {
.count = kMonthsPerYear,
.name = "ab_month_names",
.type_name = ab_month_names_en_US,
};
LocaleLib locale_lib_day_names_en_US {
.count = kDaysPerWeek,
.name = "day_names",
.type_name = day_names_en_US,
};
LocaleLib locale_lib_ab_day_names_en_US {
.count = kDaysPerWeek,
.name = "ab_day_names",
.type_name = ab_day_names_en_US,
};

inline bool iequals(const char* lhs, std::size_t lhs_len, const char* rhs) {
if (lhs == nullptr || rhs == nullptr) {
return false;
}
const std::size_t rhs_len = std::strlen(rhs);
if (lhs_len != rhs_len) {
return false;
}
for (std::size_t i = 0; i < rhs_len; ++i) {
const auto l = static_cast<unsigned char>(lhs[i]);
const auto r = static_cast<unsigned char>(rhs[i]);
if (std::tolower(l) != std::tolower(r)) {
return false;
}
}
return true;
}

} // namespace

LocaleValue locale_en_US {
.number = 0,
.name = "en_US",
.description = "English - United States",
.is_ascii = true,
.month_names = &locale_lib_month_names_en_US,
.ab_month_names = &locale_lib_ab_month_names_en_US,
.day_names = &locale_lib_day_names_en_US,
.ab_day_names = &locale_lib_ab_day_names_en_US,
};

LocaleValue* locale_values[] = {&locale_en_US, nullptr};

LocaleValue* default_locale_value = &locale_en_US;

LocaleValue* locale_value_by_name(const char* name, std::size_t length) {
if (name == nullptr) {
return nullptr;
}
for (LocaleValue** cursor = locale_values; *cursor != nullptr; ++cursor) {
if (iequals(name, length, (*cursor)->name)) {
return *cursor;
}
}
return nullptr;
}

LocaleValue* locale_value_by_number(std::uint32_t number) {
for (LocaleValue** cursor = locale_values; *cursor != nullptr; ++cursor) {
if ((*cursor)->number == number) {
return *cursor;
}
}
return nullptr;
}

} // namespace doris::vectorized
53 changes: 53 additions & 0 deletions be/src/vec/runtime/locale_value.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#pragma once

#include <cstddef>
#include <cstdint>

namespace doris::vectorized {
#include "common/compile_check_begin.h"

struct LocaleLib {
size_t count;
const char* name;
const char** type_name;
};

struct LocaleValue {
uint32_t number;
const char* name;
const char* description;
bool is_ascii;

LocaleLib* month_names;
LocaleLib* ab_month_names;
LocaleLib* day_names;
LocaleLib* ab_day_names;
};

#include "common/compile_check_end.h"

extern LocaleValue locale_en_US;
extern LocaleValue* locale_values[];
extern LocaleValue* default_locale_value;

LocaleValue* locale_value_by_name(const char* name, std::size_t length);
LocaleValue* locale_value_by_number(std::uint32_t number);

} // namespace doris::vectorized
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ public interface NereidsLoadTaskInfo {

String getTimezone();

// default String getLcTimeNames() {
// return "en_US";
// }

PartitionNames getPartitions();

LoadTask.MergeType getMergeType();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ public class NereidsStreamLoadTask implements NereidsLoadTaskInfo {
private boolean negative;
private boolean strictMode = false; // default is false
private String timezone = TimeUtils.DEFAULT_TIME_ZONE;
// private String lcTimeNames = "en_US";
private int timeout = Config.stream_load_default_timeout_second;
private long execMemLimit = 2 * 1024 * 1024 * 1024L; // default is 2GB
private LoadTask.MergeType mergeType = LoadTask.MergeType.APPEND; // default is all data is load no delete
Expand Down Expand Up @@ -260,6 +261,19 @@ public String getJsonRoot() {
return jsonRoot;
}

// @Override
// public String getLcTimeNames() {
// return lcTimeNames;
// }

// public void setLcTimeNames(String lcTimeNames) {
// if (Strings.isNullOrEmpty(lcTimeNames)) {
// this.lcTimeNames = "en_US";
// } else {
// this.lcTimeNames = lcTimeNames;
// }
// }

public void setJsonRoot(String jsonRoot) {
this.jsonRoot = jsonRoot;
}
Expand Down Expand Up @@ -381,6 +395,9 @@ public void setMultiTableBaseTaskInfo(LoadTaskInfo task) throws UserException {
this.jsonRoot = task.getJsonRoot();
this.sendBatchParallelism = task.getSendBatchParallelism();
this.loadToSingleTablet = task.isLoadToSingleTablet();
// if (task instanceof NereidsLoadTaskInfo) {
// setLcTimeNames(((NereidsLoadTaskInfo) task).getLcTimeNames());
// }
}

private void setOptionalFromTSLPutRequest(TStreamLoadPutRequest request) throws UserException {
Expand Down Expand Up @@ -438,6 +455,9 @@ private void setOptionalFromTSLPutRequest(TStreamLoadPutRequest request) throws
} else if (ConnectContext.get() != null) {
timezone = ConnectContext.get().getSessionVariable().getTimeZone();
}
// if (request.isSetLcTimeNames()) {
// setLcTimeNames(request.getLcTimeNames());
// }
if (request.isSetExecMemLimit()) {
execMemLimit = request.getExecMemLimit();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,7 @@ public Coordinator(ConnectContext context, Planner planner) {
} else {
this.queryGlobals.setTimeZone(context.getSessionVariable().getTimeZone());
}
this.queryGlobals.setLcTimeNames(context.getSessionVariable().getLcTimeNames());
this.assignedRuntimeFilters = planner.getRuntimeFilters();
this.topnFilters = planner.getTopnFilters();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,12 @@ public static CoordinatorContext buildForLoad(
queryGlobals.setTimestampMs(System.currentTimeMillis());
queryGlobals.setTimeZone(timezone);
queryGlobals.setLoadZeroTolerance(loadZeroTolerance);
ConnectContext currentCtx = ConnectContext.get();
if (currentCtx != null && currentCtx.getSessionVariable() != null) {
queryGlobals.setLcTimeNames(currentCtx.getSessionVariable().getLcTimeNames());
} else {
queryGlobals.setLcTimeNames("en_US");
}

ExecutionProfile executionProfile = new ExecutionProfile(
queryId,
Expand Down Expand Up @@ -351,6 +357,7 @@ private static TQueryGlobals initQueryGlobals(ConnectContext context) {
} else {
queryGlobals.setTimeZone(context.getSessionVariable().getTimeZone());
}
queryGlobals.setLcTimeNames(context.getSessionVariable().getLcTimeNames());
return queryGlobals;
}

Expand Down
Loading