diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index ba7e35dd55d331..462c635e1f8b3a 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -214,10 +214,12 @@ Status RuntimeState::init(const TUniqueId& fragment_instance_id, const TQueryOpt const TQueryGlobals& query_globals, ExecEnv* exec_env) { _fragment_instance_id = fragment_instance_id; _query_options = query_options; + _lc_time_names = query_globals.lc_time_names; if (query_globals.__isset.time_zone && query_globals.__isset.nano_seconds) { _timezone = query_globals.time_zone; _timestamp_ms = query_globals.timestamp_ms; _nano_seconds = query_globals.nano_seconds; + } else if (query_globals.__isset.time_zone) { _timezone = query_globals.time_zone; _timestamp_ms = query_globals.timestamp_ms; diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index c2183c6dd59128..3d89f4aa0d4ed6 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -163,6 +163,7 @@ class RuntimeState { // if possible, use timezone_obj() rather than timezone() const std::string& timezone() const { return _timezone; } const cctz::time_zone& timezone_obj() const { return _timezone_obj; } + const std::string& lc_time_names() const { return _lc_time_names; } const std::string& user() const { return _user; } const TUniqueId& query_id() const { return _query_id; } const TUniqueId& fragment_instance_id() const { return _fragment_instance_id; } @@ -747,6 +748,7 @@ class RuntimeState { int32_t _nano_seconds; std::string _timezone; cctz::time_zone _timezone_obj; + std::string _lc_time_names; TUniqueId _query_id; // fragment id for each TPipelineFragmentParams diff --git a/be/src/vec/runtime/locale_value.cpp b/be/src/vec/runtime/locale_value.cpp new file mode 100644 index 00000000000000..c43442a19f1f56 --- /dev/null +++ b/be/src/vec/runtime/locale_value.cpp @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "locale_value.h" + +#include +#include +#include +#include + +namespace doris::vectorized { +namespace { + +constexpr size_t kMonthsPerYear = 12; +constexpr size_t kDaysPerWeek = 7; + +// The name arrays keep the trailing nullptr sentinel so the data layout mirrors +// the MySQL TYPELIB-based representation. +const char* month_names_en_US[kMonthsPerYear + 1] = { + "January", "February", "March", "April", "May", "June", "July", + "August", "September", "October", "November", "December", nullptr}; +const char* ab_month_names_en_US[kMonthsPerYear + 1] = {"Jan", "Feb", "Mar", "Apr", "May", + "Jun", "Jul", "Aug", "Sep", "Oct", + "Nov", "Dec", nullptr}; +const char* day_names_en_US[kDaysPerWeek + 1] = {"Monday", "Tuesday", "Wednesday", "Thursday", + "Friday", "Saturday", "Sunday", nullptr}; +const char* ab_day_names_en_US[kDaysPerWeek + 1] = {"Mon", "Tue", "Wed", "Thu", + "Fri", "Sat", "Sun", nullptr}; + +LocaleLib locale_lib_month_names_en_US { + .count = kMonthsPerYear, + .name = "month_names", + .type_name = month_names_en_US, +}; +LocaleLib locale_lib_ab_month_names_en_US { + .count = kMonthsPerYear, + .name = "ab_month_names", + .type_name = ab_month_names_en_US, +}; +LocaleLib locale_lib_day_names_en_US { + .count = kDaysPerWeek, + .name = "day_names", + .type_name = day_names_en_US, +}; +LocaleLib locale_lib_ab_day_names_en_US { + .count = kDaysPerWeek, + .name = "ab_day_names", + .type_name = ab_day_names_en_US, +}; + +inline bool iequals(const char* lhs, std::size_t lhs_len, const char* rhs) { + if (lhs == nullptr || rhs == nullptr) { + return false; + } + const std::size_t rhs_len = std::strlen(rhs); + if (lhs_len != rhs_len) { + return false; + } + for (std::size_t i = 0; i < rhs_len; ++i) { + const auto l = static_cast(lhs[i]); + const auto r = static_cast(rhs[i]); + if (std::tolower(l) != std::tolower(r)) { + return false; + } + } + return true; +} + +} // namespace + +LocaleValue locale_en_US { + .number = 0, + .name = "en_US", + .description = "English - United States", + .is_ascii = true, + .month_names = &locale_lib_month_names_en_US, + .ab_month_names = &locale_lib_ab_month_names_en_US, + .day_names = &locale_lib_day_names_en_US, + .ab_day_names = &locale_lib_ab_day_names_en_US, +}; + +LocaleValue* locale_values[] = {&locale_en_US, nullptr}; + +LocaleValue* default_locale_value = &locale_en_US; + +LocaleValue* locale_value_by_name(const char* name, std::size_t length) { + if (name == nullptr) { + return nullptr; + } + for (LocaleValue** cursor = locale_values; *cursor != nullptr; ++cursor) { + if (iequals(name, length, (*cursor)->name)) { + return *cursor; + } + } + return nullptr; +} + +LocaleValue* locale_value_by_number(std::uint32_t number) { + for (LocaleValue** cursor = locale_values; *cursor != nullptr; ++cursor) { + if ((*cursor)->number == number) { + return *cursor; + } + } + return nullptr; +} + +} // namespace doris::vectorized diff --git a/be/src/vec/runtime/locale_value.h b/be/src/vec/runtime/locale_value.h new file mode 100644 index 00000000000000..a23ca6b8460f98 --- /dev/null +++ b/be/src/vec/runtime/locale_value.h @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +namespace doris::vectorized { +#include "common/compile_check_begin.h" + +struct LocaleLib { + size_t count; + const char* name; + const char** type_name; +}; + +struct LocaleValue { + uint32_t number; + const char* name; + const char* description; + bool is_ascii; + + LocaleLib* month_names; + LocaleLib* ab_month_names; + LocaleLib* day_names; + LocaleLib* ab_day_names; +}; + +#include "common/compile_check_end.h" + +extern LocaleValue locale_en_US; +extern LocaleValue* locale_values[]; +extern LocaleValue* default_locale_value; + +LocaleValue* locale_value_by_name(const char* name, std::size_t length); +LocaleValue* locale_value_by_number(std::uint32_t number); + +} // namespace doris::vectorized diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadTaskInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadTaskInfo.java index 0d15e3d9086427..0483260a354928 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadTaskInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadTaskInfo.java @@ -46,6 +46,10 @@ public interface NereidsLoadTaskInfo { String getTimezone(); + // default String getLcTimeNames() { + // return "en_US"; + // } + PartitionNames getPartitions(); LoadTask.MergeType getMergeType(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadTask.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadTask.java index a0cfa57b19e747..eb4210b79c30a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadTask.java @@ -70,6 +70,7 @@ public class NereidsStreamLoadTask implements NereidsLoadTaskInfo { private boolean negative; private boolean strictMode = false; // default is false private String timezone = TimeUtils.DEFAULT_TIME_ZONE; + // private String lcTimeNames = "en_US"; private int timeout = Config.stream_load_default_timeout_second; private long execMemLimit = 2 * 1024 * 1024 * 1024L; // default is 2GB private LoadTask.MergeType mergeType = LoadTask.MergeType.APPEND; // default is all data is load no delete @@ -260,6 +261,19 @@ public String getJsonRoot() { return jsonRoot; } + // @Override + // public String getLcTimeNames() { + // return lcTimeNames; + // } + + // public void setLcTimeNames(String lcTimeNames) { + // if (Strings.isNullOrEmpty(lcTimeNames)) { + // this.lcTimeNames = "en_US"; + // } else { + // this.lcTimeNames = lcTimeNames; + // } + // } + public void setJsonRoot(String jsonRoot) { this.jsonRoot = jsonRoot; } @@ -381,6 +395,9 @@ public void setMultiTableBaseTaskInfo(LoadTaskInfo task) throws UserException { this.jsonRoot = task.getJsonRoot(); this.sendBatchParallelism = task.getSendBatchParallelism(); this.loadToSingleTablet = task.isLoadToSingleTablet(); + // if (task instanceof NereidsLoadTaskInfo) { + // setLcTimeNames(((NereidsLoadTaskInfo) task).getLcTimeNames()); + // } } private void setOptionalFromTSLPutRequest(TStreamLoadPutRequest request) throws UserException { @@ -438,6 +455,9 @@ private void setOptionalFromTSLPutRequest(TStreamLoadPutRequest request) throws } else if (ConnectContext.get() != null) { timezone = ConnectContext.get().getSessionVariable().getTimeZone(); } + // if (request.isSetLcTimeNames()) { + // setLcTimeNames(request.getLcTimeNames()); + // } if (request.isSetExecMemLimit()) { execMemLimit = request.getExecMemLimit(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index 27f2348f59c687..9d44f15384e08d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -358,6 +358,7 @@ public Coordinator(ConnectContext context, Planner planner) { } else { this.queryGlobals.setTimeZone(context.getSessionVariable().getTimeZone()); } + this.queryGlobals.setLcTimeNames(context.getSessionVariable().getLcTimeNames()); this.assignedRuntimeFilters = planner.getRuntimeFilters(); this.topnFilters = planner.getTopnFilters(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java index 0fbcb5690fba17..ad92f82458b857 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java @@ -312,6 +312,12 @@ public static CoordinatorContext buildForLoad( queryGlobals.setTimestampMs(System.currentTimeMillis()); queryGlobals.setTimeZone(timezone); queryGlobals.setLoadZeroTolerance(loadZeroTolerance); + ConnectContext currentCtx = ConnectContext.get(); + if (currentCtx != null && currentCtx.getSessionVariable() != null) { + queryGlobals.setLcTimeNames(currentCtx.getSessionVariable().getLcTimeNames()); + } else { + queryGlobals.setLcTimeNames("en_US"); + } ExecutionProfile executionProfile = new ExecutionProfile( queryId, @@ -351,6 +357,7 @@ private static TQueryGlobals initQueryGlobals(ConnectContext context) { } else { queryGlobals.setTimeZone(context.getSessionVariable().getTimeZone()); } + queryGlobals.setLcTimeNames(context.getSessionVariable().getLcTimeNames()); return queryGlobals; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 6381773d09fec6..e63a0dd2426c69 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -52,6 +52,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -67,6 +68,7 @@ import java.io.IOException; import java.io.Serializable; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; import java.security.InvalidParameterException; import java.security.SecureRandom; import java.time.format.DateTimeFormatter; @@ -74,6 +76,8 @@ import java.util.Arrays; import java.util.BitSet; import java.util.HashMap; +import java.util.IllformedLocaleException; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -131,6 +135,31 @@ public class SessionVariable implements Serializable, Writable { public static final String NET_WRITE_TIMEOUT = "net_write_timeout"; public static final String NET_READ_TIMEOUT = "net_read_timeout"; public static final String TIME_ZONE = "time_zone"; + public static final String LC_TIME_NAMES = "lc_time_names"; + private static final String DEFAULT_LC_TIME_NAMES = "en_US"; + private static final Map SUPPORTED_LC_TIME_NAMES; + + static { + Map locales = new LinkedHashMap<>(); + locales.put("en_US", "English - United States"); + locales.put("de_DE", "German - Germany"); + locales.put("fr_FR", "French - France"); + locales.put("it_IT", "Italian - Italy"); + locales.put("ja_JP", "Japanese - Japan"); + locales.put("nl_NL", "Dutch - Netherlands"); + locales.put("da_DK", "Danish - Denmark"); + locales.put("es_ES", "Spanish - Spain"); + locales.put("pt_PT", "Portuguese - Portugal"); + locales.put("sv_SE", "Swedish - Sweden"); + locales.put("cs_CZ", "Czech - Czech Republic"); + locales.put("hu_HU", "Hungarian - Hungary"); + locales.put("pl_PL", "Polish - Poland"); + locales.put("ru_RU", "Russian - Russia"); + locales.put("sk_SK", "Slovak - Slovakia"); + locales.put("uk_UA", "Ukrainian - Ukraine"); + SUPPORTED_LC_TIME_NAMES = ImmutableMap.copyOf(locales); + } + public static final String SQL_SAFE_UPDATES = "sql_safe_updates"; public static final String NET_BUFFER_LENGTH = "net_buffer_length"; public static final String HAVE_QUERY_CACHE = "have_query_cache"; @@ -1126,6 +1155,10 @@ public void checkQuerySlotCount(String slotCnt) { @VariableMgr.VarAttr(name = TIME_ZONE, needForward = true, affectQueryResult = true) public String timeZone = TimeUtils.getSystemTimeZone().getID(); + @VariableMgr.VarAttr(name = LC_TIME_NAMES, needForward = true, affectQueryResult = true, + setter = "setLcTimeNames") + private String lcTimeNames = DEFAULT_LC_TIME_NAMES; + @VariableMgr.VarAttr(name = PARALLEL_EXCHANGE_INSTANCE_NUM) public int exchangeInstanceParallel = 100; @@ -3468,6 +3501,51 @@ public void setTimeZone(String timeZone) { this.timeZone = timeZone; } + private static String canonicalizeLcTimeNames(String value) { + String trimmed = value.trim(); + if (trimmed.isEmpty()) { + throw new InvalidParameterException("lc_time_names value is empty"); + } + String normalized = trimmed.replace('-', '_'); + String[] segments = normalized.split("_"); + if (segments.length != 2) { + throw new InvalidParameterException( + "lc_time_names value must be in language_COUNTRY form: " + value); + } + String language = segments[0].toLowerCase(Locale.ROOT); + String country = segments[1].toUpperCase(Locale.ROOT); + try { + new Locale.Builder().setLanguage(language).setRegion(country).build(); + } catch (IllformedLocaleException e) { + throw new InvalidParameterException("lc_time_names value is invalid: " + value); + } + return language + "_" + country; + } + + public String getLcTimeNames() { + return lcTimeNames; + } + + public void setLcTimeNames(String lcTimeNames) { + if (Strings.isNullOrEmpty(lcTimeNames)) { + this.lcTimeNames = DEFAULT_LC_TIME_NAMES; + return; + } + if ("default".equalsIgnoreCase(lcTimeNames) || "null".equalsIgnoreCase(lcTimeNames)) { + this.lcTimeNames = DEFAULT_LC_TIME_NAMES; + return; + } + String canonical = canonicalizeLcTimeNames(lcTimeNames); + if (!SUPPORTED_LC_TIME_NAMES.containsKey(canonical)) { + throw new InvalidParameterException("Unsupported lc_time_names value: " + lcTimeNames); + } + this.lcTimeNames = canonical; + } + + public Map getSupportedLcTimeNames() { + return SUPPORTED_LC_TIME_NAMES; + } + public int getSqlSafeUpdates() { return sqlSafeUpdates; } @@ -4858,6 +4936,16 @@ public void setForwardedSessionVariables(Map variables) { LOG.debug("set forward variable: {} = {}", varAttr.name(), val); } + if (!Strings.isNullOrEmpty(varAttr.setter())) { + try { + SessionVariable.class.getDeclaredMethod(varAttr.setter(), String.class) + .invoke(this, val); + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + LOG.warn("failed to set forward variable {} via setter", varAttr.name(), e); + } + continue; + } + // set config field switch (f.getType().getSimpleName()) { case "short": diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java index 8319c35b02f3da..86efb25dc24270 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java @@ -25,6 +25,7 @@ import org.junit.jupiter.api.Test; import java.lang.reflect.Field; +import java.security.InvalidParameterException; import java.util.Map; public class SessionVariablesTest extends TestWithFeService { @@ -58,8 +59,18 @@ public void testForwardSessionVariables() { Assertions.assertEquals(numOfForwardVars, vars.size()); vars.put(SessionVariable.ENABLE_PROFILE, "true"); + vars.put(SessionVariable.LC_TIME_NAMES, "EN-us"); sessionVariable.setForwardedSessionVariables(vars); Assertions.assertTrue(sessionVariable.enableProfile); + Assertions.assertEquals("en_US", sessionVariable.getLcTimeNames()); + } + + @Test + public void testLcTimeNamesSetter() { + sessionVariable.setLcTimeNames("En-us"); + Assertions.assertEquals("en_US", sessionVariable.getLcTimeNames()); + Assertions.assertThrows(InvalidParameterException.class, + () -> sessionVariable.setLcTimeNames("xx")); } @Test diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index f0399df51ecfcb..ce58c9f0fe759e 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -555,6 +555,8 @@ struct TStreamLoadPutRequest { 58: optional Descriptors.TPartialUpdateNewRowPolicy partial_update_new_key_policy 59: optional bool empty_field_as_null + // 60: optional string lc_time_names + // For cloud 1000: optional string cloud_cluster 1001: optional i64 table_id diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index d193afe412f804..4209fcef7362d8 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -487,6 +487,9 @@ struct TQueryGlobals { 4: optional bool load_zero_tolerance = false 5: optional i32 nano_seconds + + // Locale name used for month/day names formatting, e.g. en_US + 6: optional string lc_time_names }