Merge branch 'master' into fix-restore-with-async-replicated-database

.github/workflows/reusable_build_stage.yml

@@ -0,0 +1,32 @@
+### FIXME: merge reusable_test.yml and reusable_build.yml as they are almost identical
+#          and then merge reusable_build_stage.yml and reusable_test_stage.yml
+
+name: BuildStageWF
+'on':
+  workflow_call:
+    inputs:
+      stage:
+        description: stage name
+        type: string
+        required: true
+      data:
+        description: ci data
+        type: string
+        required: true
+
+jobs:
+  s:
+    if: ${{ !failure() && !cancelled() }}
+    strategy:
+      fail-fast: false
+      matrix:
+        job_name_and_runner_type: ${{ fromJson(inputs.data).stages_data[inputs.stage] }}
+    uses: ./.github/workflows/reusable_build.yml
+    with:
+      build_name: ${{ matrix.job_name_and_runner_type.job_name }}
+      runner_type: ${{ matrix.job_name_and_runner_type.runner_type }}
+      # don't forget to pass force flag (no ci cache/no reuse) - once it's needed
+      force: false
+      # for now let's do I deep checkout for builds
+      checkout_depth: 0
+      data: ${{ inputs.data }}

.github/workflows/reusable_test_stage.yml

@@ -0,0 +1,25 @@
+name: StageWF
+'on':
+  workflow_call:
+    inputs:
+      stage:
+        description: stage name
+        type: string
+        required: true
+      data:
+        description: ci data
+        type: string
+        required: true
+
+jobs:
+  s:
+    if: ${{ !failure() && !cancelled() }}
+    strategy:
+      fail-fast: false
+      matrix:
+        job_name_and_runner_type: ${{ fromJson(inputs.data).stages_data[inputs.stage] }}
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: ${{ matrix.job_name_and_runner_type.job_name }}
+      runner_type: ${{ matrix.job_name_and_runner_type.runner_type }}
+      data: ${{ inputs.data }}

docker/test/style/Dockerfile

@@ -18,7 +18,8 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
     python3-pip \
     yamllint \
     locales \
-    && pip3 install black==23.1.0 boto3 codespell==2.2.1 mypy==1.3.0 PyGithub unidiff pylint==2.6.2 \
+    && pip3 install black==23.12.0 boto3 codespell==2.2.1 mypy==1.8.0 PyGithub unidiff pylint==3.1.0 \
+        requests types-requests \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* \
     && rm -rf /root/.cache/pip

docker/test/upgrade/run.sh

@@ -79,6 +79,18 @@ remove_keeper_config "async_replication" "1"
 # create_if_not_exists feature flag doesn't exist on some older versions
 remove_keeper_config "create_if_not_exists" "[01]"
 
+#todo: remove these after 24.3 released.
+sudo cat /etc/clickhouse-server/config.d/azure_storage_conf.xml \
+  | sed "s|<object_storage_type>azure|<object_storage_type>azure_blob_storage|" \
+  > /etc/clickhouse-server/config.d/azure_storage_conf.xml.tmp
+sudo mv /etc/clickhouse-server/config.d/azure_storage_conf.xml.tmp /etc/clickhouse-server/config.d/azure_storage_conf.xml
+
+#todo: remove these after 24.3 released.
+sudo cat /etc/clickhouse-server/config.d/storage_conf.xml \
+  | sed "s|<object_storage_type>local|<object_storage_type>local_blob_storage|" \
+  > /etc/clickhouse-server/config.d/storage_conf.xml.tmp
+sudo mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml
+
 # latest_logs_cache_size_threshold setting doesn't exist on some older versions
 remove_keeper_config "latest_logs_cache_size_threshold" "[[:digit:]]\+"
 
@@ -113,6 +125,18 @@ sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \
   > /etc/clickhouse-server/config.d/keeper_port.xml.tmp
 sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml
 
+#todo: remove these after 24.3 released.
+sudo cat /etc/clickhouse-server/config.d/azure_storage_conf.xml \
+  | sed "s|<object_storage_type>azure|<object_storage_type>azure_blob_storage|" \
+  > /etc/clickhouse-server/config.d/azure_storage_conf.xml.tmp
+sudo mv /etc/clickhouse-server/config.d/azure_storage_conf.xml.tmp /etc/clickhouse-server/config.d/azure_storage_conf.xml
+
+#todo: remove these after 24.3 released.
+sudo cat /etc/clickhouse-server/config.d/storage_conf.xml \
+  | sed "s|<object_storage_type>local|<object_storage_type>local_blob_storage|" \
+  > /etc/clickhouse-server/config.d/storage_conf.xml.tmp
+sudo mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml
+
 # async_replication setting doesn't exist on some older versions
 remove_keeper_config "async_replication" "1"
 

docs/en/sql-reference/aggregate-functions/reference/grouparraysorted.md

@@ -14,8 +14,6 @@
 
  -   `N` – The number of elements to return.
 
- If the parameter is omitted, default value is the size of input.
-
  -   `column` – The value (Integer, String, Float and other Generic types).
 
  **Example**
@@ -36,13 +34,12 @@
  Gets all the String implementations of all numbers in column:
 
  ``` sql
-SELECT groupArraySorted(str) FROM (SELECT toString(number) as str FROM numbers(5));
+SELECT groupArraySorted(5)(str) FROM (SELECT toString(number) as str FROM numbers(5));
 
  ```
 
  ``` text
- ┌─groupArraySorted(str)────────┐
- │ ['0','1','2','3','4']        │
- └──────────────────────────────┘
- ```
-
+┌─groupArraySorted(5)(str)─┐
+│ ['0','1','2','3','4']    │
+└──────────────────────────┘
+ ```

programs/local/LocalServer.cpp

@@ -841,7 +841,7 @@ void LocalServer::addOptions(OptionsDescription & options_description)
 
         /// If structure argument is omitted then initial query is not generated
         ("structure,S", po::value<std::string>(), "structure of the initial table (list of column and type names)")
-        ("file,f", po::value<std::string>(), "path to file with data of the initial table (stdin if not specified)")
+        ("file,F", po::value<std::string>(), "path to file with data of the initial table (stdin if not specified)")
 
         ("input-format", po::value<std::string>(), "input format of the initial table data")
         ("output-format", po::value<std::string>(), "default output format")

.pylintrc → pyproject.toml

@@ -1,6 +1,4 @@
-# vim: ft=config
-
-[BASIC]
+[tool.pylint.BASIC]
 max-module-lines=2000
 # due to SQL
 max-line-length=200
@@ -9,11 +7,13 @@ max-branches=50
 max-nested-blocks=10
 max-statements=200
 
-[FORMAT]
-ignore-long-lines = (# )?<?https?://\S+>?$
+[tool.pylint.FORMAT]
+#ignore-long-lines = (# )?<?https?://\S+>?$
 
-[MESSAGES CONTROL]
-disable = missing-docstring,
+[tool.pylint.'MESSAGES CONTROL']
+          # pytest.mark.parametrize is not callable (not-callable)
+disable = '''
+          missing-docstring,
           too-few-public-methods,
           invalid-name,
           too-many-arguments,
@@ -26,18 +26,15 @@ disable = missing-docstring,
           wildcard-import,
           unused-wildcard-import,
           singleton-comparison,
-          # pytest.mark.parametrize is not callable (not-callable)
           not-callable,
-          # https://github.com/PyCQA/pylint/issues/3882
-          # [Python 3.9] Value 'Optional' is unsubscriptable (unsubscriptable-object) (also Union)
-          unsubscriptable-object,
-          # Drop them one day:
           redefined-outer-name,
           broad-except,
           bare-except,
           no-else-return,
           global-statement
+          '''
 
-[SIMILARITIES]
+[tool.pylint.SIMILARITIES]
 # due to SQL
 min-similarity-lines=1000
+

src/Analyzer/Passes/QueryAnalysisPass.cpp

@@ -6651,7 +6651,6 @@ void QueryAnalyzer::initializeTableExpressionData(const QueryTreeNodePtr & table
             if (column_default && column_default->kind == ColumnDefaultKind::Alias)
             {
                 auto alias_expression = buildQueryTree(column_default->expression, scope.context);
-                alias_expression = buildCastFunction(alias_expression, column_name_and_type.type, scope.context, false /*resolve*/);
                 auto column_node = std::make_shared<ColumnNode>(column_name_and_type, std::move(alias_expression), table_expression_node);
                 column_name_to_column_node.emplace(column_name_and_type.name, column_node);
                 alias_columns_to_resolve.emplace_back(column_name_and_type.name, column_node);
@@ -6684,7 +6683,9 @@ void QueryAnalyzer::initializeTableExpressionData(const QueryTreeNodePtr & table
                 alias_column_resolve_scope,
                 false /*allow_lambda_expression*/,
                 false /*allow_table_expression*/);
-
+            auto & resolved_expression = alias_column_to_resolve->getExpression();
+            if (!resolved_expression->getResultType()->equals(*alias_column_to_resolve->getResultType()))
+                resolved_expression = buildCastFunction(resolved_expression, alias_column_to_resolve->getResultType(), scope.context, true);
             column_name_to_column_node = std::move(alias_column_resolve_scope.column_name_to_column_node);
             column_name_to_column_node[alias_column_to_resolve_name] = alias_column_to_resolve;
         }

src/Backups/BackupsWorker.cpp

@@ -500,10 +500,11 @@ OperationID BackupsWorker::startMakingBackup(const ASTPtr & query, const Context
                  on_exception,
                  process_list_element_holder = process_list_element ? process_list_element->getProcessListEntry() : nullptr]
                 {
-                    CurrentThread::QueryScope query_scope(context_in_use);
+                    setThreadName("BackupWorker");
                     BackupMutablePtr backup_async;
                     try
                     {
+                        CurrentThread::QueryScope query_scope(context_in_use);
                         doBackup(
                             backup_async,
                             backup_query,
@@ -877,9 +878,10 @@ OperationID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePt
                  on_exception,
                  process_list_element_holder = process_list_element ? process_list_element->getProcessListEntry() : nullptr]
                 {
-                    CurrentThread::QueryScope query_scope(context_in_use);
+                    setThreadName("RestorerWorker");
                     try
                     {
+                        CurrentThread::QueryScope query_scope(context_in_use);
                         doRestore(
                             restore_query,
                             restore_id,